diff --git a/SOURCES/069-ipc-refactor.patch b/SOURCES/069-ipc-refactor.patch
new file mode 100644
index 0000000..22e8fda
--- /dev/null
+++ b/SOURCES/069-ipc-refactor.patch
@@ -0,0 +1,126 @@
+From 769c210a4e9494757456c2896fdf67e2eb4b76c1 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Fri, 9 Jun 2017 10:49:07 -0500
+Subject: [PATCH] Refactor: libcrmcommon,cib,lrmd: functionize allocating IPC
+ client object
+
+reduces duplication
+---
+ cib/remote.c | 6 +-----
+ include/crm/common/ipcs.h | 1 +
+ lib/common/ipc.c | 25 +++++++++++++++++++------
+ lrmd/tls_backend.c | 5 +----
+ 4 files changed, 22 insertions(+), 15 deletions(-)
+
+diff --git a/cib/remote.c b/cib/remote.c
+index 9011552..0160c7e 100644
+--- a/cib/remote.c
++++ b/cib/remote.c
+@@ -325,13 +325,9 @@ cib_remote_listen(gpointer data)
+ num_clients++;
+
+ crm_client_init();
+- new_client = calloc(1, sizeof(crm_client_t));
++ new_client = crm_client_alloc(NULL);
+ new_client->remote = calloc(1, sizeof(crm_remote_t));
+
+- new_client->id = crm_generate_uuid();
+-
+- g_hash_table_insert(client_connections, new_client->id /* Should work */ , new_client);
+-
+ if (ssock == remote_tls_fd) {
+ #ifdef HAVE_GNUTLS_GNUTLS_H
+ new_client->kind = CRM_CLIENT_TLS;
+diff --git a/include/crm/common/ipcs.h b/include/crm/common/ipcs.h
+index ba1ccef..06cade9 100644
+--- a/include/crm/common/ipcs.h
++++ b/include/crm/common/ipcs.h
+@@ -105,6 +105,7 @@ crm_client_t *crm_client_get(qb_ipcs_connection_t * c);
+ crm_client_t *crm_client_get_by_id(const char *id);
+ const char *crm_client_name(crm_client_t * c);
+
++crm_client_t *crm_client_alloc(void *key);
+ crm_client_t *crm_client_new(qb_ipcs_connection_t * c, uid_t uid, gid_t gid);
+ void crm_client_destroy(crm_client_t * c);
+ void crm_client_disconnect_all(qb_ipcs_service_t *s);
+diff --git a/lib/common/ipc.c b/lib/common/ipc.c
+index d32e373..efe6480 100644
+--- a/lib/common/ipc.c
++++ b/lib/common/ipc.c
+@@ -293,6 +293,24 @@ crm_client_disconnect_all(qb_ipcs_service_t *service)
+ }
+ }
+
++/*!
++ * \brief Allocate a new crm_client_t object and generate its ID
++ *
++ * \param[in] key What to use as connections hash table key (NULL to use ID)
++ *
++ * \return Pointer to new crm_client_t (asserts on failure)
++ */
++crm_client_t *
++crm_client_alloc(void *key)
++{
++ crm_client_t *client = calloc(1, sizeof(crm_client_t));
++
++ CRM_ASSERT(client != NULL);
++ client->id = crm_generate_uuid();
++ g_hash_table_insert(client_connections, (key? key : client->id), client);
++ return client;
++}
++
+ crm_client_t *
+ crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client)
+ {
+@@ -324,21 +342,16 @@ crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client)
+ crm_client_init();
+
+ /* TODO: Do our own auth checking, return NULL if unauthorized */
+- client = calloc(1, sizeof(crm_client_t));
+-
++ client = crm_client_alloc(c);
+ client->ipcs = c;
+ client->kind = CRM_CLIENT_IPC;
+ client->pid = crm_ipcs_client_pid(c);
+
+- client->id = crm_generate_uuid();
+-
+ crm_debug("Connecting %p for uid=%d gid=%d pid=%u id=%s", c, uid_client, gid_client, client->pid, client->id);
+
+ #if ENABLE_ACL
+ client->user = uid2username(uid_client);
+ #endif
+-
+- g_hash_table_insert(client_connections, c, client);
+ return client;
+ }
+
+diff --git a/lrmd/tls_backend.c b/lrmd/tls_backend.c
+index 8c36434..7d790cf 100644
+--- a/lrmd/tls_backend.c
++++ b/lrmd/tls_backend.c
+@@ -212,11 +212,10 @@ lrmd_remote_listen(gpointer data)
+ return TRUE;
+ }
+
+- new_client = calloc(1, sizeof(crm_client_t));
++ new_client = crm_client_alloc(NULL);
+ new_client->remote = calloc(1, sizeof(crm_remote_t));
+ new_client->kind = CRM_CLIENT_TLS;
+ new_client->remote->tls_session = session;
+- new_client->id = crm_generate_uuid();
+ new_client->remote->auth_timeout =
+ g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client);
+ crm_notice("LRMD client connection established. %p id: %s", new_client, new_client->id);
+@@ -224,8 +223,6 @@ lrmd_remote_listen(gpointer data)
+ new_client->remote->source =
+ mainloop_add_fd("lrmd-remote-client", G_PRIORITY_DEFAULT, csock, new_client,
+ &lrmd_remote_fd_cb);
+- g_hash_table_insert(client_connections, new_client->id, new_client);
+-
+ return TRUE;
+ }
+
+--
+1.8.3.1
+
diff --git a/SOURCES/070-check_positive_number.patch b/SOURCES/070-check_positive_number.patch
new file mode 100644
index 0000000..7022a7a
--- /dev/null
+++ b/SOURCES/070-check_positive_number.patch
@@ -0,0 +1,45 @@
+From 52153c502ad12c723231f5c7cb80d50a5d0691cd Mon Sep 17 00:00:00 2001
+From: aravind-kumar <karavindkumar1993@gmail.com>
+Date: Wed, 15 Mar 2017 22:46:42 +0530
+Subject: [PATCH] check_positive_number
+
+---
+ include/crm_internal.h | 1 +
+ lib/common/utils.c | 9 +++++++++
+ 2 files changed, 10 insertions(+), 0 deletions(-)
+
+diff --git a/include/crm_internal.h b/include/crm_internal.h
+index 96a68bd..31d4efe 100644
+--- a/include/crm_internal.h
++++ b/include/crm_internal.h
+@@ -126,6 +126,7 @@ gboolean check_time(const char *value);
+ gboolean check_timer(const char *value);
+ gboolean check_boolean(const char *value);
+ gboolean check_number(const char *value);
++gboolean check_positive_number(const char *value);
+ gboolean check_quorum(const char *value);
+ gboolean check_script(const char *value);
+ gboolean check_utilization(const char *value);
+diff --git a/lib/common/utils.c b/lib/common/utils.c
+index dbf84c0..32c004d 100644
+--- a/lib/common/utils.c
++++ b/lib/common/utils.c
+@@ -162,6 +162,15 @@ check_number(const char *value)
+ }
+
+ gboolean
++check_positive_number(const char* value)
++{
++ if (check_number(value) && (value[0] != '-') && !(safe_str_eq(value,"0"))) {
++ return TRUE;
++ }
++ return FALSE;
++}
++
++gboolean
+ check_quorum(const char *value)
+ {
+ if (safe_str_eq(value, "stop")) {
+--
+1.8.3.1
+
diff --git a/SOURCES/071-cluster-ipc-limit.patch b/SOURCES/071-cluster-ipc-limit.patch
new file mode 100644
index 0000000..7a0e350
--- /dev/null
+++ b/SOURCES/071-cluster-ipc-limit.patch
@@ -0,0 +1,517 @@
+From ae780515cd4db1e6f23db9f75a628ce9c39bdd49 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Fri, 9 Jun 2017 12:35:36 -0500
+Subject: [PATCH 1/8] Refactor: libcrmcommon: remember when IPC client is root
+ or cluster user
+
+will allow using a different eviction threshold
+---
+ include/crm/common/ipcs.h | 3 ++-
+ lib/common/ipc.c | 13 ++++++++++---
+ 2 files changed, 12 insertions(+), 4 deletions(-)
+
+diff --git a/include/crm/common/ipcs.h b/include/crm/common/ipcs.h
+index 52338e3..43b7b60 100644
+--- a/include/crm/common/ipcs.h
++++ b/include/crm/common/ipcs.h
+@@ -60,7 +60,8 @@ struct crm_remote_s {
+
+ enum crm_client_flags
+ {
+- crm_client_flag_ipc_proxied = 0x00001, /* ipc_proxy code only */
++ crm_client_flag_ipc_proxied = 0x00001, /* ipc_proxy code only */
++ crm_client_flag_ipc_privileged = 0x00002, /* root or cluster user */
+ };
+
+ struct crm_client_s {
+diff --git a/lib/common/ipc.c b/lib/common/ipc.c
+index 50980ec..e0a7a5c 100644
+--- a/lib/common/ipc.c
++++ b/lib/common/ipc.c
+@@ -314,6 +314,7 @@ crm_client_alloc(void *key)
+ crm_client_t *
+ crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client)
+ {
++ static gid_t uid_cluster = 0;
+ static gid_t gid_cluster = 0;
+
+ crm_client_t *client = NULL;
+@@ -323,11 +324,12 @@ crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client)
+ return NULL;
+ }
+
+- if (gid_cluster == 0) {
+- if(crm_user_lookup(CRM_DAEMON_USER, NULL, &gid_cluster) < 0) {
++ if (uid_cluster == 0) {
++ if (crm_user_lookup(CRM_DAEMON_USER, &uid_cluster, &gid_cluster) < 0) {
+ static bool have_error = FALSE;
+ if(have_error == FALSE) {
+- crm_warn("Could not find group for user %s", CRM_DAEMON_USER);
++ crm_warn("Could not find user and group IDs for user %s",
++ CRM_DAEMON_USER);
+ have_error = TRUE;
+ }
+ }
+@@ -347,6 +349,11 @@ crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client)
+ client->kind = CRM_CLIENT_IPC;
+ client->pid = crm_ipcs_client_pid(c);
+
++ if ((uid_client == 0) || (uid_client == uid_cluster)) {
++ /* Remember when a connection came from root or hacluster */
++ set_bit(client->flags, crm_client_flag_ipc_privileged);
++ }
++
+ crm_debug("Connecting %p for uid=%d gid=%d pid=%u id=%s", c, uid_client, gid_client, client->pid, client->id);
+
+ #if ENABLE_ACL
+--
+1.8.3.1
+
+
+From f93ce6bdd6be5d2670ab0cd8dd10d3b8b9972a65 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Fri, 9 Jun 2017 14:46:16 -0500
+Subject: [PATCH 2/8] Low: libcrmcommon: add function for testing daemon name
+
+---
+ include/crm/common/util.h | 2 ++
+ lib/common/utils.c | 21 +++++++++++++++++++++
+ 2 files changed, 23 insertions(+)
+
+diff --git a/include/crm/common/util.h b/include/crm/common/util.h
+index 682b346..aa192f9 100644
+--- a/include/crm/common/util.h
++++ b/include/crm/common/util.h
+@@ -26,6 +26,7 @@
+
+ # include <sys/types.h>
+ # include <stdlib.h>
++# include <stdbool.h>
+ # include <limits.h>
+ # include <signal.h>
+ # include <sysexits.h>
+@@ -127,6 +128,7 @@ gboolean did_rsc_op_fail(lrmd_event_data_t * event, int target_rc);
+ char *crm_md5sum(const char *buffer);
+
+ char *crm_generate_uuid(void);
++bool crm_is_daemon_name(const char *name);
+
+ int crm_user_lookup(const char *name, uid_t * uid, gid_t * gid);
+
+diff --git a/lib/common/utils.c b/lib/common/utils.c
+index 27ed60d..a652197 100644
+--- a/lib/common/utils.c
++++ b/lib/common/utils.c
+@@ -1959,6 +1959,27 @@ crm_generate_uuid(void)
+ return buffer;
+ }
+
++/*!
++ * \brief Check whether a string represents a cluster daemon name
++ *
++ * \param[in] name String to check
++ *
++ * \return TRUE if name is standard client name used by daemons, FALSE otherwise
++ */
++bool
++crm_is_daemon_name(const char *name)
++{
++ return (name &&
++ (!strcmp(name, CRM_SYSTEM_CRMD)
++ || !strcmp(name, CRM_SYSTEM_STONITHD)
++ || !strcmp(name, T_ATTRD)
++ || !strcmp(name, CRM_SYSTEM_CIB)
++ || !strcmp(name, CRM_SYSTEM_MCP)
++ || !strcmp(name, CRM_SYSTEM_DC)
++ || !strcmp(name, CRM_SYSTEM_TENGINE)
++ || !strcmp(name, CRM_SYSTEM_LRMD)));
++}
++
+ #include <md5.h>
+
+ char *
+--
+1.8.3.1
+
+
+From 19497fa9785c854084253167b7dc54a8c026e1ad Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Fri, 9 Jun 2017 11:11:50 -0500
+Subject: [PATCH 3/8] Low: libcrmcommon: support setting max queue length per
+ IPC client
+
+---
+ include/crm/common/ipcs.h | 3 +++
+ lib/common/ipc.c | 39 ++++++++++++++++++++++++++-------------
+ 2 files changed, 29 insertions(+), 13 deletions(-)
+
+diff --git a/include/crm/common/ipcs.h b/include/crm/common/ipcs.h
+index 43b7b60..d2db212 100644
+--- a/include/crm/common/ipcs.h
++++ b/include/crm/common/ipcs.h
+@@ -19,6 +19,7 @@
+ #ifndef CRM_COMMON_IPCS__H
+ # define CRM_COMMON_IPCS__H
+
++# include <stdbool.h>
+ # include <qb/qbipcs.h>
+ # ifdef HAVE_GNUTLS_GNUTLS_H
+ # undef KEYFILE
+@@ -95,6 +96,7 @@ struct crm_client_s {
+ struct crm_remote_s *remote; /* TCP/TLS */
+
+ unsigned int queue_backlog; /* IPC queue length after last flush */
++ unsigned int queue_max; /* Evict client whose queue grows this big */
+ };
+
+ extern GHashTable *client_connections;
+@@ -110,6 +112,7 @@ crm_client_t *crm_client_alloc(void *key);
+ crm_client_t *crm_client_new(qb_ipcs_connection_t * c, uid_t uid, gid_t gid);
+ void crm_client_destroy(crm_client_t * c);
+ void crm_client_disconnect_all(qb_ipcs_service_t *s);
++bool crm_set_client_queue_max(crm_client_t *client, const char *qmax);
+
+ void crm_ipcs_send_ack(crm_client_t * c, uint32_t request, uint32_t flags,
+ const char *tag, const char *function, int line);
+diff --git a/lib/common/ipc.c b/lib/common/ipc.c
+index e0a7a5c..3110334 100644
+--- a/lib/common/ipc.c
++++ b/lib/common/ipc.c
+@@ -37,8 +37,9 @@
+
+ #define PCMK_IPC_VERSION 1
+
+-/* Evict clients whose event queue grows this large */
+-#define PCMK_IPC_MAX_QUEUE 500
++/* Evict clients whose event queue grows this large (by default) */
++#define PCMK_IPC_DEFAULT_QUEUE_MAX 500
++#define PCMK_IPC_DEFAULT_QUEUE_MAX_S "500"
+
+ struct crm_ipc_response_header {
+ struct qb_ipc_response_header qb;
+@@ -409,6 +410,24 @@ crm_client_destroy(crm_client_t * c)
+ free(c);
+ }
+
++/*!
++ * \brief Raise IPC eviction threshold for a client, if allowed
++ *
++ * \param[in,out] client Client to modify
++ * \param[in] queue_max New threshold (as string)
++ *
++ * \return TRUE if change was allowed, FALSE otherwise
++ */
++bool
++crm_set_client_queue_max(crm_client_t *client, const char *qmax)
++{
++ if (is_set(client->flags, crm_client_flag_ipc_privileged)) {
++ client->queue_max = crm_parse_int(qmax, PCMK_IPC_DEFAULT_QUEUE_MAX_S);
++ return TRUE;
++ }
++ return FALSE;
++}
++
+ int
+ crm_ipcs_client_pid(qb_ipcs_connection_t * c)
+ {
+@@ -553,18 +572,12 @@ crm_ipcs_flush_events(crm_client_t * c)
+ }
+
+ if (queue_len) {
+- /* We want to allow clients to briefly fall behind on processing
+- * incoming messages, but drop completely unresponsive clients so the
+- * connection doesn't consume resources indefinitely.
+- *
+- * @TODO It is possible that the queue could reasonably grow large in a
+- * short time. An example is a reprobe of hundreds of resources on many
+- * nodes resulting in a surge of CIB replies to the crmd. We could
+- * possibly give cluster daemons a higher threshold here, and/or prevent
+- * such a surge by throttling LRM history writes in the crmd.
+- */
+
+- if (queue_len > PCMK_IPC_MAX_QUEUE) {
++ /* Allow clients to briefly fall behind on processing incoming messages,
++ * but drop completely unresponsive clients so the connection doesn't
++ * consume resources indefinitely.
++ */
++ if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) {
+ if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) {
+ /* Don't evict for a new or shrinking backlog */
+ crm_warn("Client with process ID %u has a backlog of %u messages "
+--
+1.8.3.1
+
+
+From 324e241d9d9666dfe6543ddf88d965b12d5a164f Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Mon, 12 Jun 2017 17:47:12 -0500
+Subject: [PATCH 4/8] Low: libcib: always use current values when unpacking
+ config
+
+Previously, cib_read_config() called unpack_instance_attributes() with
+overwrite=FALSE. This meant that changes to an option would not take effect
+unless the option was not set before.
+
+The only significant use of cib_read_config() was in cib_acl_enabled(), which
+used a new, empty hash table for every call, so the issue didn't matter.
+
+The cib daemon also used cib_read_config() to maintain a global config_hash,
+which was affected by the issue, but didn't matter because it was never used.
+This change will allow config_hash to be used.
+---
+ lib/cib/cib_utils.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
+index ab48f16..8aeed67 100644
+--- a/lib/cib/cib_utils.c
++++ b/lib/cib/cib_utils.c
+@@ -743,7 +743,7 @@ cib_read_config(GHashTable * options, xmlNode * current_cib)
+ config = get_object_root(XML_CIB_TAG_CRMCONFIG, current_cib);
+ if (config) {
+ unpack_instance_attributes(current_cib, config, XML_CIB_TAG_PROPSET, NULL, options,
+- CIB_OPTIONS_FIRST, FALSE, now);
++ CIB_OPTIONS_FIRST, TRUE, now);
+ }
+
+ verify_cib_options(options);
+--
+1.8.3.1
+
+
+From b986acbe131216aaa372681e97dfc0f2ef8f70ad Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Tue, 13 Jun 2017 16:03:36 -0500
+Subject: [PATCH 5/8] Low: libcib: correctly search for v2 patchset changes
+
+cib_internal_config_changed() was never updated for v2 patch format
+---
+ lib/cib/cib_utils.c | 23 ++++++++++++-----------
+ 1 file changed, 12 insertions(+), 11 deletions(-)
+
+diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
+index 8aeed67..f639ada 100644
+--- a/lib/cib/cib_utils.c
++++ b/lib/cib/cib_utils.c
+@@ -794,23 +794,24 @@ cib_apply_patch_event(xmlNode * event, xmlNode * input, xmlNode ** output, int l
+ return rc;
+ }
+
++/* v2 and v2 patch formats */
++#define XPATH_CONFIG_CHANGE \
++ "//" XML_CIB_TAG_CRMCONFIG " | " \
++ "//" XML_DIFF_CHANGE "[contains(@" XML_DIFF_PATH ",'/" XML_CIB_TAG_CRMCONFIG "/')]"
++
+ gboolean
+-cib_internal_config_changed(xmlNode * diff)
++cib_internal_config_changed(xmlNode *diff)
+ {
+ gboolean changed = FALSE;
+- xmlXPathObject *xpathObj = NULL;
+
+- if (diff == NULL) {
+- return FALSE;
+- }
++ if (diff) {
++ xmlXPathObject *xpathObj = xpath_search(diff, XPATH_CONFIG_CHANGE);
+
+- xpathObj = xpath_search(diff, "//" XML_CIB_TAG_CRMCONFIG);
+- if (numXpathResults(xpathObj) > 0) {
+- changed = TRUE;
++ if (numXpathResults(xpathObj) > 0) {
++ changed = TRUE;
++ }
++ freeXpathObject(xpathObj);
+ }
+-
+- freeXpathObject(xpathObj);
+-
+ return changed;
+ }
+
+--
+1.8.3.1
+
+
+From 3e5cdd9b329b3328a88912db48c4e4a4d6f94563 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Fri, 9 Jun 2017 15:33:27 -0500
+Subject: [PATCH 6/8] Feature: cib,libcib: support option for IPC eviction
+ threshold
+
+Only the cib utilizes the option currently, but the other daemons
+could easily do the same, if they are subject to large IPC bursts.
+---
+ cib/callbacks.c | 13 +++++++++++++
+ cib/callbacks.h | 10 ++++++++++
+ lib/cib/cib_utils.c | 24 ++++++++++++++++++++----
+ 3 files changed, 43 insertions(+), 4 deletions(-)
+
+diff --git a/cib/callbacks.c b/cib/callbacks.c
+index 4708f10..544a920 100644
+--- a/cib/callbacks.c
++++ b/cib/callbacks.c
+@@ -281,6 +281,19 @@ cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean
+ cib_client->name = crm_itoa(cib_client->pid);
+ } else {
+ cib_client->name = strdup(value);
++ if (crm_is_daemon_name(value)) {
++ set_bit(cib_client->options, cib_is_daemon);
++ }
++ }
++ }
++
++ /* Allow cluster daemons more leeway before being evicted */
++ if (is_set(cib_client->options, cib_is_daemon)) {
++ const char *qmax = cib_config_lookup("cluster-ipc-limit");
++
++ if (crm_set_client_queue_max(cib_client, qmax)) {
++ crm_trace("IPC threshold for %s[%u] is now %u",
++ cib_client->name, cib_client->pid, cib_client->queue_max);
+ }
+ }
+
+diff --git a/cib/callbacks.h b/cib/callbacks.h
+index b4d48d6..bddff09 100644
+--- a/cib/callbacks.h
++++ b/cib/callbacks.h
+@@ -19,6 +19,7 @@
+ #include <stdio.h>
+ #include <sys/types.h>
+ #include <unistd.h>
++#include <glib.h>
+
+ #include <crm/crm.h>
+ #include <crm/cib.h>
+@@ -43,6 +44,9 @@ enum cib_notifications
+ cib_notify_replace = 0x0004,
+ cib_notify_confirm = 0x0008,
+ cib_notify_diff = 0x0010,
++
++ /* not a notification, but uses the same IPC bitmask */
++ cib_is_daemon = 0x1000, /* whether client is another cluster daemon */
+ };
+ /* *INDENT-ON* */
+
+@@ -80,3 +84,9 @@ extern void cib_ha_peer_callback(HA_Message * msg, void *private_data);
+ extern int cib_ccm_dispatch(gpointer user_data);
+ extern void cib_ccm_msg_callback(oc_ed_t event, void *cookie, size_t size, const void *data);
+ #endif
++
++static inline const char *
++cib_config_lookup(const char *opt)
++{
++ return g_hash_table_lookup(config_hash, opt);
++}
+diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c
+index f639ada..adbf9c8 100644
+--- a/lib/cib/cib_utils.c
++++ b/lib/cib/cib_utils.c
+@@ -699,10 +699,26 @@ cib_native_notify(gpointer data, gpointer user_data)
+ }
+
+ pe_cluster_option cib_opts[] = {
+- /* name, old-name, validate, default, description */
+- {"enable-acl", NULL, "boolean", NULL, "false", &check_boolean,
+- "Enable CIB ACL", NULL}
+- ,
++ /*
++ * name, legacy name,
++ * type, allowed values, default, validator,
++ * short description,
++ * long description
++ */
++ {
++ "enable-acl", NULL,
++ "boolean", NULL, "false", &check_boolean,
++ "Enable CIB ACL",
++ NULL
++ },
++ {
++ "cluster-ipc-limit", NULL,
++ "integer", NULL, "500", &check_positive_number,
++ "Maximum IPC message backlog before disconnecting a cluster daemon",
++ "Raise this if log has \"Evicting client\" messages for cluster daemon"
++ " PIDs (a good value is the number of resources in the cluster"
++ " multiplied by the number of nodes)"
++ },
+ };
+
+ void
+--
+1.8.3.1
+
+
+From 3f855794adbe9ad46aaca3848d591f5dc26d3371 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Tue, 13 Jun 2017 18:07:45 -0500
+Subject: [PATCH 7/8] Doc: Pacemaker Explained: document cluster-ipc-limit
+
+---
+ doc/Pacemaker_Explained/en-US/Ch-Options.txt | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+
+diff --git a/doc/Pacemaker_Explained/en-US/Ch-Options.txt b/doc/Pacemaker_Explained/en-US/Ch-Options.txt
+index f4e1af7..ec0c6b9 100644
+--- a/doc/Pacemaker_Explained/en-US/Ch-Options.txt
++++ b/doc/Pacemaker_Explained/en-US/Ch-Options.txt
+@@ -267,6 +267,15 @@ take effect, we can optionally poll the cluster's status for changes. A value
+ of 0 disables polling. Positive values are an interval (in seconds unless other
+ SI units are specified, e.g. 5min).
+
++| cluster-ipc-limit | 500 |
++indexterm:[cluster-ipc-limit,Cluster Option]
++indexterm:[Cluster,Option,cluster-ipc-limit]
++The maximum IPC message backlog before one cluster daemon will disconnect
++another. This is of use in large clusters, for which a good value is the number
++of resources in the cluster multiplied by the number of nodes. The default of
++500 is also the minimum. Raise this if you see "Evicting client" messages for
++cluster daemon PIDs in the logs.
++
+ | pe-error-series-max | -1 |
+ indexterm:[pe-error-series-max,Cluster Option]
+ indexterm:[Cluster,Option,pe-error-series-max]
+--
+1.8.3.1
+
+
+From bc6d723a94221419b15650a52e8ed1d843a32ff1 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Thu, 15 Jun 2017 14:20:27 -0500
+Subject: [PATCH 8/8] Refactor: libcrmcommon: avoid redundant constant
+ definitions
+
+---
+ lib/common/ipc.c | 9 ++++++---
+ 1 file changed, 6 insertions(+), 3 deletions(-)
+
+diff --git a/lib/common/ipc.c b/lib/common/ipc.c
+index 3110334..c238bca 100644
+--- a/lib/common/ipc.c
++++ b/lib/common/ipc.c
+@@ -39,7 +39,6 @@
+
+ /* Evict clients whose event queue grows this large (by default) */
+ #define PCMK_IPC_DEFAULT_QUEUE_MAX 500
+-#define PCMK_IPC_DEFAULT_QUEUE_MAX_S "500"
+
+ struct crm_ipc_response_header {
+ struct qb_ipc_response_header qb;
+@@ -422,8 +421,12 @@ bool
+ crm_set_client_queue_max(crm_client_t *client, const char *qmax)
+ {
+ if (is_set(client->flags, crm_client_flag_ipc_privileged)) {
+- client->queue_max = crm_parse_int(qmax, PCMK_IPC_DEFAULT_QUEUE_MAX_S);
+- return TRUE;
++ int qmax_int = crm_int_helper(qmax, NULL);
++
++ if ((errno == 0) && (qmax_int > 0)) {
++ client->queue_max = qmax_int;
++ return TRUE;
++ }
+ }
+ return FALSE;
+ }
+--
+1.8.3.1
+
diff --git a/SOURCES/072-bundle-placement.patch b/SOURCES/072-bundle-placement.patch
new file mode 100644
index 0000000..2f669f5
--- /dev/null
+++ b/SOURCES/072-bundle-placement.patch
@@ -0,0 +1,74 @@
+From 648f048aeaee089ec74953107e6252543b710966 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Wed, 21 Jun 2017 20:40:34 -0500
+Subject: [PATCH] Fix: libpe_status: properly disallow other resources from
+ bundle nodes
+
+---
+ lib/pengine/container.c | 34 +++++++++++++++++++++++++++++-----
+ 1 file changed, 29 insertions(+), 5 deletions(-)
+
+diff --git a/lib/pengine/container.c b/lib/pengine/container.c
+index e916446..cf81253 100644
+--- a/lib/pengine/container.c
++++ b/lib/pengine/container.c
+@@ -335,6 +335,29 @@ create_docker_resource(
+ return TRUE;
+ }
+
++/*!
++ * \brief Ban a node from a resource's (and its children's) allowed nodes list
++ *
++ * \param[in,out] rsc Resource to modify
++ * \param[in] uname Name of node to ban
++ */
++static void
++disallow_node(resource_t *rsc, const char *uname)
++{
++ gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname);
++
++ if (match) {
++ ((pe_node_t *) match)->weight = -INFINITY;
++ }
++ if (rsc->children) {
++ GListPtr child;
++
++ for (child = rsc->children; child != NULL; child = child->next) {
++ disallow_node((resource_t *) (child->data), uname);
++ }
++ }
++}
++
+ static bool
+ create_remote_resource(
+ resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple,
+@@ -413,7 +436,7 @@ create_remote_resource(
+ }
+
+ /* unpack_remote_nodes() ensures that each remote node and guest node
+- * has a node_t entry. Ideally, it would do the same for bundle nodes.
++ * has a pe_node_t entry. Ideally, it would do the same for bundle nodes.
+ * Unfortunately, a bundle has to be mostly unpacked before it's obvious
+ * what nodes will be needed, so we do it just above.
+ *
+@@ -424,12 +447,13 @@ create_remote_resource(
+ * This adds a node *copy* to each resource's allowed nodes, and these
+ * copies will have the wrong weight.
+ *
+- * As a hacky workaround, clear those copies here.
++ * As a hacky workaround, fix those copies here.
++ *
++ * @TODO Possible alternative: ensure bundles are unpacked before other
++ * resources, so the weight is correct before any copies are made.
+ */
+ for (rsc_iter = data_set->resources; rsc_iter; rsc_iter = rsc_iter->next) {
+- resource_t *rsc = (resource_t *) rsc_iter->data;
+-
+- g_hash_table_remove(rsc->allowed_nodes, uname);
++ disallow_node((resource_t *) (rsc_iter->data), uname);
+ }
+
+ tuple->node = node_copy(node);
+--
+1.8.3.1
+
diff --git a/SOURCES/073-shutdown-logging.patch b/SOURCES/073-shutdown-logging.patch
new file mode 100644
index 0000000..553634e
--- /dev/null
+++ b/SOURCES/073-shutdown-logging.patch
@@ -0,0 +1,1616 @@
+From db1de472c1d1a4bef98b67f181a6e4fc8b5cbcfb Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Mon, 29 May 2017 19:24:50 +1000
+Subject: [PATCH] Log: PE: Improve logging of node fencing and shutdown
+
+---
+ lib/pengine/unpack.c | 58 ++++++++++++----------
+ lib/pengine/utils.c | 2 +-
+ pengine/allocate.c | 47 ++++++++++++++++--
+ pengine/allocate.h | 3 +-
+ pengine/native.c | 4 +-
+ pengine/test10/594.summary | 2 +
+ pengine/test10/662.summary | 1 +
+ pengine/test10/797.summary | 1 +
+ pengine/test10/829.summary | 1 +
+ pengine/test10/bug-1572-1.summary | 1 +
+ pengine/test10/bug-1572-2.summary | 1 +
+ pengine/test10/bug-1573.summary | 1 +
+ pengine/test10/bug-1820-1.summary | 1 +
+ pengine/test10/bug-1820.summary | 1 +
+ pengine/test10/bug-1822.summary | 1 +
+ pengine/test10/bug-5028-bottom.summary | 1 +
+ pengine/test10/bug-5028-detach.summary | 1 +
+ pengine/test10/bug-5028.summary | 1 +
+ pengine/test10/bug-5186-partial-migrate.summary | 1 +
+ pengine/test10/bug-cl-5247.summary | 1 +
+ pengine/test10/bug-lf-2508.summary | 1 +
+ pengine/test10/bug-lf-2551.summary | 1 +
+ pengine/test10/bug-lf-2606.summary | 1 +
+ pengine/test10/bug-rh-1097457.summary | 1 +
+ .../test10/colocate-primitive-with-clone.summary | 1 +
+ pengine/test10/concurrent-fencing.summary | 3 ++
+ pengine/test10/guest-node-host-dies.summary | 3 ++
+ pengine/test10/inc12.summary | 6 +++
+ pengine/test10/interleave-pseudo-stop.summary | 1 +
+ pengine/test10/master-7.summary | 1 +
+ pengine/test10/master-8.summary | 1 +
+ pengine/test10/master-9.summary | 1 +
+ pengine/test10/migrate-fencing.summary | 1 +
+ pengine/test10/migrate-shutdown.summary | 4 ++
+ pengine/test10/novell-239082.summary | 1 +
+ pengine/test10/novell-252693.summary | 1 +
+ pengine/test10/params-2.summary | 1 +
+ pengine/test10/rec-node-11.summary | 1 +
+ pengine/test10/rec-node-12.summary | 1 +
+ pengine/test10/rec-node-13.summary | 1 +
+ pengine/test10/rec-node-14.summary | 3 ++
+ pengine/test10/rec-node-15.summary | 1 +
+ pengine/test10/rec-node-2.summary | 1 +
+ pengine/test10/rec-node-4.summary | 1 +
+ pengine/test10/rec-node-6.summary | 1 +
+ pengine/test10/rec-node-7.summary | 1 +
+ pengine/test10/rec-rsc-5.summary | 1 +
+ .../test10/remote-fence-before-reconnect.summary | 1 +
+ pengine/test10/remote-fence-unclean.summary | 1 +
+ pengine/test10/remote-fence-unclean2.summary | 1 +
+ pengine/test10/remote-partial-migrate2.summary | 1 +
+ pengine/test10/remote-recover-all.summary | 3 ++
+ pengine/test10/remote-recover-connection.summary | 1 +
+ pengine/test10/remote-recover-fail.summary | 1 +
+ pengine/test10/remote-recover-no-resources.summary | 2 +
+ pengine/test10/remote-recover-unknown.summary | 3 ++
+ pengine/test10/remote-recovery.summary | 1 +
+ pengine/test10/remote-unclean2.summary | 1 +
+ pengine/test10/simple7.summary | 1 +
+ .../test10/start-then-stop-with-unfence.summary | 2 +
+ pengine/test10/stonith-0.summary | 2 +
+ pengine/test10/stonith-1.summary | 1 +
+ pengine/test10/stonith-2.summary | 1 +
+ pengine/test10/stonith-3.summary | 1 +
+ pengine/test10/stonith-4.summary | 4 ++
+ pengine/test10/stop-failure-no-quorum.summary | 1 +
+ pengine/test10/stop-failure-with-fencing.summary | 1 +
+ pengine/test10/systemhealth1.summary | 2 +
+ pengine/test10/systemhealth2.summary | 1 +
+ pengine/test10/systemhealth3.summary | 1 +
+ pengine/test10/systemhealthm1.summary | 2 +
+ pengine/test10/systemhealthm2.summary | 1 +
+ pengine/test10/systemhealthm3.summary | 1 +
+ pengine/test10/systemhealthn1.summary | 2 +
+ pengine/test10/systemhealthn2.summary | 1 +
+ pengine/test10/systemhealthn3.summary | 1 +
+ pengine/test10/systemhealtho1.summary | 2 +
+ pengine/test10/systemhealtho2.summary | 1 +
+ pengine/test10/systemhealtho3.summary | 1 +
+ pengine/test10/systemhealthp1.summary | 2 +
+ pengine/test10/systemhealthp2.summary | 1 +
+ pengine/test10/systemhealthp3.summary | 1 +
+ pengine/test10/ticket-clone-21.summary | 2 +
+ pengine/test10/ticket-clone-9.summary | 2 +
+ pengine/test10/ticket-group-21.summary | 1 +
+ pengine/test10/ticket-group-9.summary | 1 +
+ pengine/test10/ticket-master-21.summary | 1 +
+ pengine/test10/ticket-master-9.summary | 1 +
+ pengine/test10/ticket-primitive-21.summary | 1 +
+ pengine/test10/ticket-primitive-9.summary | 1 +
+ pengine/test10/unfence-definition.summary | 5 ++
+ pengine/test10/unfence-parameters.summary | 5 ++
+ pengine/test10/unfence-startup.summary | 5 ++
+ pengine/test10/unmanaged-master.summary | 2 +
+ pengine/test10/whitebox-fail1.summary | 1 +
+ pengine/test10/whitebox-fail2.summary | 1 +
+ .../test10/whitebox-imply-stop-on-fence.summary | 3 ++
+ pengine/test10/whitebox-ms-ordering.summary | 2 +
+ .../test10/whitebox-unexpectedly-running.summary | 1 +
+ tools/crm_simulate.c | 1 +
+ 100 files changed, 223 insertions(+), 34 deletions(-)
+
+diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
+index 8caf522..377100c 100644
+--- a/lib/pengine/unpack.c
++++ b/lib/pengine/unpack.c
+@@ -62,6 +62,7 @@ is_dangling_container_remote_node(node_t *node)
+ return FALSE;
+ }
+
++
+ void
+ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
+ {
+@@ -73,12 +74,12 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
+
+ if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {
+ if (!is_set(rsc->flags, pe_rsc_managed)) {
+- crm_notice("Not fencing guest node %s because the container is "
+- "unmanaged, otherwise we would do so recovering %s "
+- "%s", node->details->uname, rsc->id, reason);
++ crm_notice("Not fencing node %s due to '%s': container %s is"
++ " unmanaged"
++ "%s", node->details->uname, reason, rsc->id);
+ } else {
+- crm_warn("Guest node %s will be fenced (by recovering %s) %s",
+- node->details->uname, rsc->id, reason);
++ crm_warn("Remote node %s will be fenced due to '%s' by recovering %s",
++ node->details->uname, rsc->id, reason);
+
+ /* We don't mark the node as unclean because that would prevent the
+ * node from running resources. We want to allow it to run resources
+@@ -88,8 +89,9 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
+ set_bit(rsc->flags, pe_rsc_failed);
+ }
+ }
++
+ } else if (is_dangling_container_remote_node(node)) {
+- crm_info("Cleaning up dangling connection resource for guest node %s %s"
++ crm_info("Cleaning up dangling connection resource for guest node %s due to '%s'"
+ " (fencing is already done, guest resource no longer exists)",
+ node->details->uname, reason);
+ set_bit(node->details->remote_rsc->flags, pe_rsc_failed);
+@@ -98,28 +100,32 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
+ resource_t *rsc = node->details->remote_rsc;
+
+ if (rsc && (!is_set(rsc->flags, pe_rsc_managed))) {
+- crm_notice("Not fencing node %s because connection is unmanaged, "
+- "otherwise would %s", node->details->uname, reason);
++ crm_notice("Not fencing node %s due to '%s': connection is unmanaged",
++ node->details->uname, reason);
+ } else if(node->details->remote_requires_reset == FALSE) {
+ node->details->remote_requires_reset = TRUE;
+ if (pe_can_fence(data_set, node)) {
+- crm_warn("Node %s will be fenced %s", node->details->uname, reason);
++ crm_warn("Remote node %s will be fenced due to %s", node->details->uname, reason);
+ } else {
+- crm_warn("Node %s is unclean %s", node->details->uname, reason);
++ crm_warn("Remote node %s is unclean due to %s", node->details->uname, reason);
+ }
+ }
+ node->details->unclean = TRUE;
+
+- } else if (node->details->unclean == FALSE) {
++ } else if (node->details->unclean) {
+ if (pe_can_fence(data_set, node)) {
+- crm_warn("Node %s will be fenced %s", node->details->uname, reason);
++ crm_trace("Node %s would also be fenced due to '%s'", node->details->uname, reason);
+ } else {
+- crm_warn("Node %s is unclean %s", node->details->uname, reason);
++ crm_trace("Node %s is also unclean due to '%s'", node->details->uname, reason);
+ }
++
++ } else if (pe_can_fence(data_set, node)) {
++ crm_warn("Node %s will be fenced due to %s", node->details->uname, reason);
+ node->details->unclean = TRUE;
+
+ } else {
+- crm_trace("Node %s would also be fenced '%s'", node->details->uname, reason);
++ crm_warn("Node %s is unclean due to %s", node->details->uname, reason);
++ node->details->unclean = TRUE;
+ }
+ }
+
+@@ -1260,7 +1266,7 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set)
+ /* Everything else should flow from this automatically
+ * At least until the PE becomes able to migrate off healthy resources
+ */
+- pe_fence_node(data_set, this_node, "because the cluster does not have quorum");
++ pe_fence_node(data_set, this_node, "cluster does not have quorum");
+ }
+ }
+ }
+@@ -1316,7 +1322,7 @@ determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_s
+
+ } else {
+ /* mark it unclean */
+- pe_fence_node(data_set, this_node, "because node is unexpectedly down");
++ pe_fence_node(data_set, this_node, "peer is unexpectedly down");
+ crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",
+ crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));
+ }
+@@ -1373,10 +1379,10 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat
+ online = crm_is_true(is_peer);
+
+ } else if (in_cluster == NULL) {
+- pe_fence_node(data_set, this_node, "because the peer has not been seen by the cluster");
++ pe_fence_node(data_set, this_node, "peer has not been seen by the cluster");
+
+ } else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) {
+- pe_fence_node(data_set, this_node, "because it failed the pacemaker membership criteria");
++ pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria");
+
+ } else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) {
+
+@@ -1395,14 +1401,14 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat
+ online = FALSE;
+
+ } else if (crm_is_true(in_cluster) == FALSE) {
+- pe_fence_node(data_set, this_node, "because the node is no longer part of the cluster");
++ pe_fence_node(data_set, this_node, "peer is no longer part of the cluster");
+
+ } else if (crm_is_true(is_peer) == FALSE) {
+- pe_fence_node(data_set, this_node, "because our peer process is no longer available");
++ pe_fence_node(data_set, this_node, "peer process is no longer available");
+
+ /* Everything is running at this point, now check join state */
+ } else if (do_terminate) {
+- pe_fence_node(data_set, this_node, "because termination was requested");
++ pe_fence_node(data_set, this_node, "termination was requested");
+
+ } else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {
+ crm_info("Node %s is active", this_node->details->uname);
+@@ -1414,7 +1420,7 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat
+ this_node->details->pending = TRUE;
+
+ } else {
+- pe_fence_node(data_set, this_node, "because the peer was in an unknown state");
++ pe_fence_node(data_set, this_node, "peer was in an unknown state");
+ crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",
+ this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),
+ crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);
+@@ -1923,14 +1929,14 @@ process_rsc_state(resource_t * rsc, node_t * node,
+ * another node without requiring the baremetal remote nodes to be fenced
+ * as well. */
+ node->details->unseen = TRUE;
+- reason = crm_strdup_printf("because %s is active there. Fencing will be revoked if remote-node connection can be re-established on another cluster-node.", rsc->id);
++ reason = crm_strdup_printf("%s is active there. Fencing will be revoked if remote-node connection can be re-established on another cluster-node.", rsc->id);
+ }
+ should_fence = TRUE;
+ }
+
+ if (should_fence) {
+ if (reason == NULL) {
+- reason = crm_strdup_printf("because %s is thought to be active there", rsc->id);
++ reason = crm_strdup_printf("%s is thought to be active there", rsc->id);
+ }
+ pe_fence_node(data_set, node, reason);
+ }
+@@ -1953,7 +1959,7 @@ process_rsc_state(resource_t * rsc, node_t * node,
+ /* treat it as if it is still running
+ * but also mark the node as unclean
+ */
+- pe_fence_node(data_set, node, "because of resource failure(s)");
++ pe_fence_node(data_set, node, "resource failure(s)");
+ break;
+
+ case action_fail_standby:
+@@ -2009,7 +2015,7 @@ process_rsc_state(resource_t * rsc, node_t * node,
+
+ /* connection resource to baremetal resource failed in a way that
+ * should result in fencing the remote-node. */
+- pe_fence_node(data_set, tmpnode, "because of connection failure(s)");
++ pe_fence_node(data_set, tmpnode, "of connection failure(s)");
+ }
+ }
+
+diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
+index 7acd90c..44fbb51 100644
+--- a/lib/pengine/utils.c
++++ b/lib/pengine/utils.c
+@@ -517,7 +517,7 @@ custom_action(resource_t * rsc, char *key, const char *task,
+ if (is_set(action->rsc->flags, pe_rsc_managed)
+ && save_action && a_task == stop_rsc
+ && action->node->details->unclean == FALSE) {
+- pe_fence_node(data_set, action->node, "because of unrunnable resource actions");
++ pe_fence_node(data_set, action->node, "resource actions are unrunnable");
+ }
+
+ } else if (action->node->details->pending) {
+diff --git a/pengine/allocate.c b/pengine/allocate.c
+index 795ed56..0020af6 100644
+--- a/pengine/allocate.c
++++ b/pengine/allocate.c
+@@ -467,7 +467,7 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki
+ set_bit(action_clear->flags, pe_action_runnable);
+
+ crm_notice("Clearing failure of %s on %s "
+- "because action definition changed " CRM_XS " %s",
++ "action definition changed " CRM_XS " %s",
+ rsc->id, node->details->uname, action_clear->uuid);
+ }
+ }
+@@ -1798,7 +1798,7 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set)
+ CRM_ASSERT(container);
+
+ if(is_set(container->flags, pe_rsc_failed)) {
+- pe_fence_node(data_set, action->node, " because the container failed");
++ pe_fence_node(data_set, action->node, "container failed");
+ }
+
+ crm_trace("%s %s %s %s %d", action->uuid, action->task, remote_rsc->id, container->id, is_set(container->flags, pe_rsc_failed));
+@@ -1967,7 +1967,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set)
+ * way to stop it, it is necessary to fence the
+ * node.
+ */
+- pe_fence_node(data_set, action->node, "because resources are active and the connection is unrecoverable");
++ pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable");
+ }
+
+ custom_action_order(action->rsc, NULL, action,
+@@ -2009,7 +2009,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set)
+ * Since we have no way to find out, it is
+ * necessary to fence the node.
+ */
+- pe_fence_node(data_set, action->node, "because resources are in an unknown state and the connection is unrecoverable");
++ pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable");
+ }
+
+ if(cluster_node && state == remote_state_stopped) {
+@@ -2270,6 +2270,7 @@ stage7(pe_working_set_t * data_set)
+
+ crm_trace("Processing reloads");
+
++ LogNodeActions(data_set, FALSE);
+ for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {
+ resource_t *rsc = (resource_t *) gIter->data;
+
+@@ -2379,6 +2380,44 @@ stage8(pe_working_set_t * data_set)
+ }
+
+ void
++LogNodeActions(pe_working_set_t * data_set, gboolean terminal)
++{
++ GListPtr gIter = NULL;
++
++ for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {
++ char *node_name = NULL;
++ const char *task = NULL;
++ action_t *action = (action_t *) gIter->data;
++
++ if (action->rsc != NULL) {
++ continue;
++ }
++
++ if (is_container_remote_node(action->node)) {
++ node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);
++ } else if(action->node) {
++ node_name = crm_strdup_printf("%s", action->node->details->uname);
++ }
++
++ if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
++ task = "Shutdown";
++ } else if (safe_str_eq(action->task, CRM_OP_FENCE)) {
++ task = "Fence";
++ }
++
++ if(task == NULL) {
++ /* Nothing to report */
++ } else if(terminal) {
++ printf(" * %s %s\n", task, node_name);
++ } else {
++ crm_notice(" * %s %s\n", task, node_name);
++ }
++
++ free(node_name);
++ }
++}
++
++void
+ cleanup_alloc_calculations(pe_working_set_t * data_set)
+ {
+ if (data_set == NULL) {
+diff --git a/pengine/allocate.h b/pengine/allocate.h
+index 3d7b7ce..d89943d 100644
+--- a/pengine/allocate.h
++++ b/pengine/allocate.h
+@@ -152,7 +152,8 @@ extern gboolean unpack_location(xmlNode * xml_obj, pe_working_set_t * data_set);
+
+ extern gboolean unpack_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set);
+
+-extern void LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal);
++void LogNodeActions(pe_working_set_t * data_set, gboolean terminal);
++void LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal);
+ void container_LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal);
+
+ extern void cleanup_alloc_calculations(pe_working_set_t * data_set);
+diff --git a/pengine/native.c b/pengine/native.c
+index 2c4e2a1..dd5ff18 100644
+--- a/pengine/native.c
++++ b/pengine/native.c
+@@ -1818,7 +1818,7 @@ rsc_ticket_constraint(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working
+ for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) {
+ node_t *node = (node_t *) gIter->data;
+
+- pe_fence_node(data_set, node, "because deadman ticket was lost");
++ pe_fence_node(data_set, node, "deadman ticket was lost");
+ }
+ break;
+
+@@ -2782,7 +2782,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete,
+
+ if (force == FALSE && running != NULL) {
+ /* we already know the status of the resource on this node */
+- pe_rsc_trace(rsc, "Skipping active: %s on %s", rsc->id, node->details->uname);
++ pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname);
+ return FALSE;
+ }
+
+diff --git a/pengine/test10/594.summary b/pengine/test10/594.summary
+index 6c208ee..d9fe8c1 100644
+--- a/pengine/test10/594.summary
++++ b/pengine/test10/594.summary
+@@ -13,6 +13,8 @@ Online: [ hadev1 hadev2 ]
+ child_DoFencing:2 (stonith:ssh): Started hadev1
+
+ Transition Summary:
++ * Fence hadev3
++ * Shutdown hadev2
+ * Move DcIPaddr (Started hadev2 -> hadev1)
+ * Move rsc_hadev2 (Started hadev2 -> hadev1)
+ * Stop child_DoFencing:0 (hadev2)
+diff --git a/pengine/test10/662.summary b/pengine/test10/662.summary
+index 8d73b1d..1726f35 100644
+--- a/pengine/test10/662.summary
++++ b/pengine/test10/662.summary
+@@ -14,6 +14,7 @@ Online: [ c001n02 c001n03 c001n04 c001n09 ]
+ child_DoFencing:3 (stonith:ssh): Started c001n09
+
+ Transition Summary:
++ * Shutdown c001n02
+ * Move rsc_c001n02 (Started c001n02 -> c001n03)
+ * Stop child_DoFencing:0 (c001n02)
+
+diff --git a/pengine/test10/797.summary b/pengine/test10/797.summary
+index 6e78255..3184eae 100644
+--- a/pengine/test10/797.summary
++++ b/pengine/test10/797.summary
+@@ -15,6 +15,7 @@ Online: [ c001n01 c001n02 c001n03 ]
+ child_DoFencing:3 (stonith:ssh): Stopped
+
+ Transition Summary:
++ * Shutdown c001n02
+ * Stop DcIPaddr (Started c001n03)
+ * Stop rsc_c001n08 (Started c001n02)
+ * Stop rsc_c001n02 (Started c001n02)
+diff --git a/pengine/test10/829.summary b/pengine/test10/829.summary
+index d95a2ff..a9d25e0 100644
+--- a/pengine/test10/829.summary
++++ b/pengine/test10/829.summary
+@@ -15,6 +15,7 @@ Online: [ c001n01 c001n03 c001n08 ]
+ child_DoFencing:3 (stonith:ssh): Started c001n08
+
+ Transition Summary:
++ * Fence c001n02
+ * Move rsc_c001n02 (Started c001n02 -> c001n01)
+ * Stop child_DoFencing:0 (c001n02)
+
+diff --git a/pengine/test10/bug-1572-1.summary b/pengine/test10/bug-1572-1.summary
+index 4280f7b..6c37bb4 100644
+--- a/pengine/test10/bug-1572-1.summary
++++ b/pengine/test10/bug-1572-1.summary
+@@ -11,6 +11,7 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ]
+ IPaddr_147_81_84_133 (ocf::heartbeat:IPaddr): Started arc-tkincaidlx.wsicorp.com
+
+ Transition Summary:
++ * Shutdown arc-dknightlx
+ * Stop rsc_drbd_7788:0 (arc-dknightlx)
+ * Restart rsc_drbd_7788:1 (Master arc-tkincaidlx.wsicorp.com)
+ * Restart fs_mirror (Started arc-tkincaidlx.wsicorp.com)
+diff --git a/pengine/test10/bug-1572-2.summary b/pengine/test10/bug-1572-2.summary
+index 6174027..a4235a7 100644
+--- a/pengine/test10/bug-1572-2.summary
++++ b/pengine/test10/bug-1572-2.summary
+@@ -11,6 +11,7 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ]
+ IPaddr_147_81_84_133 (ocf::heartbeat:IPaddr): Started arc-tkincaidlx.wsicorp.com
+
+ Transition Summary:
++ * Shutdown arc-dknightlx
+ * Stop rsc_drbd_7788:0 (arc-dknightlx)
+ * Demote rsc_drbd_7788:1 (Master -> Slave arc-tkincaidlx.wsicorp.com)
+ * Stop fs_mirror (arc-tkincaidlx.wsicorp.com)
+diff --git a/pengine/test10/bug-1573.summary b/pengine/test10/bug-1573.summary
+index 0c1fe3d..8fb2820 100644
+--- a/pengine/test10/bug-1573.summary
++++ b/pengine/test10/bug-1573.summary
+@@ -11,6 +11,7 @@ OFFLINE: [ xen-c ]
+ apache_6 (ocf::heartbeat:apache): Stopped
+
+ Transition Summary:
++ * Shutdown xen-b
+ * Stop IPaddr_192_168_1_102 (xen-b)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/bug-1820-1.summary b/pengine/test10/bug-1820-1.summary
+index 5edcbcb..db41ac4 100644
+--- a/pengine/test10/bug-1820-1.summary
++++ b/pengine/test10/bug-1820-1.summary
+@@ -8,6 +8,7 @@ Online: [ star world ]
+ test2 (ocf::heartbeat:Xen): Started star
+
+ Transition Summary:
++ * Shutdown star
+ * Start p1 (world)
+ * Migrate test1 (Started star -> world)
+ * Migrate test2 (Started star -> world)
+diff --git a/pengine/test10/bug-1820.summary b/pengine/test10/bug-1820.summary
+index 592fa67..6d9c021 100644
+--- a/pengine/test10/bug-1820.summary
++++ b/pengine/test10/bug-1820.summary
+@@ -7,6 +7,7 @@ Online: [ star world ]
+ test2 (ocf::heartbeat:Xen): Started star
+
+ Transition Summary:
++ * Shutdown star
+ * Migrate test1 (Started star -> world)
+ * Migrate test2 (Started star -> world)
+
+diff --git a/pengine/test10/bug-1822.summary b/pengine/test10/bug-1822.summary
+index afb9fd1..325e408 100644
+--- a/pengine/test10/bug-1822.summary
++++ b/pengine/test10/bug-1822.summary
+@@ -11,6 +11,7 @@ Online: [ process1a process2b ]
+ master_slave_procdctl:1 (ocf::heartbeat:procdctl): Master process1a
+
+ Transition Summary:
++ * Shutdown process1a
+ * Demote master_slave_Stateful:1 (Master -> Stopped process1a)
+ * Demote master_slave_procdctl:1 (Master -> Stopped process1a)
+
+diff --git a/pengine/test10/bug-5028-bottom.summary b/pengine/test10/bug-5028-bottom.summary
+index fc5cf8f..b43ba4e 100644
+--- a/pengine/test10/bug-5028-bottom.summary
++++ b/pengine/test10/bug-5028-bottom.summary
+@@ -7,6 +7,7 @@ Online: [ bl460g6a bl460g6b ]
+ dummy02 (ocf::heartbeat:Dummy-stop-NG): Started bl460g6a
+
+ Transition Summary:
++ * Shutdown bl460g6a
+ * Stop dummy02 (bl460g6a)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/bug-5028-detach.summary b/pengine/test10/bug-5028-detach.summary
+index 5e93b2a..712bfa1 100644
+--- a/pengine/test10/bug-5028-detach.summary
++++ b/pengine/test10/bug-5028-detach.summary
+@@ -10,6 +10,7 @@ Online: [ bl460g6a bl460g6b ]
+ dummy02 (ocf::heartbeat:Dummy-stop-NG): FAILED bl460g6a ( blocked )
+
+ Transition Summary:
++ * Shutdown bl460g6a
+
+ Executing cluster transition:
+ * Cluster action: do_shutdown on bl460g6a
+diff --git a/pengine/test10/bug-5028.summary b/pengine/test10/bug-5028.summary
+index ad7657c..a85f75b 100644
+--- a/pengine/test10/bug-5028.summary
++++ b/pengine/test10/bug-5028.summary
+@@ -7,6 +7,7 @@ Online: [ bl460g6a bl460g6b ]
+ dummy02 (ocf::heartbeat:Dummy-stop-NG): FAILED bl460g6a ( blocked )
+
+ Transition Summary:
++ * Shutdown bl460g6a
+ * Stop dummy01 (Started bl460g6a - blocked)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary
+index 4d0b10e..63ed2d7 100644
+--- a/pengine/test10/bug-5186-partial-migrate.summary
++++ b/pengine/test10/bug-5186-partial-migrate.summary
+@@ -25,6 +25,7 @@ Online: [ bl460g1n6 bl460g1n8 ]
+ Started: [ bl460g1n6 bl460g1n8 ]
+
+ Transition Summary:
++ * Fence bl460g1n7
+ * Move prmDummy (Started bl460g1n7 -> bl460g1n6)
+ * Move prmVM2 (Started bl460g1n7 -> bl460g1n8)
+ * Move prmStonith8-1 (Started bl460g1n7 -> bl460g1n6)
+diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary
+index a13754f..8ea3ff0 100644
+--- a/pengine/test10/bug-cl-5247.summary
++++ b/pengine/test10/bug-cl-5247.summary
+@@ -18,6 +18,7 @@ Containers: [ pgsr01:prmDB1 ]
+ Stopped: [ bl460g8n3 bl460g8n4 ]
+
+ Transition Summary:
++ * Fence pgsr02 (resource: prmDB2)
+ * Stop prmDB2 (bl460g8n4)
+ * Restart prmStonith1-2 (Started bl460g8n4)
+ * Restart prmStonith2-2 (Started bl460g8n3)
+diff --git a/pengine/test10/bug-lf-2508.summary b/pengine/test10/bug-lf-2508.summary
+index af5e4e2..7b436d4 100644
+--- a/pengine/test10/bug-lf-2508.summary
++++ b/pengine/test10/bug-lf-2508.summary
+@@ -35,6 +35,7 @@ Online: [ srv01 srv03 srv04 ]
+ Stopped: [ srv04 ]
+
+ Transition Summary:
++ * Fence srv02
+ * Start Dummy01 (srv01)
+ * Move Dummy02 (Started srv02 -> srv04)
+ * Stop prmStonith1-1:1 (srv02)
+diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary
+index 158eb73..ffb7c6d 100644
+--- a/pengine/test10/bug-lf-2551.summary
++++ b/pengine/test10/bug-lf-2551.summary
+@@ -81,6 +81,7 @@ Online: [ hex-0 hex-7 hex-8 ]
+ vm-64 (ocf::heartbeat:Xen): Stopped
+
+ Transition Summary:
++ * Fence hex-9
+ * Move fencing-sbd (Started hex-9 -> hex-0)
+ * Move dummy1 (Started hex-9 -> hex-0)
+ * Stop dlm:3 (hex-9)
+diff --git a/pengine/test10/bug-lf-2606.summary b/pengine/test10/bug-lf-2606.summary
+index 3e74d4b..ab93bb3 100644
+--- a/pengine/test10/bug-lf-2606.summary
++++ b/pengine/test10/bug-lf-2606.summary
+@@ -12,6 +12,7 @@ Online: [ node1 ]
+ Slaves: [ node1 ]
+
+ Transition Summary:
++ * Fence node2
+ * Stop rsc1 (node2)
+ * Move rsc2 (Started node2 -> node1)
+ * Demote rsc3:1 (Master -> Stopped node2)
+diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary
+index f8d2c14..8fe474a 100644
+--- a/pengine/test10/bug-rh-1097457.summary
++++ b/pengine/test10/bug-rh-1097457.summary
+@@ -32,6 +32,7 @@ Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ]
+ Started: [ lamaVM1 lamaVM2 lamaVM3 ]
+
+ Transition Summary:
++ * Fence lamaVM2 (resource: VM2)
+ * Recover VM2 (Started lama3)
+ * Recover FSlun3 (Started lamaVM2 -> lama2)
+ * Restart FAKE4 (Started lamaVM2)
+diff --git a/pengine/test10/colocate-primitive-with-clone.summary b/pengine/test10/colocate-primitive-with-clone.summary
+index 5e4c511..e0f685f 100644
+--- a/pengine/test10/colocate-primitive-with-clone.summary
++++ b/pengine/test10/colocate-primitive-with-clone.summary
+@@ -53,6 +53,7 @@ Online: [ srv01 srv02 srv03 srv04 ]
+ Stopped: [ srv01 ]
+
+ Transition Summary:
++ * Shutdown srv01
+ * Start UmVIPcheck (srv04)
+ * Start UmIPaddr (srv04)
+ * Start UmDummy01 (srv04)
+diff --git a/pengine/test10/concurrent-fencing.summary b/pengine/test10/concurrent-fencing.summary
+index 10b2fde..a274c3b 100644
+--- a/pengine/test10/concurrent-fencing.summary
++++ b/pengine/test10/concurrent-fencing.summary
+@@ -8,6 +8,9 @@ Node node3 (uuid3): UNCLEAN (offline)
+ lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped
+
+ Transition Summary:
++ * Fence node3
++ * Fence node2
++ * Fence node1
+
+ Executing cluster transition:
+ * Fencing node3 (reboot)
+diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary
+index c6180ca..717c43d 100644
+--- a/pengine/test10/guest-node-host-dies.summary
++++ b/pengine/test10/guest-node-host-dies.summary
+@@ -11,6 +11,9 @@ Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+ Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
+
+ Transition Summary:
++ * Fence rhel7-1
++ * Fence lxc2 (resource: container2)
++ * Fence lxc1 (resource: container1)
+ * Restart Fencing (Started rhel7-4)
+ * Move rsc_rhel7-1 (Started rhel7-1 -> rhel7-5)
+ * Recover container1 (Started rhel7-1 -> rhel7-2)
+diff --git a/pengine/test10/inc12.summary b/pengine/test10/inc12.summary
+index 3df5a59..5068b7e 100644
+--- a/pengine/test10/inc12.summary
++++ b/pengine/test10/inc12.summary
+@@ -32,6 +32,12 @@ Online: [ c001n02 c001n03 c001n04 c001n05 c001n06 c001n07 ]
+ ocf_msdummy:11 (ocf::heartbeat:Stateful): Slave c001n02
+
+ Transition Summary:
++ * Shutdown c001n07
++ * Shutdown c001n06
++ * Shutdown c001n05
++ * Shutdown c001n04
++ * Shutdown c001n03
++ * Shutdown c001n02
+ * Stop ocf_192.168.100.181 (c001n02)
+ * Stop heartbeat_192.168.100.182 (c001n02)
+ * Stop ocf_192.168.100.183 (c001n02)
+diff --git a/pengine/test10/interleave-pseudo-stop.summary b/pengine/test10/interleave-pseudo-stop.summary
+index a682462..cf30da0 100644
+--- a/pengine/test10/interleave-pseudo-stop.summary
++++ b/pengine/test10/interleave-pseudo-stop.summary
+@@ -17,6 +17,7 @@ Online: [ node2 ]
+ Started: [ node2 ]
+
+ Transition Summary:
++ * Fence node1
+ * Stop stonithclone:1 (node1)
+ * Stop evmsclone:1 (node1)
+ * Stop imagestoreclone:1 (node1)
+diff --git a/pengine/test10/master-7.summary b/pengine/test10/master-7.summary
+index 7472542..348b4ee 100644
+--- a/pengine/test10/master-7.summary
++++ b/pengine/test10/master-7.summary
+@@ -29,6 +29,7 @@ Online: [ c001n02 c001n03 c001n08 ]
+ ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08
+
+ Transition Summary:
++ * Fence c001n01
+ * Move DcIPaddr (Started c001n01 -> c001n03)
+ * Move ocf_192.168.100.181 (Started c001n03 -> c001n02)
+ * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02)
+diff --git a/pengine/test10/master-8.summary b/pengine/test10/master-8.summary
+index 450737b..b77c884 100644
+--- a/pengine/test10/master-8.summary
++++ b/pengine/test10/master-8.summary
+@@ -29,6 +29,7 @@ Online: [ c001n02 c001n03 c001n08 ]
+ ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08
+
+ Transition Summary:
++ * Fence c001n01
+ * Move DcIPaddr (Started c001n01 -> c001n03)
+ * Move ocf_192.168.100.181 (Started c001n03 -> c001n02)
+ * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02)
+diff --git a/pengine/test10/master-9.summary b/pengine/test10/master-9.summary
+index 4f57a79..c1cd4a1 100644
+--- a/pengine/test10/master-9.summary
++++ b/pengine/test10/master-9.summary
+@@ -30,6 +30,7 @@ Online: [ ibm1 va1 ]
+ ocf_msdummy:7 (ocf::heartbeat:/usr/lib64/heartbeat/cts/OCFMSDummy): Stopped
+
+ Transition Summary:
++ * Shutdown ibm1
+ * Start DcIPaddr (va1 - blocked)
+ * Start ocf_127.0.0.11 (va1 - blocked)
+ * Start heartbeat_127.0.0.12 (va1 - blocked)
+diff --git a/pengine/test10/migrate-fencing.summary b/pengine/test10/migrate-fencing.summary
+index e8bcd45..831e49a 100644
+--- a/pengine/test10/migrate-fencing.summary
++++ b/pengine/test10/migrate-fencing.summary
+@@ -22,6 +22,7 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ]
+ Slaves: [ pcmk-1 pcmk-2 pcmk-3 ]
+
+ Transition Summary:
++ * Fence pcmk-4
+ * Stop FencingChild:0 (pcmk-4)
+ * Move r192.168.101.181 (Started pcmk-4 -> pcmk-1)
+ * Move r192.168.101.182 (Started pcmk-4 -> pcmk-1)
+diff --git a/pengine/test10/migrate-shutdown.summary b/pengine/test10/migrate-shutdown.summary
+index e634a5c..630d58d 100644
+--- a/pengine/test10/migrate-shutdown.summary
++++ b/pengine/test10/migrate-shutdown.summary
+@@ -22,6 +22,10 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 pcmk-4 ]
+ Stopped: [ pcmk-3 ]
+
+ Transition Summary:
++ * Shutdown pcmk-4
++ * Shutdown pcmk-3
++ * Shutdown pcmk-2
++ * Shutdown pcmk-1
+ * Stop Fencing (pcmk-1)
+ * Stop r192.168.122.105 (pcmk-2)
+ * Stop r192.168.122.106 (pcmk-2)
+diff --git a/pengine/test10/novell-239082.summary b/pengine/test10/novell-239082.summary
+index b2c28ca..b596de4 100644
+--- a/pengine/test10/novell-239082.summary
++++ b/pengine/test10/novell-239082.summary
+@@ -8,6 +8,7 @@ Online: [ xen-1 xen-2 ]
+ Slaves: [ xen-2 ]
+
+ Transition Summary:
++ * Shutdown xen-1
+ * Move fs_1 (Started xen-1 -> xen-2)
+ * Promote drbd0:0 (Slave -> Master xen-2)
+ * Demote drbd0:1 (Master -> Stopped xen-1)
+diff --git a/pengine/test10/novell-252693.summary b/pengine/test10/novell-252693.summary
+index 47bad6f..23f0632 100644
+--- a/pengine/test10/novell-252693.summary
++++ b/pengine/test10/novell-252693.summary
+@@ -15,6 +15,7 @@ Online: [ node1 node2 ]
+ sles10 (ocf::heartbeat:Xen): Started node1
+
+ Transition Summary:
++ * Shutdown node1
+ * Stop stonithclone:1 (node1)
+ * Stop evmsdclone:1 (node1)
+ * Stop evmsclone:1 (node1)
+diff --git a/pengine/test10/params-2.summary b/pengine/test10/params-2.summary
+index 891ea33..f4169f2 100644
+--- a/pengine/test10/params-2.summary
++++ b/pengine/test10/params-2.summary
+@@ -7,6 +7,7 @@ Online: [ node1 node2 node3 ]
+ rsc3 (lsb:apache): Stopped
+
+ Transition Summary:
++ * Shutdown node1
+ * Stop rsc1 (node1)
+ * Restart rsc2 (Started node2)
+ * Start rsc3 (node3)
+diff --git a/pengine/test10/rec-node-11.summary b/pengine/test10/rec-node-11.summary
+index 1ed0eab..eb967dd 100644
+--- a/pengine/test10/rec-node-11.summary
++++ b/pengine/test10/rec-node-11.summary
+@@ -10,6 +10,7 @@ Online: [ node2 ]
+ rsc3 (heartbeat:apache): Started node2
+
+ Transition Summary:
++ * Fence node1
+ * Start stonith-1 (node2)
+ * Move rsc1 (Started node1 -> node2)
+ * Move rsc2 (Started node1 -> node2)
+diff --git a/pengine/test10/rec-node-12.summary b/pengine/test10/rec-node-12.summary
+index 82a03ac..26f9b13 100644
+--- a/pengine/test10/rec-node-12.summary
++++ b/pengine/test10/rec-node-12.summary
+@@ -15,6 +15,7 @@ Online: [ c001n01 c001n03 c001n08 ]
+ child_DoFencing:3 (stonith:ssh): Stopped
+
+ Transition Summary:
++ * Fence c001n02
+ * Start DcIPaddr (c001n08)
+ * Start rsc_c001n08 (c001n08)
+ * Start rsc_c001n02 (c001n01)
+diff --git a/pengine/test10/rec-node-13.summary b/pengine/test10/rec-node-13.summary
+index 7c6f52b..e273a79 100644
+--- a/pengine/test10/rec-node-13.summary
++++ b/pengine/test10/rec-node-13.summary
+@@ -34,6 +34,7 @@ OFFLINE: [ c001n03 c001n05 ]
+ ocf_msdummy:11 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n07
+
+ Transition Summary:
++ * Fence c001n04
+ * Stop ocf_msdummy:6 (c001n04)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/rec-node-14.summary b/pengine/test10/rec-node-14.summary
+index d97d943..532f3d9 100644
+--- a/pengine/test10/rec-node-14.summary
++++ b/pengine/test10/rec-node-14.summary
+@@ -8,6 +8,9 @@ Node node3 (uuid3): UNCLEAN (offline)
+ lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped
+
+ Transition Summary:
++ * Fence node3
++ * Fence node2
++ * Fence node1
+
+ Executing cluster transition:
+ * Fencing node1 (reboot)
+diff --git a/pengine/test10/rec-node-15.summary b/pengine/test10/rec-node-15.summary
+index 1c3e8ea..8d88680 100644
+--- a/pengine/test10/rec-node-15.summary
++++ b/pengine/test10/rec-node-15.summary
+@@ -21,6 +21,7 @@ Online: [ sapcl01 ]
+ oralsnr_25 (ocf::heartbeat:oralsnr): Stopped
+
+ Transition Summary:
++ * Fence sapcl03
+ * Start stonith-1 (sapcl01)
+ * Move IPaddr_192_168_1_102 (Started sapcl02 -> sapcl01)
+ * Move LVM_12 (Started sapcl02 -> sapcl01)
+diff --git a/pengine/test10/rec-node-2.summary b/pengine/test10/rec-node-2.summary
+index 6dc5098..5c8db02 100644
+--- a/pengine/test10/rec-node-2.summary
++++ b/pengine/test10/rec-node-2.summary
+@@ -14,6 +14,7 @@ Online: [ node2 ]
+ rsc6 (heartbeat:apache): Stopped
+
+ Transition Summary:
++ * Fence node1
+ * Start stonith-1 (node2)
+ * Start rsc1 (node2)
+ * Start rsc2 (node2)
+diff --git a/pengine/test10/rec-node-4.summary b/pengine/test10/rec-node-4.summary
+index a64580c..761573f 100644
+--- a/pengine/test10/rec-node-4.summary
++++ b/pengine/test10/rec-node-4.summary
+@@ -8,6 +8,7 @@ Online: [ node2 ]
+ rsc2 (heartbeat:apache): Started node1 (UNCLEAN)
+
+ Transition Summary:
++ * Fence node1
+ * Start stonith-1 (node2)
+ * Move rsc1 (Started node1 -> node2)
+ * Move rsc2 (Started node1 -> node2)
+diff --git a/pengine/test10/rec-node-6.summary b/pengine/test10/rec-node-6.summary
+index cf02414..fb294fb 100644
+--- a/pengine/test10/rec-node-6.summary
++++ b/pengine/test10/rec-node-6.summary
+@@ -8,6 +8,7 @@ Online: [ node2 ]
+ rsc2 (heartbeat:apache): Started node1
+
+ Transition Summary:
++ * Fence node1
+ * Start stonith-1 (node2)
+ * Move rsc1 (Started node1 -> node2)
+ * Move rsc2 (Started node1 -> node2)
+diff --git a/pengine/test10/rec-node-7.summary b/pengine/test10/rec-node-7.summary
+index a64580c..761573f 100644
+--- a/pengine/test10/rec-node-7.summary
++++ b/pengine/test10/rec-node-7.summary
+@@ -8,6 +8,7 @@ Online: [ node2 ]
+ rsc2 (heartbeat:apache): Started node1 (UNCLEAN)
+
+ Transition Summary:
++ * Fence node1
+ * Start stonith-1 (node2)
+ * Move rsc1 (Started node1 -> node2)
+ * Move rsc2 (Started node1 -> node2)
+diff --git a/pengine/test10/rec-rsc-5.summary b/pengine/test10/rec-rsc-5.summary
+index bfb9a3c..7bcb1a3 100644
+--- a/pengine/test10/rec-rsc-5.summary
++++ b/pengine/test10/rec-rsc-5.summary
+@@ -8,6 +8,7 @@ Online: [ node1 ]
+ rsc2 (heartbeat:apache): Started node2
+
+ Transition Summary:
++ * Fence node2
+ * Start stonith-1 (node1)
+ * Recover rsc1 (Started node2 -> node1)
+ * Move rsc2 (Started node2 -> node1)
+diff --git a/pengine/test10/remote-fence-before-reconnect.summary b/pengine/test10/remote-fence-before-reconnect.summary
+index 88ca48c..520f5cf 100644
+--- a/pengine/test10/remote-fence-before-reconnect.summary
++++ b/pengine/test10/remote-fence-before-reconnect.summary
+@@ -12,6 +12,7 @@ Online: [ c7auto1 c7auto2 c7auto3 ]
+ fake5 (ocf::heartbeat:Dummy): Started c7auto3
+
+ Transition Summary:
++ * Fence c7auto4
+ * Stop c7auto4 (c7auto1)
+ * Move fake2 (Started c7auto4 -> c7auto1)
+
+diff --git a/pengine/test10/remote-fence-unclean.summary b/pengine/test10/remote-fence-unclean.summary
+index cd246e4..06940fc 100644
+--- a/pengine/test10/remote-fence-unclean.summary
++++ b/pengine/test10/remote-fence-unclean.summary
+@@ -11,6 +11,7 @@ Online: [ 18builder 18node1 18node2 ]
+ FAKE4 (ocf::heartbeat:Dummy): Started 18node1
+
+ Transition Summary:
++ * Fence remote1
+ * Recover remote1 (Started 18node1)
+ * Move FAKE2 (Started remote1 -> 18builder)
+ * Move FAKE3 (Started 18builder -> 18node1)
+diff --git a/pengine/test10/remote-fence-unclean2.summary b/pengine/test10/remote-fence-unclean2.summary
+index 78ff784..96f5d69 100644
+--- a/pengine/test10/remote-fence-unclean2.summary
++++ b/pengine/test10/remote-fence-unclean2.summary
+@@ -10,6 +10,7 @@ OFFLINE: [ rhel7-alt3 ]
+ fake (ocf::heartbeat:Dummy): Started rhel7-alt4 (UNCLEAN)
+
+ Transition Summary:
++ * Fence rhel7-alt4
+ * Stop fake (rhel7-alt4)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/remote-partial-migrate2.summary b/pengine/test10/remote-partial-migrate2.summary
+index 6a65975..197bd8c 100644
+--- a/pengine/test10/remote-partial-migrate2.summary
++++ b/pengine/test10/remote-partial-migrate2.summary
+@@ -63,6 +63,7 @@ RemoteOFFLINE: [ pcmk_remote4 ]
+ FAKE50 (ocf::heartbeat:Dummy): Started pcmk_remote5
+
+ Transition Summary:
++ * Fence pcmk4
+ * Migrate pcmk_remote2 (Started pcmk3 -> pcmk1)
+ * Start pcmk_remote4 (pcmk2)
+ * Migrate pcmk_remote5 (Started pcmk1 -> pcmk2)
+diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary
+index 881f449..3f29f70 100644
+--- a/pengine/test10/remote-recover-all.summary
++++ b/pengine/test10/remote-recover-all.summary
+@@ -38,6 +38,9 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
+ stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
+
+ Transition Summary:
++ * Fence messaging-1
++ * Fence galera-2
++ * Fence controller-1
+ * Stop messaging-1 (controller-1)
+ * Move galera-0 (Started controller-1 -> controller-2)
+ * Stop galera-2 (controller-1)
+diff --git a/pengine/test10/remote-recover-connection.summary b/pengine/test10/remote-recover-connection.summary
+index 57b5e01..43507af 100644
+--- a/pengine/test10/remote-recover-connection.summary
++++ b/pengine/test10/remote-recover-connection.summary
+@@ -38,6 +38,7 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
+ stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
+
+ Transition Summary:
++ * Fence controller-1
+ * Move messaging-1 (Started controller-1 -> controller-2)
+ * Move galera-0 (Started controller-1 -> controller-2)
+ * Move galera-2 (Started controller-1 -> controller-2)
+diff --git a/pengine/test10/remote-recover-fail.summary b/pengine/test10/remote-recover-fail.summary
+index 5953e34..ec2d701 100644
+--- a/pengine/test10/remote-recover-fail.summary
++++ b/pengine/test10/remote-recover-fail.summary
+@@ -14,6 +14,7 @@ OFFLINE: [ rhel7-auto1 ]
+ FAKE6 (ocf::heartbeat:Dummy): Started rhel7-auto4 (UNCLEAN)
+
+ Transition Summary:
++ * Fence rhel7-auto4
+ * Recover rhel7-auto4 (Started rhel7-auto2)
+ * Start FAKE1 (rhel7-auto2)
+ * Move FAKE2 (Started rhel7-auto4 -> rhel7-auto3)
+diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary
+index 8bfeb43..48355a4 100644
+--- a/pengine/test10/remote-recover-no-resources.summary
++++ b/pengine/test10/remote-recover-no-resources.summary
+@@ -38,6 +38,8 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
+ stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
+
+ Transition Summary:
++ * Fence messaging-1
++ * Fence controller-1
+ * Stop messaging-1 (controller-1)
+ * Move galera-0 (Started controller-1 -> controller-2)
+ * Stop galera-2 (controller-1)
+diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary
+index 7562f12..212af4e 100644
+--- a/pengine/test10/remote-recover-unknown.summary
++++ b/pengine/test10/remote-recover-unknown.summary
+@@ -38,6 +38,9 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
+ stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
+
+ Transition Summary:
++ * Fence messaging-1
++ * Fence galera-2
++ * Fence controller-1
+ * Stop messaging-1 (controller-1)
+ * Move galera-0 (Started controller-1 -> controller-2)
+ * Stop galera-2 (controller-1)
+diff --git a/pengine/test10/remote-recovery.summary b/pengine/test10/remote-recovery.summary
+index 57b5e01..43507af 100644
+--- a/pengine/test10/remote-recovery.summary
++++ b/pengine/test10/remote-recovery.summary
+@@ -38,6 +38,7 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ]
+ stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN)
+
+ Transition Summary:
++ * Fence controller-1
+ * Move messaging-1 (Started controller-1 -> controller-2)
+ * Move galera-0 (Started controller-1 -> controller-2)
+ * Move galera-2 (Started controller-1 -> controller-2)
+diff --git a/pengine/test10/remote-unclean2.summary b/pengine/test10/remote-unclean2.summary
+index 0a73cbd..ad4af90 100644
+--- a/pengine/test10/remote-unclean2.summary
++++ b/pengine/test10/remote-unclean2.summary
+@@ -7,6 +7,7 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ]
+ rhel7-auto4 (ocf::pacemaker:remote): FAILED rhel7-auto1
+
+ Transition Summary:
++ * Fence rhel7-auto4
+ * Recover rhel7-auto4 (Started rhel7-auto1)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/simple7.summary b/pengine/test10/simple7.summary
+index 2c2818f..021c15f 100644
+--- a/pengine/test10/simple7.summary
++++ b/pengine/test10/simple7.summary
+@@ -5,6 +5,7 @@ Online: [ node1 ]
+ rsc1 (heartbeat:apache): Started node1
+
+ Transition Summary:
++ * Shutdown node1
+ * Stop rsc1 (node1)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/start-then-stop-with-unfence.summary b/pengine/test10/start-then-stop-with-unfence.summary
+index df7d9e3..4430cc0 100644
+--- a/pengine/test10/start-then-stop-with-unfence.summary
++++ b/pengine/test10/start-then-stop-with-unfence.summary
+@@ -11,6 +11,8 @@ Online: [ rhel7-node1.example.com rhel7-node2.example.com ]
+ Stopped: [ rhel7-node1.example.com ]
+
+ Transition Summary:
++ * Fence rhel7-node2.example.com
++ * Fence rhel7-node1.example.com
+ * Start mpath-node1 (rhel7-node1.example.com)
+ * Move ip1 (Started rhel7-node2.example.com -> rhel7-node1.example.com)
+ * Start jrummy:1 (rhel7-node1.example.com)
+diff --git a/pengine/test10/stonith-0.summary b/pengine/test10/stonith-0.summary
+index 0fcaf80..24008a1 100644
+--- a/pengine/test10/stonith-0.summary
++++ b/pengine/test10/stonith-0.summary
+@@ -37,6 +37,8 @@ Online: [ c001n02 c001n04 c001n06 c001n07 c001n08 ]
+ ocf_msdummy:13 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n06
+
+ Transition Summary:
++ * Fence c001n05
++ * Fence c001n03
+ * Move ocf_192.168.100.181 (Started c001n03 -> c001n02)
+ * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02)
+ * Recover ocf_192.168.100.183 (Started c001n03 -> c001n02)
+diff --git a/pengine/test10/stonith-1.summary b/pengine/test10/stonith-1.summary
+index 589e402..8556021 100644
+--- a/pengine/test10/stonith-1.summary
++++ b/pengine/test10/stonith-1.summary
+@@ -28,6 +28,7 @@ Online: [ sles-1 sles-2 sles-4 ]
+ ocf_msdummy:7 (ocf::heartbeat:Stateful): Stopped
+
+ Transition Summary:
++ * Fence sles-3
+ * Start r192.168.100.183 (sles-1)
+ * Move migrator (Started sles-3 -> sles-4)
+ * Move rsc_sles-3 (Started sles-3 -> sles-4)
+diff --git a/pengine/test10/stonith-2.summary b/pengine/test10/stonith-2.summary
+index b02c9b4..0f7cb99 100644
+--- a/pengine/test10/stonith-2.summary
++++ b/pengine/test10/stonith-2.summary
+@@ -33,6 +33,7 @@ Online: [ sles-1 sles-2 sles-3 sles-4 sles-6 ]
+ ocf_msdummy:11 (ocf::heartbeat:Stateful): Slave sles-3
+
+ Transition Summary:
++ * Fence sles-5
+ * Start rsc_sles-5 (sles-6)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/stonith-3.summary b/pengine/test10/stonith-3.summary
+index 64b21d4..cb25cc3 100644
+--- a/pengine/test10/stonith-3.summary
++++ b/pengine/test10/stonith-3.summary
+@@ -8,6 +8,7 @@ Online: [ rh5node2 ]
+ Stopped: [ rh5node1 rh5node2 ]
+
+ Transition Summary:
++ * Fence rh5node1
+ * Start prmIpPostgreSQLDB (rh5node2)
+ * Start prmStonith:0 (rh5node2)
+
+diff --git a/pengine/test10/stonith-4.summary b/pengine/test10/stonith-4.summary
+index 78118c5..dc7cd89 100644
+--- a/pengine/test10/stonith-4.summary
++++ b/pengine/test10/stonith-4.summary
+@@ -14,6 +14,10 @@ OFFLINE: [ pcmk-4 pcmk-6 ]
+ Fencing (stonith:fence_xvm): Stopped
+
+ Transition Summary:
++ * Fence pcmk-8
++ * Fence pcmk-7
++ * Fence pcmk-5
++ * Fence pcmk-10
+ * Start Fencing (pcmk-1 - blocked)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/stop-failure-no-quorum.summary b/pengine/test10/stop-failure-no-quorum.summary
+index 7c14ce2..d864f1a 100644
+--- a/pengine/test10/stop-failure-no-quorum.summary
++++ b/pengine/test10/stop-failure-no-quorum.summary
+@@ -15,6 +15,7 @@ Online: [ pcmk-1 ]
+ Fencing (stonith:fence_xvm): Stopped
+
+ Transition Summary:
++ * Fence pcmk-2
+ * Start dlm:0 (pcmk-1 - blocked)
+ * Stop clvm:0 (pcmk-2)
+ * Start clvm:2 (pcmk-1 - blocked)
+diff --git a/pengine/test10/stop-failure-with-fencing.summary b/pengine/test10/stop-failure-with-fencing.summary
+index 72417e6..e01b6c4 100644
+--- a/pengine/test10/stop-failure-with-fencing.summary
++++ b/pengine/test10/stop-failure-with-fencing.summary
+@@ -14,6 +14,7 @@ Online: [ pcmk-1 ]
+ Fencing (stonith:fence_xvm): Stopped
+
+ Transition Summary:
++ * Fence pcmk-2
+ * Start dlm:0 (pcmk-1 - blocked)
+ * Stop clvm:0 (pcmk-2)
+ * Start clvm:1 (pcmk-1 - blocked)
+diff --git a/pengine/test10/systemhealth1.summary b/pengine/test10/systemhealth1.summary
+index bbdbcaf..37b0b49 100644
+--- a/pengine/test10/systemhealth1.summary
++++ b/pengine/test10/systemhealth1.summary
+@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline)
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
++ * Fence hs21c
+
+ Executing cluster transition:
+ * Fencing hs21d (reboot)
+diff --git a/pengine/test10/systemhealth2.summary b/pengine/test10/systemhealth2.summary
+index 86c6bd4..a37ce18 100644
+--- a/pengine/test10/systemhealth2.summary
++++ b/pengine/test10/systemhealth2.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+ * Start stonith-1 (hs21c)
+ * Start apache_1 (hs21c)
+ * Start nfs_1 (hs21c)
+diff --git a/pengine/test10/systemhealth3.summary b/pengine/test10/systemhealth3.summary
+index 86c6bd4..a37ce18 100644
+--- a/pengine/test10/systemhealth3.summary
++++ b/pengine/test10/systemhealth3.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+ * Start stonith-1 (hs21c)
+ * Start apache_1 (hs21c)
+ * Start nfs_1 (hs21c)
+diff --git a/pengine/test10/systemhealthm1.summary b/pengine/test10/systemhealthm1.summary
+index bbdbcaf..37b0b49 100644
+--- a/pengine/test10/systemhealthm1.summary
++++ b/pengine/test10/systemhealthm1.summary
+@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline)
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
++ * Fence hs21c
+
+ Executing cluster transition:
+ * Fencing hs21d (reboot)
+diff --git a/pengine/test10/systemhealthm2.summary b/pengine/test10/systemhealthm2.summary
+index 86c6bd4..a37ce18 100644
+--- a/pengine/test10/systemhealthm2.summary
++++ b/pengine/test10/systemhealthm2.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+ * Start stonith-1 (hs21c)
+ * Start apache_1 (hs21c)
+ * Start nfs_1 (hs21c)
+diff --git a/pengine/test10/systemhealthm3.summary b/pengine/test10/systemhealthm3.summary
+index 760023d..b10d3f2 100644
+--- a/pengine/test10/systemhealthm3.summary
++++ b/pengine/test10/systemhealthm3.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+
+ Executing cluster transition:
+ * Resource action: stonith-1 monitor on hs21c
+diff --git a/pengine/test10/systemhealthn1.summary b/pengine/test10/systemhealthn1.summary
+index bbdbcaf..37b0b49 100644
+--- a/pengine/test10/systemhealthn1.summary
++++ b/pengine/test10/systemhealthn1.summary
+@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline)
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
++ * Fence hs21c
+
+ Executing cluster transition:
+ * Fencing hs21d (reboot)
+diff --git a/pengine/test10/systemhealthn2.summary b/pengine/test10/systemhealthn2.summary
+index 86c6bd4..a37ce18 100644
+--- a/pengine/test10/systemhealthn2.summary
++++ b/pengine/test10/systemhealthn2.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+ * Start stonith-1 (hs21c)
+ * Start apache_1 (hs21c)
+ * Start nfs_1 (hs21c)
+diff --git a/pengine/test10/systemhealthn3.summary b/pengine/test10/systemhealthn3.summary
+index 86c6bd4..a37ce18 100644
+--- a/pengine/test10/systemhealthn3.summary
++++ b/pengine/test10/systemhealthn3.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+ * Start stonith-1 (hs21c)
+ * Start apache_1 (hs21c)
+ * Start nfs_1 (hs21c)
+diff --git a/pengine/test10/systemhealtho1.summary b/pengine/test10/systemhealtho1.summary
+index bbdbcaf..37b0b49 100644
+--- a/pengine/test10/systemhealtho1.summary
++++ b/pengine/test10/systemhealtho1.summary
+@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline)
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
++ * Fence hs21c
+
+ Executing cluster transition:
+ * Fencing hs21d (reboot)
+diff --git a/pengine/test10/systemhealtho2.summary b/pengine/test10/systemhealtho2.summary
+index 760023d..b10d3f2 100644
+--- a/pengine/test10/systemhealtho2.summary
++++ b/pengine/test10/systemhealtho2.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+
+ Executing cluster transition:
+ * Resource action: stonith-1 monitor on hs21c
+diff --git a/pengine/test10/systemhealtho3.summary b/pengine/test10/systemhealtho3.summary
+index 760023d..b10d3f2 100644
+--- a/pengine/test10/systemhealtho3.summary
++++ b/pengine/test10/systemhealtho3.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+
+ Executing cluster transition:
+ * Resource action: stonith-1 monitor on hs21c
+diff --git a/pengine/test10/systemhealthp1.summary b/pengine/test10/systemhealthp1.summary
+index bbdbcaf..37b0b49 100644
+--- a/pengine/test10/systemhealthp1.summary
++++ b/pengine/test10/systemhealthp1.summary
+@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline)
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
++ * Fence hs21c
+
+ Executing cluster transition:
+ * Fencing hs21d (reboot)
+diff --git a/pengine/test10/systemhealthp2.summary b/pengine/test10/systemhealthp2.summary
+index 94c63d6..6605e12 100644
+--- a/pengine/test10/systemhealthp2.summary
++++ b/pengine/test10/systemhealthp2.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+ * Start apache_1 (hs21c)
+ * Start nfs_1 (hs21c)
+
+diff --git a/pengine/test10/systemhealthp3.summary b/pengine/test10/systemhealthp3.summary
+index 760023d..b10d3f2 100644
+--- a/pengine/test10/systemhealthp3.summary
++++ b/pengine/test10/systemhealthp3.summary
+@@ -8,6 +8,7 @@ Online: [ hs21c ]
+ nfs_1 (ocf::heartbeat:Filesystem): Stopped
+
+ Transition Summary:
++ * Fence hs21d
+
+ Executing cluster transition:
+ * Resource action: stonith-1 monitor on hs21c
+diff --git a/pengine/test10/ticket-clone-21.summary b/pengine/test10/ticket-clone-21.summary
+index 5ae23ec..50df628 100644
+--- a/pengine/test10/ticket-clone-21.summary
++++ b/pengine/test10/ticket-clone-21.summary
+@@ -7,6 +7,8 @@ Online: [ node1 node2 ]
+ Started: [ node1 node2 ]
+
+ Transition Summary:
++ * Fence node2
++ * Fence node1
+ * Stop rsc_stonith (node1)
+ * Stop rsc1:0 (node1)
+ * Stop rsc1:1 (node2)
+diff --git a/pengine/test10/ticket-clone-9.summary b/pengine/test10/ticket-clone-9.summary
+index 5ae23ec..50df628 100644
+--- a/pengine/test10/ticket-clone-9.summary
++++ b/pengine/test10/ticket-clone-9.summary
+@@ -7,6 +7,8 @@ Online: [ node1 node2 ]
+ Started: [ node1 node2 ]
+
+ Transition Summary:
++ * Fence node2
++ * Fence node1
+ * Stop rsc_stonith (node1)
+ * Stop rsc1:0 (node1)
+ * Stop rsc1:1 (node2)
+diff --git a/pengine/test10/ticket-group-21.summary b/pengine/test10/ticket-group-21.summary
+index 9027d2c..eab14a9 100644
+--- a/pengine/test10/ticket-group-21.summary
++++ b/pengine/test10/ticket-group-21.summary
+@@ -8,6 +8,7 @@ Online: [ node1 node2 ]
+ rsc2 (ocf::pacemaker:Dummy): Started node2
+
+ Transition Summary:
++ * Fence node2
+ * Stop rsc1 (node2)
+ * Stop rsc2 (node2)
+
+diff --git a/pengine/test10/ticket-group-9.summary b/pengine/test10/ticket-group-9.summary
+index 9027d2c..eab14a9 100644
+--- a/pengine/test10/ticket-group-9.summary
++++ b/pengine/test10/ticket-group-9.summary
+@@ -8,6 +8,7 @@ Online: [ node1 node2 ]
+ rsc2 (ocf::pacemaker:Dummy): Started node2
+
+ Transition Summary:
++ * Fence node2
+ * Stop rsc1 (node2)
+ * Stop rsc2 (node2)
+
+diff --git a/pengine/test10/ticket-master-21.summary b/pengine/test10/ticket-master-21.summary
+index b228696..a107a38 100644
+--- a/pengine/test10/ticket-master-21.summary
++++ b/pengine/test10/ticket-master-21.summary
+@@ -8,6 +8,7 @@ Online: [ node1 node2 ]
+ Slaves: [ node2 ]
+
+ Transition Summary:
++ * Fence node1
+ * Move rsc_stonith (Started node1 -> node2)
+ * Demote rsc1:0 (Master -> Stopped node1)
+
+diff --git a/pengine/test10/ticket-master-9.summary b/pengine/test10/ticket-master-9.summary
+index b228696..a107a38 100644
+--- a/pengine/test10/ticket-master-9.summary
++++ b/pengine/test10/ticket-master-9.summary
+@@ -8,6 +8,7 @@ Online: [ node1 node2 ]
+ Slaves: [ node2 ]
+
+ Transition Summary:
++ * Fence node1
+ * Move rsc_stonith (Started node1 -> node2)
+ * Demote rsc1:0 (Master -> Stopped node1)
+
+diff --git a/pengine/test10/ticket-primitive-21.summary b/pengine/test10/ticket-primitive-21.summary
+index 6c523fe..918b743 100644
+--- a/pengine/test10/ticket-primitive-21.summary
++++ b/pengine/test10/ticket-primitive-21.summary
+@@ -6,6 +6,7 @@ Online: [ node1 node2 ]
+ rsc1 (ocf::pacemaker:Dummy): Started node2
+
+ Transition Summary:
++ * Fence node2
+ * Stop rsc1 (node2)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/ticket-primitive-9.summary b/pengine/test10/ticket-primitive-9.summary
+index 6c523fe..918b743 100644
+--- a/pengine/test10/ticket-primitive-9.summary
++++ b/pengine/test10/ticket-primitive-9.summary
+@@ -6,6 +6,7 @@ Online: [ node1 node2 ]
+ rsc1 (ocf::pacemaker:Dummy): Started node2
+
+ Transition Summary:
++ * Fence node2
+ * Stop rsc1 (node2)
+
+ Executing cluster transition:
+diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary
+index 05f8003..03506a3 100644
+--- a/pengine/test10/unfence-definition.summary
++++ b/pengine/test10/unfence-definition.summary
+@@ -12,6 +12,11 @@ Online: [ virt-1 virt-2 virt-3 ]
+ Stopped: [ virt-2 virt-3 virt-4 ]
+
+ Transition Summary:
++ * Fence virt-4
++ * Fence virt-4
++ * Fence virt-3
++ * Fence virt-2
++ * Fence virt-1
+ * Restart fencing (Started virt-1)
+ * Restart dlm:0 (Started virt-1)
+ * Start dlm:2 (virt-3)
+diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary
+index 41fed90..5cc4a4b 100644
+--- a/pengine/test10/unfence-parameters.summary
++++ b/pengine/test10/unfence-parameters.summary
+@@ -12,6 +12,11 @@ Online: [ virt-1 virt-2 virt-3 ]
+ Stopped: [ virt-2 virt-3 virt-4 ]
+
+ Transition Summary:
++ * Fence virt-4
++ * Fence virt-4
++ * Fence virt-3
++ * Fence virt-2
++ * Fence virt-1
+ * Restart fencing (Started virt-1)
+ * Restart dlm:0 (Started virt-1)
+ * Restart dlm:1 (Started virt-2)
+diff --git a/pengine/test10/unfence-startup.summary b/pengine/test10/unfence-startup.summary
+index 76bc0fc..6cd38ad 100644
+--- a/pengine/test10/unfence-startup.summary
++++ b/pengine/test10/unfence-startup.summary
+@@ -12,6 +12,11 @@ Online: [ virt-1 virt-2 virt-3 ]
+ Stopped: [ virt-2 virt-3 virt-4 ]
+
+ Transition Summary:
++ * Fence virt-4
++ * Fence virt-4
++ * Fence virt-3
++ * Fence virt-2
++ * Fence virt-1
+ * Start dlm:2 (virt-3)
+ * Start clvmd:1 (virt-2)
+ * Start clvmd:2 (virt-3)
+diff --git a/pengine/test10/unmanaged-master.summary b/pengine/test10/unmanaged-master.summary
+index 024179a..66a8748 100644
+--- a/pengine/test10/unmanaged-master.summary
++++ b/pengine/test10/unmanaged-master.summary
+@@ -27,6 +27,8 @@ OFFLINE: [ pcmk-3 pcmk-4 ]
+ Stopped: [ pcmk-3 pcmk-4 ]
+
+ Transition Summary:
++ * Shutdown pcmk-2
++ * Shutdown pcmk-1
+
+ Executing cluster transition:
+ * Cluster action: do_shutdown on pcmk-2
+diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary
+index 1872e9a..5f70a66 100644
+--- a/pengine/test10/whitebox-fail1.summary
++++ b/pengine/test10/whitebox-fail1.summary
+@@ -14,6 +14,7 @@ Containers: [ lxc2:container2 ]
+ D (ocf::pacemaker:Dummy): Started 18node1
+
+ Transition Summary:
++ * Fence lxc1 (resource: container1)
+ * Recover container1 (Started 18node2)
+ * Recover M:4 (Started lxc1)
+ * Recover B (Started lxc1)
+diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary
+index 5db6588..2922f16 100644
+--- a/pengine/test10/whitebox-fail2.summary
++++ b/pengine/test10/whitebox-fail2.summary
+@@ -14,6 +14,7 @@ Containers: [ lxc2:container2 ]
+ D (ocf::pacemaker:Dummy): Started 18node1
+
+ Transition Summary:
++ * Fence lxc1 (resource: container1)
+ * Recover container1 (Started 18node2)
+ * Recover M:4 (Started lxc1)
+ * Recover B (Started lxc1)
+diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary
+index 50a3446..31cc4a5 100644
+--- a/pengine/test10/whitebox-imply-stop-on-fence.summary
++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary
+@@ -25,6 +25,9 @@ Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ]
+ vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01
+
+ Transition Summary:
++ * Fence lxc-02_kiff-01 (resource: R-lxc-02_kiff-01)
++ * Fence lxc-01_kiff-01 (resource: R-lxc-01_kiff-01)
++ * Fence kiff-01
+ * Move fence-kiff-02 (Started kiff-01 -> kiff-02)
+ * Stop dlm:0 (kiff-01)
+ * Stop clvmd:0 (kiff-01)
+diff --git a/pengine/test10/whitebox-ms-ordering.summary b/pengine/test10/whitebox-ms-ordering.summary
+index 365cf8d..fcdef66 100644
+--- a/pengine/test10/whitebox-ms-ordering.summary
++++ b/pengine/test10/whitebox-ms-ordering.summary
+@@ -9,6 +9,8 @@ Online: [ 18node1 18node2 18node3 ]
+ Stopped: [ 18node1 18node2 18node3 ]
+
+ Transition Summary:
++ * Fence lxc2 (resource: container2)
++ * Fence lxc1 (resource: container1)
+ * Start container1 (18node1)
+ * Start container2 (18node1)
+ * Recover lxc-ms:0 (Master lxc1)
+diff --git a/pengine/test10/whitebox-unexpectedly-running.summary b/pengine/test10/whitebox-unexpectedly-running.summary
+index 0b5ca42..ed0a5bd 100644
+--- a/pengine/test10/whitebox-unexpectedly-running.summary
++++ b/pengine/test10/whitebox-unexpectedly-running.summary
+@@ -5,6 +5,7 @@ Online: [ 18builder ]
+ FAKE (ocf::pacemaker:Dummy): FAILED 18builder
+
+ Transition Summary:
++ * Fence remote1 (resource: FAKE)
+ * Recover FAKE (Started 18builder)
+ * Start remote1 (18builder)
+
+diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c
+index 2cc578d..aaaf0aa 100644
+--- a/tools/crm_simulate.c
++++ b/tools/crm_simulate.c
+@@ -892,6 +892,7 @@ main(int argc, char **argv)
+ || modified ? "\n" : "");
+ fflush(stdout);
+
++ LogNodeActions(&data_set, TRUE);
+ for (gIter = data_set.resources; gIter != NULL; gIter = gIter->next) {
+ resource_t *rsc = (resource_t *) gIter->data;
+
+--
+1.8.3.1
+
diff --git a/SOURCES/074-bundle-ordering.patch b/SOURCES/074-bundle-ordering.patch
new file mode 100644
index 0000000..15d7335
--- /dev/null
+++ b/SOURCES/074-bundle-ordering.patch
@@ -0,0 +1,17008 @@
+From 557ad15ec500f8edf0356d3ffcd880965ae0de1a Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Thu, 15 Jun 2017 10:47:04 +1000
+Subject: [PATCH 1/8] Feature: PE: Implement bundle ordering
+
+---
+ pengine/allocate.c | 8 ++--
+ pengine/allocate.h | 4 +-
+ pengine/clone.c | 51 +++++++++++--------------
+ pengine/container.c | 82 ++++++++++++++++++++++++++++++++++++++-
+ pengine/master.c | 108 ++++++++++++++++++++++------------------------------
+ pengine/utils.c | 12 ++++++
+ pengine/utils.h | 1 +
+ 7 files changed, 170 insertions(+), 96 deletions(-)
+
+diff --git a/pengine/allocate.c b/pengine/allocate.c
+index f2987cc..8f15282 100644
+--- a/pengine/allocate.c
++++ b/pengine/allocate.c
+@@ -1860,9 +1860,11 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set)
+ * recurring monitors to be restarted, even if just
+ * the connection was re-established
+ */
+- custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+- action->rsc, NULL, action,
+- pe_order_preserve | pe_order_runnable_left | pe_order_implies_then, data_set);
++ if(task != no_action) {
++ custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
++ action->rsc, NULL, action,
++ pe_order_preserve | pe_order_runnable_left | pe_order_implies_then, data_set);
++ }
+ } else {
+ custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL,
+ action->rsc, NULL, action,
+diff --git a/pengine/allocate.h b/pengine/allocate.h
+index d89943d..9a30b80 100644
+--- a/pengine/allocate.h
++++ b/pengine/allocate.h
+@@ -179,6 +179,8 @@ gboolean update_action_flags(action_t * action, enum pe_action_flags flags, cons
+ gboolean update_action(action_t * action);
+ void complex_set_cmds(resource_t * rsc);
+
+-
++void master_promotion_constraints(resource_t * rsc, pe_working_set_t * data_set);
++void clone_create_pseudo_actions(
++ resource_t * rsc, GListPtr children, notify_data_t **start_notify, notify_data_t **stop_notify, pe_working_set_t * data_set);
+
+ #endif
+diff --git a/pengine/clone.c b/pengine/clone.c
+index 51338d6..a51677a 100644
+--- a/pengine/clone.c
++++ b/pengine/clone.c
+@@ -805,6 +805,17 @@ child_ordering_constraints(resource_t * rsc, pe_working_set_t * data_set)
+ void
+ clone_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+ {
++ clone_variant_data_t *clone_data = NULL;
++
++ get_clone_variant_data(clone_data, rsc);
++ clone_create_pseudo_actions(rsc, rsc->children, &clone_data->start_notify, &clone_data->stop_notify,data_set);
++ child_ordering_constraints(rsc, data_set);
++}
++
++void
++clone_create_pseudo_actions(
++ resource_t * rsc, GListPtr children, notify_data_t **start_notify, notify_data_t **stop_notify, pe_working_set_t * data_set)
++{
+ gboolean child_active = FALSE;
+ gboolean child_starting = FALSE;
+ gboolean child_stopping = FALSE;
+@@ -816,14 +827,9 @@ clone_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+ action_t *start = NULL;
+ action_t *started = NULL;
+
+- GListPtr gIter = rsc->children;
+- clone_variant_data_t *clone_data = NULL;
+-
+- get_clone_variant_data(clone_data, rsc);
+-
+ pe_rsc_trace(rsc, "Creating actions for %s", rsc->id);
+
+- for (; gIter != NULL; gIter = gIter->next) {
++ for (GListPtr gIter = children; gIter != NULL; gIter = gIter->next) {
+ resource_t *child_rsc = (resource_t *) gIter->data;
+ gboolean starting = FALSE;
+ gboolean stopping = FALSE;
+@@ -839,42 +845,31 @@ clone_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+ }
+
+ /* start */
+- start = start_action(rsc, NULL, !child_starting);
+- started = custom_action(rsc, started_key(rsc),
+- RSC_STARTED, NULL, !child_starting, TRUE, data_set);
+-
+- update_action_flags(start, pe_action_pseudo | pe_action_runnable, __FUNCTION__, __LINE__);
+- update_action_flags(started, pe_action_pseudo, __FUNCTION__, __LINE__);
++ start = create_pseudo_resource_op(rsc, RSC_START, !child_starting, TRUE, data_set);
++ started = create_pseudo_resource_op(rsc, RSC_STARTED, !child_starting, FALSE, data_set);
+ started->priority = INFINITY;
+
+ if (child_active || child_starting) {
+ update_action_flags(started, pe_action_runnable, __FUNCTION__, __LINE__);
+ }
+
+- child_ordering_constraints(rsc, data_set);
+- if (clone_data->start_notify == NULL) {
+- clone_data->start_notify =
+- create_notification_boundaries(rsc, RSC_START, start, started, data_set);
++ if (start_notify != NULL && *start_notify == NULL) {
++ *start_notify = create_notification_boundaries(rsc, RSC_START, start, started, data_set);
+ }
+
+ /* stop */
+- stop = stop_action(rsc, NULL, !child_stopping);
+- stopped = custom_action(rsc, stopped_key(rsc),
+- RSC_STOPPED, NULL, !child_stopping, TRUE, data_set);
+-
++ stop = create_pseudo_resource_op(rsc, RSC_STOP, !child_stopping, TRUE, data_set);
++ stopped = create_pseudo_resource_op(rsc, RSC_STOPPED, !child_stopping, TRUE, data_set);
+ stopped->priority = INFINITY;
+- update_action_flags(stop, pe_action_pseudo | pe_action_runnable, __FUNCTION__, __LINE__);
+ if (allow_dependent_migrations) {
+ update_action_flags(stop, pe_action_migrate_runnable, __FUNCTION__, __LINE__);
+ }
+- update_action_flags(stopped, pe_action_pseudo | pe_action_runnable, __FUNCTION__, __LINE__);
+- if (clone_data->stop_notify == NULL) {
+- clone_data->stop_notify =
+- create_notification_boundaries(rsc, RSC_STOP, stop, stopped, data_set);
+
+- if (clone_data->stop_notify && clone_data->start_notify) {
+- order_actions(clone_data->stop_notify->post_done, clone_data->start_notify->pre,
+- pe_order_optional);
++ if (stop_notify != NULL && *stop_notify == NULL) {
++ *stop_notify = create_notification_boundaries(rsc, RSC_STOP, stop, stopped, data_set);
++
++ if (*stop_notify && *start_notify) {
++ order_actions((*stop_notify)->post_done, (*start_notify)->pre, pe_order_optional);
+ }
+ }
+ }
+diff --git a/pengine/container.c b/pengine/container.c
+index 58f6fca..d648e0f 100644
+--- a/pengine/container.c
++++ b/pengine/container.c
+@@ -122,6 +122,8 @@ container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
+ void
+ container_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+ {
++ pe_action_t *action = NULL;
++ GListPtr containers = NULL;
+ container_variant_data_t *container_data = NULL;
+
+ CRM_CHECK(rsc != NULL, return);
+@@ -136,15 +138,32 @@ container_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+ }
+ if(tuple->docker) {
+ tuple->docker->cmds->create_actions(tuple->docker, data_set);
++ containers = g_list_append(containers, tuple->docker);
+ }
+ if(tuple->remote) {
+ tuple->remote->cmds->create_actions(tuple->remote, data_set);
+ }
+ }
+
++ clone_create_pseudo_actions(rsc, containers, NULL, NULL, data_set);
++
+ if(container_data->child) {
+ container_data->child->cmds->create_actions(container_data->child, data_set);
++
++ if(container_data->child->variant == pe_master) {
++ /* promote */
++ action = create_pseudo_resource_op(rsc, RSC_PROMOTE, TRUE, TRUE, data_set);
++ action = create_pseudo_resource_op(rsc, RSC_PROMOTED, TRUE, TRUE, data_set);
++ action->priority = INFINITY;
++
++ /* demote */
++ action = create_pseudo_resource_op(rsc, RSC_DEMOTE, TRUE, TRUE, data_set);
++ action = create_pseudo_resource_op(rsc, RSC_DEMOTED, TRUE, TRUE, data_set);
++ action->priority = INFINITY;
++ }
+ }
++
++ g_list_free(containers);
+ }
+
+ void
+@@ -155,12 +174,38 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+ CRM_CHECK(rsc != NULL, return);
+
+ get_container_variant_data(container_data, rsc);
++
++ if(container_data->child) {
++ new_rsc_order(rsc, RSC_START, container_data->child, RSC_START, pe_order_implies_first_printed, data_set);
++ new_rsc_order(rsc, RSC_STOP, container_data->child, RSC_STOP, pe_order_implies_first_printed, data_set);
++
++ if(container_data->child->children) {
++ new_rsc_order(container_data->child, RSC_STARTED, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
++ new_rsc_order(container_data->child, RSC_STOPPED, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set);
++ } else {
++ new_rsc_order(container_data->child, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
++ new_rsc_order(container_data->child, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set);
++ }
++ }
++
+ for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
+ container_grouping_t *tuple = (container_grouping_t *)gIter->data;
+
+ CRM_ASSERT(tuple);
+- if(tuple->docker) {
+- tuple->docker->cmds->internal_constraints(tuple->docker, data_set);
++ CRM_ASSERT(tuple->docker);
++
++ tuple->docker->cmds->internal_constraints(tuple->docker, data_set);
++
++ order_start_start(rsc, tuple->docker, pe_order_runnable_left | pe_order_implies_first_printed);
++
++ if(tuple->child) {
++ order_stop_stop(rsc, tuple->child, pe_order_implies_first_printed);
++
++ } else {
++ order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed);
++ new_rsc_order(tuple->docker, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
++ new_rsc_order(tuple->docker, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed,
++ data_set);
+ }
+
+ if(tuple->ip) {
+@@ -194,10 +239,31 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+
+ if(container_data->child) {
+ container_data->child->cmds->internal_constraints(container_data->child, data_set);
++ if(container_data->child->variant == pe_master) {
++ master_promotion_constraints(rsc, data_set);
++
++ /* child demoted before global demoted */
++ new_rsc_order(container_data->child, RSC_DEMOTED, rsc, RSC_DEMOTED, pe_order_implies_then_printed, data_set);
++
++ /* global demote before child demote */
++ new_rsc_order(rsc, RSC_DEMOTE, container_data->child, RSC_DEMOTE, pe_order_implies_first_printed, data_set);
++
++ /* child promoted before global promoted */
++ new_rsc_order(container_data->child, RSC_PROMOTED, rsc, RSC_PROMOTED, pe_order_implies_then_printed, data_set);
++
++ /* global promote before child promote */
++ new_rsc_order(rsc, RSC_PROMOTE, container_data->child, RSC_PROMOTE, pe_order_implies_first_printed, data_set);
++ }
++
++ } else {
++// int type = pe_order_optional | pe_order_implies_then | pe_order_restart;
++// custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL,
++// rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, pe_order_optional, data_set);
+ }
+ }
+
+
++
+ static resource_t *
+ find_compatible_tuple_by_node(resource_t * rsc_lh, node_t * candidate, resource_t * rsc,
+ enum rsc_role_e filter, gboolean current)
+@@ -336,6 +402,10 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum
+ enum pe_action_flags filter, enum pe_ordering type)
+ {
+ enum pe_graph_flags changed = pe_graph_none;
++
++ // At the point we need to force container X to stop because
++ // resource Y needs to stop, here is where we'd implement that
++
+ return changed;
+ }
+
+@@ -359,6 +429,14 @@ container_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
+ tuple->ip->cmds->rsc_location(tuple->ip, constraint);
+ }
+ }
++
++ if(container_data->child && (constraint->role_filter == RSC_ROLE_SLAVE || constraint->role_filter == RSC_ROLE_MASTER)) {
++ // Translate the node into container names running on that node
++ crm_err("Applying constraint %s", constraint->id);
++ container_data->child->cmds->rsc_location(container_data->child, constraint);
++ container_data->child->rsc_location = g_list_prepend(container_data->child->rsc_location, constraint);
++ crm_err("Added %d location constraints to %s", g_list_length(container_data->child->rsc_location), container_data->child->id);
++ }
+ }
+
+ void
+diff --git a/pengine/master.c b/pengine/master.c
+index f6fcad3..93e5186 100644
+--- a/pengine/master.c
++++ b/pengine/master.c
+@@ -139,27 +139,26 @@ master_update_pseudo_status(resource_t * rsc, gboolean * demoting, gboolean * pr
+ }
+ }
+
+-#define apply_master_location(list) do { \
+- gIter2 = list; \
+- for(; gIter2 != NULL; gIter2 = gIter2->next) { \
+- rsc_to_node_t *cons = (rsc_to_node_t*)gIter2->data; \
+- \
+- cons_node = NULL; \
+- if(cons->role_filter == RSC_ROLE_MASTER) { \
+- pe_rsc_trace(rsc, "Applying %s to %s", \
+- cons->id, child_rsc->id); \
+- cons_node = pe_find_node_id( \
+- cons->node_list_rh, chosen->details->id); \
+- } \
+- if(cons_node != NULL) { \
+- int new_priority = merge_weights( \
+- child_rsc->priority, cons_node->weight); \
+- pe_rsc_trace(rsc, "\t%s: %d->%d (%d)", child_rsc->id, \
+- child_rsc->priority, new_priority, cons_node->weight); \
+- child_rsc->priority = new_priority; \
+- } \
+- } \
+- } while(0)
++static void apply_master_location(resource_t *child, GListPtr location_constraints, pe_node_t *chosen)
++{
++ for(GListPtr gIter = location_constraints; gIter != NULL; gIter = gIter->next) {
++ pe_node_t *cons_node = NULL;
++ rsc_to_node_t *cons = (rsc_to_node_t*)gIter->data;
++
++ if(cons->role_filter == RSC_ROLE_MASTER) {
++ pe_rsc_trace(child, "Applying %s to %s", cons->id, child->id);
++ cons_node = pe_find_node_id(cons->node_list_rh, chosen->details->id);
++ }
++ if(cons_node != NULL) {
++ int new_priority = merge_weights(child->priority, cons_node->weight);
++ pe_rsc_trace(child, "\t%s[%s]: %d -> %d (%d)", child->id, cons_node->details->uname,
++ child->priority, new_priority, cons_node->weight);
++ crm_err("\t%s[%s]: %d -> %d (%d)", child->id, cons_node->details->uname,
++ child->priority, new_priority, cons_node->weight);
++ child->priority = new_priority;
++ }
++ }
++}
+
+ static node_t *
+ can_be_master(resource_t * rsc)
+@@ -290,8 +289,7 @@ master_promotion_order(resource_t * rsc, pe_working_set_t * data_set)
+ pe_rsc_trace(rsc, "Merging weights for %s", rsc->id);
+ set_bit(rsc->flags, pe_rsc_merging);
+
+- gIter = rsc->children;
+- for (; gIter != NULL; gIter = gIter->next) {
++ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
+ resource_t *child = (resource_t *) gIter->data;
+
+ pe_rsc_trace(rsc, "Sort index: %s = %d", child->id, child->sort_index);
+@@ -639,7 +637,6 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
+ GHashTableIter iter;
+ node_t *node = NULL;
+ node_t *chosen = NULL;
+- node_t *cons_node = NULL;
+ enum rsc_role_e next_role = RSC_ROLE_UNKNOWN;
+
+ char score[33];
+@@ -672,8 +669,7 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
+ /*
+ * assign priority
+ */
+- gIter = rsc->children;
+- for (; gIter != NULL; gIter = gIter->next) {
++ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
+ GListPtr list = NULL;
+ resource_t *child_rsc = (resource_t *) gIter->data;
+
+@@ -723,11 +719,11 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
+ crm_err("Unknown resource role: %d for %s", next_role, child_rsc->id));
+ }
+
+- apply_master_location(child_rsc->rsc_location);
+- apply_master_location(rsc->rsc_location);
++ apply_master_location(child_rsc, child_rsc->rsc_location, chosen);
++ crm_err("Applying %d location constraints for %s", g_list_length(rsc->rsc_location), rsc->id);
++ apply_master_location(child_rsc, rsc->rsc_location, chosen);
+
+- gIter2 = child_rsc->rsc_cons;
+- for (; gIter2 != NULL; gIter2 = gIter2->next) {
++ for (gIter2 = child_rsc->rsc_cons; gIter2 != NULL; gIter2 = gIter2->next) {
+ rsc_colocation_t *cons = (rsc_colocation_t *) gIter2->data;
+
+ child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->rsc_rh, cons);
+@@ -746,8 +742,7 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
+
+ /* mark the first N as masters */
+
+- gIter = rsc->children;
+- for (; gIter != NULL; gIter = gIter->next) {
++ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) {
+ resource_t *child_rsc = (resource_t *) gIter->data;
+ score2char_stack(child_rsc->sort_index, score, len);
+
+@@ -840,20 +835,9 @@ master_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+ }
+
+ /* promote */
+- action = promote_action(rsc, NULL, !any_promoting);
+- action_complete = custom_action(rsc, promoted_key(rsc),
+- RSC_PROMOTED, NULL, !any_promoting, TRUE, data_set);
+-
++ action = create_pseudo_resource_op(rsc, RSC_PROMOTE, !any_promoting, TRUE, data_set);
++ action_complete = create_pseudo_resource_op(rsc, RSC_PROMOTED, !any_promoting, TRUE, data_set);
+ action_complete->priority = INFINITY;
+- update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__);
+- update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__);
+- update_action_flags(action_complete, pe_action_pseudo, __FUNCTION__, __LINE__);
+- update_action_flags(action_complete, pe_action_runnable, __FUNCTION__, __LINE__);
+-
+- if (clone_data->masters_allocated > 0) {
+- update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__);
+- update_action_flags(action_complete, pe_action_runnable, __FUNCTION__, __LINE__);
+- }
+
+ child_promoting_constraints(clone_data, pe_order_optional,
+ rsc, NULL, last_promote_rsc, data_set);
+@@ -864,16 +848,10 @@ master_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+ }
+
+ /* demote */
+- action = demote_action(rsc, NULL, !any_demoting);
+- action_complete = custom_action(rsc, demoted_key(rsc),
+- RSC_DEMOTED, NULL, !any_demoting, TRUE, data_set);
++ action = create_pseudo_resource_op(rsc, RSC_DEMOTE, !any_demoting, TRUE, data_set);
++ action_complete = create_pseudo_resource_op(rsc, RSC_DEMOTED, !any_demoting, TRUE, data_set);
+ action_complete->priority = INFINITY;
+
+- update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__);
+- update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__);
+- update_action_flags(action_complete, pe_action_pseudo, __FUNCTION__, __LINE__);
+- update_action_flags(action_complete, pe_action_runnable, __FUNCTION__, __LINE__);
+-
+ child_demoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_demote_rsc, data_set);
+
+ if (clone_data->demote_notify == NULL) {
+@@ -908,16 +886,8 @@ master_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+ }
+
+ void
+-master_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
++master_promotion_constraints(resource_t * rsc, pe_working_set_t * data_set)
+ {
+- GListPtr gIter = rsc->children;
+- resource_t *last_rsc = NULL;
+- clone_variant_data_t *clone_data = NULL;
+-
+- get_clone_variant_data(clone_data, rsc);
+-
+- clone_internal_constraints(rsc, data_set);
+-
+ /* global stopped before start */
+ new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set);
+
+@@ -938,6 +908,20 @@ master_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+
+ /* global demoted before promote */
+ new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE, pe_order_optional, data_set);
++}
++
++
++void
++master_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
++{
++ GListPtr gIter = rsc->children;
++ resource_t *last_rsc = NULL;
++ clone_variant_data_t *clone_data = NULL;
++
++ get_clone_variant_data(clone_data, rsc);
++
++ clone_internal_constraints(rsc, data_set);
++ master_promotion_constraints(rsc, data_set);
+
+ for (; gIter != NULL; gIter = gIter->next) {
+ resource_t *child_rsc = (resource_t *) gIter->data;
+diff --git a/pengine/utils.c b/pengine/utils.c
+index a587e58..0cc6381 100644
+--- a/pengine/utils.c
++++ b/pengine/utils.c
+@@ -422,3 +422,15 @@ can_run_any(GHashTable * nodes)
+
+ return FALSE;
+ }
++
++pe_action_t *
++create_pseudo_resource_op(resource_t * rsc, const char *task, bool optional, bool runnable, pe_working_set_t *data_set)
++{
++ pe_action_t *action = custom_action(rsc, generate_op_key(rsc->id, task, 0), task, NULL, optional, TRUE, data_set);
++ update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__);
++ update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__);
++ if(runnable) {
++ update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__);
++ }
++ return action;
++}
+diff --git a/pengine/utils.h b/pengine/utils.h
+index 79fd33d..10e7201 100644
+--- a/pengine/utils.h
++++ b/pengine/utils.h
+@@ -71,6 +71,7 @@ extern void calculate_utilization(GHashTable * current_utilization,
+ GHashTable * utilization, gboolean plus);
+
+ extern void process_utilization(resource_t * rsc, node_t ** prefer, pe_working_set_t * data_set);
++pe_action_t *create_pseudo_resource_op(resource_t * rsc, const char *task, bool optional, bool runnable, pe_working_set_t *data_set);
+
+ # define STONITH_UP "stonith_up"
+ # define STONITH_DONE "stonith_complete"
+--
+1.8.3.1
+
+
+From 356261e246b6aa46d6647eb16f09ff2dceeb1313 Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Thu, 15 Jun 2017 10:47:13 +1000
+Subject: [PATCH 2/8] Test: PE: Implement bundle ordering
+
+---
+ pengine/regression.sh | 6 +
+ pengine/test10/bundle-order-partial-start-2.dot | 51 ++
+ pengine/test10/bundle-order-partial-start-2.exp | 299 ++++++++
+ pengine/test10/bundle-order-partial-start-2.scores | 197 +++++
+ .../test10/bundle-order-partial-start-2.summary | 75 ++
+ pengine/test10/bundle-order-partial-start-2.xml | 387 ++++++++++
+ pengine/test10/bundle-order-partial-start.dot | 65 ++
+ pengine/test10/bundle-order-partial-start.exp | 375 ++++++++++
+ pengine/test10/bundle-order-partial-start.scores | 197 +++++
+ pengine/test10/bundle-order-partial-start.summary | 82 ++
+ pengine/test10/bundle-order-partial-start.xml | 379 ++++++++++
+ pengine/test10/bundle-order-partial-stop.dot | 194 +++++
+ pengine/test10/bundle-order-partial-stop.exp | 734 ++++++++++++++++++
+ pengine/test10/bundle-order-partial-stop.scores | 199 +++++
+ pengine/test10/bundle-order-partial-stop.summary | 114 +++
+ pengine/test10/bundle-order-partial-stop.xml | 421 +++++++++++
+ pengine/test10/bundle-order-partial.dot | 2 +
+ pengine/test10/bundle-order-partial.exp | 1 +
+ pengine/test10/bundle-order-partial.scores | 197 +++++
+ pengine/test10/bundle-order-partial.summary | 47 ++
+ pengine/test10/bundle-order-partial.xml | 421 +++++++++++
+ pengine/test10/bundle-order-startup.dot | 139 ++++
+ pengine/test10/bundle-order-startup.exp | 825 +++++++++++++++++++++
+ pengine/test10/bundle-order-startup.scores | 197 +++++
+ pengine/test10/bundle-order-startup.summary | 126 ++++
+ pengine/test10/bundle-order-startup.xml | 315 ++++++++
+ pengine/test10/bundle-order-stop.dot | 194 +++++
+ pengine/test10/bundle-order-stop.exp | 734 ++++++++++++++++++
+ pengine/test10/bundle-order-stop.scores | 199 +++++
+ pengine/test10/bundle-order-stop.summary | 114 +++
+ pengine/test10/bundle-order-stop.xml | 421 +++++++++++
+ 31 files changed, 7707 insertions(+)
+ create mode 100644 pengine/test10/bundle-order-partial-start-2.dot
+ create mode 100644 pengine/test10/bundle-order-partial-start-2.exp
+ create mode 100644 pengine/test10/bundle-order-partial-start-2.scores
+ create mode 100644 pengine/test10/bundle-order-partial-start-2.summary
+ create mode 100644 pengine/test10/bundle-order-partial-start-2.xml
+ create mode 100644 pengine/test10/bundle-order-partial-start.dot
+ create mode 100644 pengine/test10/bundle-order-partial-start.exp
+ create mode 100644 pengine/test10/bundle-order-partial-start.scores
+ create mode 100644 pengine/test10/bundle-order-partial-start.summary
+ create mode 100644 pengine/test10/bundle-order-partial-start.xml
+ create mode 100644 pengine/test10/bundle-order-partial-stop.dot
+ create mode 100644 pengine/test10/bundle-order-partial-stop.exp
+ create mode 100644 pengine/test10/bundle-order-partial-stop.scores
+ create mode 100644 pengine/test10/bundle-order-partial-stop.summary
+ create mode 100644 pengine/test10/bundle-order-partial-stop.xml
+ create mode 100644 pengine/test10/bundle-order-partial.dot
+ create mode 100644 pengine/test10/bundle-order-partial.exp
+ create mode 100644 pengine/test10/bundle-order-partial.scores
+ create mode 100644 pengine/test10/bundle-order-partial.summary
+ create mode 100644 pengine/test10/bundle-order-partial.xml
+ create mode 100644 pengine/test10/bundle-order-startup.dot
+ create mode 100644 pengine/test10/bundle-order-startup.exp
+ create mode 100644 pengine/test10/bundle-order-startup.scores
+ create mode 100644 pengine/test10/bundle-order-startup.summary
+ create mode 100644 pengine/test10/bundle-order-startup.xml
+ create mode 100644 pengine/test10/bundle-order-stop.dot
+ create mode 100644 pengine/test10/bundle-order-stop.exp
+ create mode 100644 pengine/test10/bundle-order-stop.scores
+ create mode 100644 pengine/test10/bundle-order-stop.summary
+ create mode 100644 pengine/test10/bundle-order-stop.xml
+
+diff --git a/pengine/regression.sh b/pengine/regression.sh
+index df449e0..e97b54b 100755
+--- a/pengine/regression.sh
++++ b/pengine/regression.sh
+@@ -802,6 +802,12 @@ do_test container-is-remote-node "Place resource within container when container
+ do_test bug-rh-1097457 "Kill user defined container/contents ordering"
+ do_test bug-cl-5247 "Graph loop when recovering m/s resource in a container"
+
++do_test bundle-order-startup "Bundle startup ordering"
++do_test bundle-order-partial-start "Bundle startup ordering when some dependancies are already running"
++do_test bundle-order-partial-start-2 "Bundle startup ordering when some dependancies and the container are already running"
++do_test bundle-order-stop "Bundle stop ordering"
++do_test bundle-order-partial-stop "Bundle startup ordering when some dependancies are already stopped"
++
+ echo ""
+ do_test whitebox-fail1 "Fail whitebox container rsc."
+ do_test whitebox-fail2 "Fail whitebox container rsc lrmd connection."
+diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot
+new file mode 100644
+index 0000000..59a8b15
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start-2.dot
+@@ -0,0 +1,51 @@
++digraph "g" {
++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold]
++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold]
++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold]
++"haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle_running_0" -> "galera-bundle_start_0" [ style = bold]
++"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
++"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold]
++"rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle_running_0" -> "galera-bundle_start_0" [ style = bold]
++"rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold]
++"rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold]
++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold]
++"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold]
++"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_promoted_0" -> "galera-bundle_start_0" [ style = bold]
++"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold]
++"redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold]
++"redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++}
+diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp
+new file mode 100644
+index 0000000..621332c
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start-2.exp
+@@ -0,0 +1,299 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0">
++ <action_set>
++ <rsc_op id="25" operation="monitor" operation_key="rabbitmq:0_monitor_10000" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_timeout="40000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="24" operation="start" operation_key="rabbitmq:0_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="24" operation="start" operation_key="rabbitmq:0_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_timeout="200000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="26" operation="start" operation_key="rabbitmq-bundle-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="2">
++ <action_set>
++ <rsc_op id="40" operation="monitor" operation_key="galera:0_monitor_30000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_role="Slave" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="38" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="39" operation="monitor" operation_key="galera:0_monitor_20000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="38" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="4">
++ <action_set>
++ <rsc_op id="38" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="41" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="64" operation="monitor" operation_key="redis_monitor_20000" internal_operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="63" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="6">
++ <action_set>
++ <rsc_op id="63" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="120000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="69" operation="promote" operation_key="redis-bundle-master_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <rsc_op id="90" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="8">
++ <action_set>
++ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="15" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="91" operation="start" operation_key="haproxy-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <rsc_op id="15" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="10" priority="1000000">
++ <action_set>
++ <pseudo_event id="92" operation="running" operation_key="haproxy-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <pseudo_event id="91" operation="start" operation_key="haproxy-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="12" priority="1000000">
++ <action_set>
++ <pseudo_event id="74" operation="promoted" operation_key="redis-bundle_promoted_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="70" operation="promoted" operation_key="redis-bundle-master_promoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13">
++ <action_set>
++ <pseudo_event id="73" operation="promote" operation_key="redis-bundle_promote_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="14" priority="1000000">
++ <action_set>
++ <pseudo_event id="70" operation="promoted" operation_key="redis-bundle-master_promoted_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="63" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15">
++ <action_set>
++ <pseudo_event id="69" operation="promote" operation_key="redis-bundle-master_promote_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="73" operation="promote" operation_key="redis-bundle_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16" priority="1000000">
++ <action_set>
++ <pseudo_event id="42" operation="running" operation_key="galera-bundle-master_running_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="38" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="41" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="17">
++ <action_set>
++ <pseudo_event id="41" operation="start" operation_key="galera-bundle-master_start_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="34" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="18" priority="1000000">
++ <action_set>
++ <pseudo_event id="35" operation="running" operation_key="galera-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="42" operation="running" operation_key="galera-bundle-master_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="19">
++ <action_set>
++ <pseudo_event id="34" operation="start" operation_key="galera-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="21" operation="running" operation_key="rabbitmq-bundle_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="74" operation="promoted" operation_key="redis-bundle_promoted_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="92" operation="running" operation_key="haproxy-bundle_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20" priority="1000000">
++ <action_set>
++ <pseudo_event id="27" operation="running" operation_key="rabbitmq-bundle-clone_running_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="24" operation="start" operation_key="rabbitmq:0_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="26" operation="start" operation_key="rabbitmq-bundle-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="21">
++ <action_set>
++ <pseudo_event id="26" operation="start" operation_key="rabbitmq-bundle-clone_start_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="20" operation="start" operation_key="rabbitmq-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="22" priority="1000000">
++ <action_set>
++ <pseudo_event id="21" operation="running" operation_key="rabbitmq-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="27" operation="running" operation_key="rabbitmq-bundle-clone_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="23">
++ <action_set>
++ <pseudo_event id="20" operation="start" operation_key="rabbitmq-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/bundle-order-partial-start-2.scores b/pengine/test10/bundle-order-partial-start-2.scores
+new file mode 100644
+index 0000000..f5e86de
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start-2.scores
+@@ -0,0 +1,197 @@
++Allocation scores:
++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:0 allocation score on undercloud: -INFINITY
++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY
++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: redis:0 allocation score on undercloud: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on undercloud: 0
++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on undercloud: INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on undercloud: 0
++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera:0 allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on undercloud: 0
++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle allocation score on undercloud: 0
++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0
++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 0
++container_color: rabbitmq:0 allocation score on undercloud: 0
++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on undercloud: 0
++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on undercloud: 0
++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:0 allocation score on undercloud: 0
++galera:0 promotion score on galera-bundle-0: -1
++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on undercloud: INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: galera:0 allocation score on galera-bundle-0: INFINITY
++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:0 allocation score on undercloud: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on undercloud: INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on undercloud: INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: redis:0 allocation score on undercloud: -INFINITY
++redis:0 promotion score on redis-bundle-0: 3000
+diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary
+new file mode 100644
+index 0000000..859ca25
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start-2.summary
+@@ -0,0 +1,75 @@
++
++Current cluster status:
++Online: [ undercloud ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped undercloud
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Stopped undercloud
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Slave undercloud
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud
++
++Transition Summary:
++ * Start rabbitmq:0 (rabbitmq-bundle-0)
++ * Start galera:0 (galera-bundle-0)
++ * Promote redis:0 (Slave -> Master redis-bundle-0)
++ * Start haproxy-bundle-docker-0 (undercloud)
++
++Executing cluster transition:
++ * Resource action: haproxy-bundle-docker-0 monitor on undercloud
++ * Pseudo action: haproxy-bundle_start_0
++ * Pseudo action: redis-bundle_promote_0
++ * Pseudo action: redis-bundle-master_promote_0
++ * Pseudo action: rabbitmq-bundle_start_0
++ * Resource action: redis promote on redis-bundle-0
++ * Resource action: haproxy-bundle-docker-0 start on undercloud
++ * Pseudo action: haproxy-bundle_running_0
++ * Pseudo action: redis-bundle-master_promoted_0
++ * Pseudo action: rabbitmq-bundle-clone_start_0
++ * Resource action: rabbitmq:0 start on rabbitmq-bundle-0
++ * Resource action: redis monitor=20000 on redis-bundle-0
++ * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud
++ * Pseudo action: redis-bundle_promoted_0
++ * Pseudo action: rabbitmq-bundle-clone_running_0
++ * Pseudo action: rabbitmq-bundle_running_0
++ * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0
++ * Pseudo action: galera-bundle_start_0
++ * Pseudo action: galera-bundle-master_start_0
++ * Resource action: galera:0 start on galera-bundle-0
++ * Pseudo action: galera-bundle-master_running_0
++ * Pseudo action: galera-bundle_running_0
++ * Resource action: galera:0 monitor=30000 on galera-bundle-0
++ * Resource action: galera:0 monitor=20000 on galera-bundle-0
++
++Revised cluster status:
++Online: [ undercloud ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Slave undercloud
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud
++
+diff --git a/pengine/test10/bundle-order-partial-start-2.xml b/pengine/test10/bundle-order-partial-start-2.xml
+new file mode 100644
+index 0000000..9c4d40c
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start-2.xml
+@@ -0,0 +1,387 @@
++<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="undercloud">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
++ <instance_attributes id="nodes-rabbitmq-bundle-0">
++ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@undercloud"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <bundle id="rabbitmq-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3121"/>
++ <storage>
++ <storage-mapping id="rabbitmq-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/rabbitmq.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="rabbitmq-cfg-data" options="ro" source-dir="/var/lib/config-data/rabbitmq/etc/rabbitmq" target-dir="/etc/rabbitmq"/>
++ <storage-mapping id="rabbitmq-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="rabbitmq-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="rabbitmq-lib" options="rw" source-dir="/var/lib/rabbitmq" target-dir="/var/lib/rabbitmq"/>
++ <storage-mapping id="rabbitmq-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="rabbitmq-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="rabbitmq-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster">
++ <instance_attributes id="rabbitmq-instance_attributes">
++ <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </instance_attributes>
++ <meta_attributes id="rabbitmq-meta_attributes">
++ <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="rabbitmq-monitor-interval-10" interval="10" name="monitor" timeout="40"/>
++ <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="galera-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
++ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
++ <instance_attributes id="galera-instance_attributes">
++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:undercloud"/>
++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://undercloud"/>
++ </instance_attributes>
++ <meta_attributes id="galera-meta_attributes">
++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
++ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
++ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="redis-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3124"/>
++ <storage>
++ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
++ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
++ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
++ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
++ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
++ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
++ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
++ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
++ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
++ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <primitive class="ocf" id="ip-192.168.122.254" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.254-instance_attributes">
++ <nvpair id="ip-192.168.122.254-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.254-instance_attributes-ip" name="ip" value="192.168.122.254"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.254-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.254-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.254-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.250" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.250-instance_attributes">
++ <nvpair id="ip-192.168.122.250-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.250-instance_attributes-ip" name="ip" value="192.168.122.250"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.250-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.250-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.250-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.249" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.249-instance_attributes">
++ <nvpair id="ip-192.168.122.249-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.249-instance_attributes-ip" name="ip" value="192.168.122.249"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.249-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.249-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.249-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.253" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.253-instance_attributes">
++ <nvpair id="ip-192.168.122.253-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.253-instance_attributes-ip" name="ip" value="192.168.122.253"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.253-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.253-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.253-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.247" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.247-instance_attributes">
++ <nvpair id="ip-192.168.122.247-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.247-instance_attributes-ip" name="ip" value="192.168.122.247"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.247-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.247-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.247-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.248" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.248-instance_attributes">
++ <nvpair id="ip-192.168.122.248-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.248-instance_attributes-ip" name="ip" value="192.168.122.248"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.248-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.248-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.248-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ </bundle>
++ <bundle id="openstack-cinder-volume">
++ <docker run-command="/bin/bash /usr/local/bin/kolla_start" network="host" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" options="--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1"/>
++ <storage>
++ <storage-mapping target-dir="/var/lib/kolla/config_files/config.json" options="ro" id="cinder-volume-cfg-files" source-dir="/var/lib/kolla/config_files/cinder_volume.json"/>
++ <storage-mapping target-dir="/etc/cinder" options="ro" id="cinder-volume-cfg-data" source-dir="/var/lib/config-data/cinder/etc/cinder"/>
++ <storage-mapping target-dir="/etc/hosts" options="ro" id="cinder-volume-hosts" source-dir="/etc/hosts"/>
++ <storage-mapping target-dir="/etc/localtime" options="ro" id="cinder-volume-localtime" source-dir="/etc/localtime"/>
++ <storage-mapping target-dir="/dev" options="rw" id="cinder-volume-dev" source-dir="/dev"/>
++ <storage-mapping target-dir="/run" options="rw" id="cinder-volume-run" source-dir="/run"/>
++ <storage-mapping target-dir="/sys" options="rw" id="cinder-volume-sys" source-dir="/sys"/>
++ <storage-mapping target-dir="/lib/modules" options="ro" id="cinder-lib-modules" source-dir="/lib/modules"/>
++ <storage-mapping target-dir="/etc/iscsi" options="rw" id="cinder-volume-iscsi" source-dir="/etc/iscsi"/>
++ <storage-mapping target-dir="/var/lib/cinder" options="rw" id="cinder-volume-var-lib-cinder" source-dir="/var/lib/cinder"/>
++ <storage-mapping target-dir="/var/log/cinder" options="rw" id="cinder-volume-var-log" source-dir="/var/log/containers/cinder"/>
++ </storage>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-rabbitmq-bundle" resource-discovery="exclusive" rsc="rabbitmq-bundle">
++ <rule id="location-rabbitmq-bundle-rule" score="0">
++ <expression attribute="rabbitmq-role" id="location-rabbitmq-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
++ <rule id="location-galera-bundle-rule" score="0">
++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
++ <rule id="location-redis-bundle-rule" score="0">
++ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.254" resource-discovery="exclusive" rsc="ip-192.168.122.254">
++ <rule id="location-ip-192.168.122.254-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.254-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.250" resource-discovery="exclusive" rsc="ip-192.168.122.250">
++ <rule id="location-ip-192.168.122.250-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.250-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.249" resource-discovery="exclusive" rsc="ip-192.168.122.249">
++ <rule id="location-ip-192.168.122.249-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.249-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.253" resource-discovery="exclusive" rsc="ip-192.168.122.253">
++ <rule id="location-ip-192.168.122.253-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.253-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.247" resource-discovery="exclusive" rsc="ip-192.168.122.247">
++ <rule id="location-ip-192.168.122.247-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.247-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.248" resource-discovery="exclusive" rsc="ip-192.168.122.248">
++ <rule id="location-ip-192.168.122.248-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.248-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_order first="ip-192.168.122.254" first-action="start" id="order-ip-192.168.122.254-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.254-haproxy-bundle-INFINITY" rsc="ip-192.168.122.254" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.250" first-action="start" id="order-ip-192.168.122.250-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.250-haproxy-bundle-INFINITY" rsc="ip-192.168.122.250" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.249" first-action="start" id="order-ip-192.168.122.249-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.249-haproxy-bundle-INFINITY" rsc="ip-192.168.122.249" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.253" first-action="start" id="order-ip-192.168.122.253-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.253-haproxy-bundle-INFINITY" rsc="ip-192.168.122.253" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.247" first-action="start" id="order-ip-192.168.122.247-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.247-haproxy-bundle-INFINITY" rsc="ip-192.168.122.247" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.248" first-action="start" id="order-ip-192.168.122.248-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.248-haproxy-bundle-INFINITY" rsc="ip-192.168.122.248" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_colocation id="colocation-rabbit-with-haproxy-bundle-INFINITY" rsc="rabbitmq-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="haproxy-bundle" first-action="start" id="order-1" kind="Mandatory" then="galera-bundle" then-action="start"/>
++ <rsc_order first="rabbitmq-bundle" first-action="start" id="order-2" kind="Mandatory" then="galera-bundle" then-action="start"/>
++ <rsc_order first="redis-bundle" first-action="promote" id="order-3" kind="Mandatory" then="galera-bundle" then-action="start"/>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="undercloud" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="1">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="10" rc-code="0" op-status="0" interval="0" last-run="1496303575" last-rc-change="1496303575" exec-time="850" queue-time="0" op-digest="50baee2f0a77a08b0616f0cf07ab724b"/>
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_monitor_60000" operation_key="rabbitmq-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303576" exec-time="76" queue-time="0" op-digest="25cb53ea3977a912bf4c0f70fe9e3c21"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0">
++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1496303576" last-rc-change="1496303576" exec-time="0" queue-time="0" op-digest="ee16760fa8c27e6585c1e35c244d7652" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="rabbitmq-bundle-0_monitor_60000" operation_key="rabbitmq-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303578" exec-time="0" queue-time="0" op-digest="92f557a57dfbb50be88282a4674ea681"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="16:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;16:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="17" rc-code="0" op-status="0" interval="0" last-run="1496303595" last-rc-change="1496303595" exec-time="920" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
++ <lrm_rsc_op id="galera-bundle-docker-0_monitor_60000" operation_key="galera-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="17:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;17:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="18" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303596" exec-time="70" queue-time="0" op-digest="f7e456f82e276e07f726e6e86e93bbb3"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0">
++ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="18:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;18:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1496303596" last-rc-change="1496303596" exec-time="0" queue-time="0" op-digest="edf22e374195d93e82f3c9cc174d519f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="galera-bundle-0_monitor_60000" operation_key="galera-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="20:9:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;20:9:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303600" exec-time="0" queue-time="0" op-digest="40c47455f6993c120f9a064ae51260db"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="824" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
++ <lrm_rsc_op id="redis-bundle-docker-0_monitor_60000" operation_key="redis-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="29" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303664" exec-time="74" queue-time="0" op-digest="81e9d490c493f8d22fab2822ec0bed8a"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
++ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="5" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="0" queue-time="0" op-digest="9a6281aab223af75359e5b5a07161162" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="redis-bundle-0_monitor_60000" operation_key="redis-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="6" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303667" exec-time="0" queue-time="0" op-digest="04ffb214f519838e579c97e8aa002054"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.254" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.254_last_0" operation_key="ip-192.168.122.254_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1496303683" last-rc-change="1496303683" exec-time="63" queue-time="0" op-digest="d6875d68a09b5550030b8b6b7cdc9294"/>
++ <lrm_rsc_op id="ip-192.168.122.254_monitor_10000" operation_key="ip-192.168.122.254_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="36" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303683" exec-time="32" queue-time="0" op-digest="5c1c6d0e79751e4002b12de513351a48"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.250" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.250_last_0" operation_key="ip-192.168.122.250_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="41" rc-code="0" op-status="0" interval="0" last-run="1496303696" last-rc-change="1496303696" exec-time="53" queue-time="0" op-digest="b33b44b3e9e535336108cb43f7474ef6"/>
++ <lrm_rsc_op id="ip-192.168.122.250_monitor_10000" operation_key="ip-192.168.122.250_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="42" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303696" exec-time="32" queue-time="0" op-digest="a4aec96753744d49ca9a798d3b0f12aa"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.249" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.249_last_0" operation_key="ip-192.168.122.249_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="47" rc-code="0" op-status="0" interval="0" last-run="1496303710" last-rc-change="1496303710" exec-time="52" queue-time="0" op-digest="22e58cb8beefb7810d8bf03d0d3a906d"/>
++ <lrm_rsc_op id="ip-192.168.122.249_monitor_10000" operation_key="ip-192.168.122.249_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="48" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303710" exec-time="30" queue-time="1" op-digest="860855adc72293b04b1becaeea73ced1"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.253" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.253_last_0" operation_key="ip-192.168.122.253_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1496303723" last-rc-change="1496303723" exec-time="69" queue-time="0" op-digest="f36f4ccbd5f16e4e335c65f0802ddc20"/>
++ <lrm_rsc_op id="ip-192.168.122.253_monitor_10000" operation_key="ip-192.168.122.253_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="54" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303724" exec-time="39" queue-time="0" op-digest="0ee9a624c731239cdbef5c9118cc2bbe"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.247" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.247_last_0" operation_key="ip-192.168.122.247_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="59" rc-code="0" op-status="0" interval="0" last-run="1496303737" last-rc-change="1496303737" exec-time="57" queue-time="0" op-digest="37013238bd139c2b589a9508ff4b5c07"/>
++ <lrm_rsc_op id="ip-192.168.122.247_monitor_10000" operation_key="ip-192.168.122.247_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="60" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303737" exec-time="35" queue-time="0" op-digest="22cd67ba6dcb54e4f4e6ea3450a52b77"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.248" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.248_last_0" operation_key="ip-192.168.122.248_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="65" rc-code="0" op-status="0" interval="0" last-run="1496303750" last-rc-change="1496303750" exec-time="52" queue-time="0" op-digest="7565857faaf5a927e16322d5da3e4e38"/>
++ <lrm_rsc_op id="ip-192.168.122.248_monitor_10000" operation_key="ip-192.168.122.248_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="66" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303750" exec-time="30" queue-time="0" op-digest="2182cf0cd059930b67d43545b4950c6f"/>
++ </lrm_resource>
++ <lrm_resource id="openstack-cinder-volume-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_0" operation_key="openstack-cinder-volume-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="161" rc-code="0" op-status="0" interval="0" last-run="1496305247" last-rc-change="1496305247" exec-time="838" queue-time="0" op-digest="3aa35269c96594499406635b536dcf19"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_monitor_60000" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="162" rc-code="0" op-status="0" interval="60000" last-rc-change="1496305248" exec-time="63" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_failure_0" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:7;14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="158" rc-code="7" op-status="0" interval="60000" last-rc-change="1496305247" exec-time="0" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="1">
++ <instance_attributes id="status-1">
++ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
++ <nvpair id="status-1-fail-count-openstack-cinder-volume-docker-0" name="fail-count-openstack-cinder-volume-docker-0" value="21"/>
++ <nvpair id="status-1-last-failure-openstack-cinder-volume-docker-0" name="last-failure-openstack-cinder-volume-docker-0" value="1496305247"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state remote_node="true" id="redis-bundle-0" uname="redis-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="redis-bundle-0">
++ <instance_attributes id="status-redis-bundle-0">
++ <nvpair id="status-redis-bundle-0-master-redis" name="master-redis" value="3000"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="redis-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="58" rc-code="0" op-status="0" interval="0" last-run="1496303672" last-rc-change="1496303672" exec-time="1583" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ </status>
++</cib>
+diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot
+new file mode 100644
+index 0000000..06c3620
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start.dot
+@@ -0,0 +1,65 @@
++digraph "g" {
++"galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold]
++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
++"galera-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold]
++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold]
++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold]
++"haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle_running_0" -> "galera-bundle_start_0" [ style = bold]
++"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
++"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold]
++"rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle_running_0" -> "galera-bundle_start_0" [ style = bold]
++"rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold]
++"rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold]
++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold]
++"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold]
++"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_promoted_0" -> "galera-bundle_start_0" [ style = bold]
++"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold]
++"redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold]
++"redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++}
+diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp
+new file mode 100644
+index 0000000..b48bb03
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start.exp
+@@ -0,0 +1,375 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0">
++ <action_set>
++ <rsc_op id="24" operation="monitor" operation_key="rabbitmq:0_monitor_10000" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_timeout="40000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="23" operation="start" operation_key="rabbitmq:0_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="23" operation="start" operation_key="rabbitmq:0_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_timeout="200000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="25" operation="start" operation_key="rabbitmq-bundle-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="2">
++ <action_set>
++ <rsc_op id="39" operation="monitor" operation_key="galera:0_monitor_30000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_role="Slave" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="37" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="38" operation="monitor" operation_key="galera:0_monitor_20000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="37" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="4">
++ <action_set>
++ <rsc_op id="37" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="40" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="30" operation="monitor" operation_key="galera-bundle-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="6">
++ <action_set>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="33" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="8">
++ <action_set>
++ <rsc_op id="32" operation="monitor" operation_key="galera-bundle-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="10">
++ <action_set>
++ <rsc_op id="63" operation="monitor" operation_key="redis_monitor_20000" internal_operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="62" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <rsc_op id="62" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="120000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="68" operation="promote" operation_key="redis-bundle-master_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="12">
++ <action_set>
++ <rsc_op id="89" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="88" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13">
++ <action_set>
++ <rsc_op id="88" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="14" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="90" operation="start" operation_key="haproxy-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14">
++ <action_set>
++ <rsc_op id="14" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="15" priority="1000000">
++ <action_set>
++ <pseudo_event id="91" operation="running" operation_key="haproxy-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="88" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16">
++ <action_set>
++ <pseudo_event id="90" operation="start" operation_key="haproxy-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="17" priority="1000000">
++ <action_set>
++ <pseudo_event id="73" operation="promoted" operation_key="redis-bundle_promoted_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="69" operation="promoted" operation_key="redis-bundle-master_promoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="18">
++ <action_set>
++ <pseudo_event id="72" operation="promote" operation_key="redis-bundle_promote_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="19" priority="1000000">
++ <action_set>
++ <pseudo_event id="69" operation="promoted" operation_key="redis-bundle-master_promoted_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="62" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20">
++ <action_set>
++ <pseudo_event id="68" operation="promote" operation_key="redis-bundle-master_promote_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="72" operation="promote" operation_key="redis-bundle_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="21" priority="1000000">
++ <action_set>
++ <pseudo_event id="41" operation="running" operation_key="galera-bundle-master_running_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="37" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="40" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="22">
++ <action_set>
++ <pseudo_event id="40" operation="start" operation_key="galera-bundle-master_start_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="33" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="23" priority="1000000">
++ <action_set>
++ <pseudo_event id="34" operation="running" operation_key="galera-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="41" operation="running" operation_key="galera-bundle-master_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="24">
++ <action_set>
++ <pseudo_event id="33" operation="start" operation_key="galera-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="20" operation="running" operation_key="rabbitmq-bundle_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="73" operation="promoted" operation_key="redis-bundle_promoted_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="91" operation="running" operation_key="haproxy-bundle_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="25" priority="1000000">
++ <action_set>
++ <pseudo_event id="26" operation="running" operation_key="rabbitmq-bundle-clone_running_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="23" operation="start" operation_key="rabbitmq:0_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="25" operation="start" operation_key="rabbitmq-bundle-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="26">
++ <action_set>
++ <pseudo_event id="25" operation="start" operation_key="rabbitmq-bundle-clone_start_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="19" operation="start" operation_key="rabbitmq-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="27" priority="1000000">
++ <action_set>
++ <pseudo_event id="20" operation="running" operation_key="rabbitmq-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="26" operation="running" operation_key="rabbitmq-bundle-clone_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="28">
++ <action_set>
++ <pseudo_event id="19" operation="start" operation_key="rabbitmq-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/bundle-order-partial-start.scores b/pengine/test10/bundle-order-partial-start.scores
+new file mode 100644
+index 0000000..ea702d0
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start.scores
+@@ -0,0 +1,197 @@
++Allocation scores:
++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:0 allocation score on undercloud: -INFINITY
++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY
++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: redis:0 allocation score on undercloud: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on undercloud: 0
++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on undercloud: 0
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on undercloud: 0
++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on undercloud: 0
++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera:0 allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on undercloud: 0
++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle allocation score on undercloud: 0
++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0
++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 0
++container_color: rabbitmq:0 allocation score on undercloud: 0
++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on undercloud: 0
++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on undercloud: 0
++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:0 allocation score on undercloud: 0
++galera:0 promotion score on galera-bundle-0: -1
++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on undercloud: 10000
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on undercloud: 0
++native_color: galera:0 allocation score on galera-bundle-0: INFINITY
++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:0 allocation score on undercloud: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on undercloud: INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on undercloud: INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: redis:0 allocation score on undercloud: -INFINITY
++redis:0 promotion score on redis-bundle-0: 3000
+diff --git a/pengine/test10/bundle-order-partial-start.summary b/pengine/test10/bundle-order-partial-start.summary
+new file mode 100644
+index 0000000..e56e55f
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start.summary
+@@ -0,0 +1,82 @@
++
++Current cluster status:
++Online: [ undercloud ]
++Containers: [ rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped undercloud
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Stopped
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Slave undercloud
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud
++
++Transition Summary:
++ * Start rabbitmq:0 (rabbitmq-bundle-0)
++ * Start galera-bundle-docker-0 (undercloud)
++ * Start galera-bundle-0 (undercloud)
++ * Start galera:0 (galera-bundle-0)
++ * Promote redis:0 (Slave -> Master redis-bundle-0)
++ * Start haproxy-bundle-docker-0 (undercloud)
++
++Executing cluster transition:
++ * Resource action: galera-bundle-docker-0 monitor on undercloud
++ * Resource action: haproxy-bundle-docker-0 monitor on undercloud
++ * Pseudo action: haproxy-bundle_start_0
++ * Pseudo action: redis-bundle_promote_0
++ * Pseudo action: redis-bundle-master_promote_0
++ * Pseudo action: rabbitmq-bundle_start_0
++ * Resource action: redis promote on redis-bundle-0
++ * Resource action: haproxy-bundle-docker-0 start on undercloud
++ * Pseudo action: haproxy-bundle_running_0
++ * Pseudo action: redis-bundle-master_promoted_0
++ * Pseudo action: rabbitmq-bundle-clone_start_0
++ * Resource action: rabbitmq:0 start on rabbitmq-bundle-0
++ * Resource action: redis monitor=20000 on redis-bundle-0
++ * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud
++ * Pseudo action: redis-bundle_promoted_0
++ * Pseudo action: rabbitmq-bundle-clone_running_0
++ * Pseudo action: rabbitmq-bundle_running_0
++ * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0
++ * Pseudo action: galera-bundle_start_0
++ * Resource action: galera-bundle-docker-0 start on undercloud
++ * Resource action: galera-bundle-0 start on undercloud
++ * Pseudo action: galera-bundle-master_start_0
++ * Resource action: galera:0 start on galera-bundle-0
++ * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud
++ * Resource action: galera-bundle-0 monitor=60000 on undercloud
++ * Pseudo action: galera-bundle-master_running_0
++ * Pseudo action: galera-bundle_running_0
++ * Resource action: galera:0 monitor=30000 on galera-bundle-0
++ * Resource action: galera:0 monitor=20000 on galera-bundle-0
++
++Revised cluster status:
++Online: [ undercloud ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Slave undercloud
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud
++
+diff --git a/pengine/test10/bundle-order-partial-start.xml b/pengine/test10/bundle-order-partial-start.xml
+new file mode 100644
+index 0000000..907dbfb
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-start.xml
+@@ -0,0 +1,379 @@
++<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="undercloud">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
++ <instance_attributes id="nodes-rabbitmq-bundle-0">
++ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@undercloud"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <bundle id="rabbitmq-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3121"/>
++ <storage>
++ <storage-mapping id="rabbitmq-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/rabbitmq.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="rabbitmq-cfg-data" options="ro" source-dir="/var/lib/config-data/rabbitmq/etc/rabbitmq" target-dir="/etc/rabbitmq"/>
++ <storage-mapping id="rabbitmq-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="rabbitmq-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="rabbitmq-lib" options="rw" source-dir="/var/lib/rabbitmq" target-dir="/var/lib/rabbitmq"/>
++ <storage-mapping id="rabbitmq-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="rabbitmq-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="rabbitmq-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster">
++ <instance_attributes id="rabbitmq-instance_attributes">
++ <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </instance_attributes>
++ <meta_attributes id="rabbitmq-meta_attributes">
++ <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="rabbitmq-monitor-interval-10" interval="10" name="monitor" timeout="40"/>
++ <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="galera-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
++ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
++ <instance_attributes id="galera-instance_attributes">
++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:undercloud"/>
++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://undercloud"/>
++ </instance_attributes>
++ <meta_attributes id="galera-meta_attributes">
++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
++ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
++ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="redis-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3124"/>
++ <storage>
++ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
++ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
++ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
++ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
++ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
++ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
++ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
++ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
++ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
++ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <primitive class="ocf" id="ip-192.168.122.254" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.254-instance_attributes">
++ <nvpair id="ip-192.168.122.254-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.254-instance_attributes-ip" name="ip" value="192.168.122.254"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.254-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.254-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.254-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.250" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.250-instance_attributes">
++ <nvpair id="ip-192.168.122.250-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.250-instance_attributes-ip" name="ip" value="192.168.122.250"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.250-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.250-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.250-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.249" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.249-instance_attributes">
++ <nvpair id="ip-192.168.122.249-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.249-instance_attributes-ip" name="ip" value="192.168.122.249"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.249-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.249-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.249-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.253" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.253-instance_attributes">
++ <nvpair id="ip-192.168.122.253-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.253-instance_attributes-ip" name="ip" value="192.168.122.253"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.253-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.253-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.253-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.247" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.247-instance_attributes">
++ <nvpair id="ip-192.168.122.247-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.247-instance_attributes-ip" name="ip" value="192.168.122.247"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.247-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.247-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.247-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.248" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.248-instance_attributes">
++ <nvpair id="ip-192.168.122.248-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.248-instance_attributes-ip" name="ip" value="192.168.122.248"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.248-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.248-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.248-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ </bundle>
++ <bundle id="openstack-cinder-volume">
++ <docker run-command="/bin/bash /usr/local/bin/kolla_start" network="host" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" options="--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1"/>
++ <storage>
++ <storage-mapping target-dir="/var/lib/kolla/config_files/config.json" options="ro" id="cinder-volume-cfg-files" source-dir="/var/lib/kolla/config_files/cinder_volume.json"/>
++ <storage-mapping target-dir="/etc/cinder" options="ro" id="cinder-volume-cfg-data" source-dir="/var/lib/config-data/cinder/etc/cinder"/>
++ <storage-mapping target-dir="/etc/hosts" options="ro" id="cinder-volume-hosts" source-dir="/etc/hosts"/>
++ <storage-mapping target-dir="/etc/localtime" options="ro" id="cinder-volume-localtime" source-dir="/etc/localtime"/>
++ <storage-mapping target-dir="/dev" options="rw" id="cinder-volume-dev" source-dir="/dev"/>
++ <storage-mapping target-dir="/run" options="rw" id="cinder-volume-run" source-dir="/run"/>
++ <storage-mapping target-dir="/sys" options="rw" id="cinder-volume-sys" source-dir="/sys"/>
++ <storage-mapping target-dir="/lib/modules" options="ro" id="cinder-lib-modules" source-dir="/lib/modules"/>
++ <storage-mapping target-dir="/etc/iscsi" options="rw" id="cinder-volume-iscsi" source-dir="/etc/iscsi"/>
++ <storage-mapping target-dir="/var/lib/cinder" options="rw" id="cinder-volume-var-lib-cinder" source-dir="/var/lib/cinder"/>
++ <storage-mapping target-dir="/var/log/cinder" options="rw" id="cinder-volume-var-log" source-dir="/var/log/containers/cinder"/>
++ </storage>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-rabbitmq-bundle" resource-discovery="exclusive" rsc="rabbitmq-bundle">
++ <rule id="location-rabbitmq-bundle-rule" score="0">
++ <expression attribute="rabbitmq-role" id="location-rabbitmq-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
++ <rule id="location-galera-bundle-rule" score="0">
++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
++ <rule id="location-redis-bundle-rule" score="0">
++ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.254" resource-discovery="exclusive" rsc="ip-192.168.122.254">
++ <rule id="location-ip-192.168.122.254-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.254-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.250" resource-discovery="exclusive" rsc="ip-192.168.122.250">
++ <rule id="location-ip-192.168.122.250-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.250-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.249" resource-discovery="exclusive" rsc="ip-192.168.122.249">
++ <rule id="location-ip-192.168.122.249-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.249-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.253" resource-discovery="exclusive" rsc="ip-192.168.122.253">
++ <rule id="location-ip-192.168.122.253-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.253-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.247" resource-discovery="exclusive" rsc="ip-192.168.122.247">
++ <rule id="location-ip-192.168.122.247-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.247-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.248" resource-discovery="exclusive" rsc="ip-192.168.122.248">
++ <rule id="location-ip-192.168.122.248-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.248-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_order first="ip-192.168.122.254" first-action="start" id="order-ip-192.168.122.254-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.254-haproxy-bundle-INFINITY" rsc="ip-192.168.122.254" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.250" first-action="start" id="order-ip-192.168.122.250-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.250-haproxy-bundle-INFINITY" rsc="ip-192.168.122.250" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.249" first-action="start" id="order-ip-192.168.122.249-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.249-haproxy-bundle-INFINITY" rsc="ip-192.168.122.249" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.253" first-action="start" id="order-ip-192.168.122.253-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.253-haproxy-bundle-INFINITY" rsc="ip-192.168.122.253" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.247" first-action="start" id="order-ip-192.168.122.247-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.247-haproxy-bundle-INFINITY" rsc="ip-192.168.122.247" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.248" first-action="start" id="order-ip-192.168.122.248-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.248-haproxy-bundle-INFINITY" rsc="ip-192.168.122.248" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_colocation id="colocation-rabbit-with-haproxy-bundle-INFINITY" rsc="rabbitmq-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="haproxy-bundle" first-action="start" id="order-1" kind="Mandatory" then="galera-bundle" then-action="start"/>
++ <rsc_order first="rabbitmq-bundle" first-action="start" id="order-2" kind="Mandatory" then="galera-bundle" then-action="start"/>
++ <rsc_order first="redis-bundle" first-action="promote" id="order-3" kind="Mandatory" then="galera-bundle" then-action="start"/>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="undercloud" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="1">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="10" rc-code="0" op-status="0" interval="0" last-run="1496303575" last-rc-change="1496303575" exec-time="850" queue-time="0" op-digest="50baee2f0a77a08b0616f0cf07ab724b"/>
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_monitor_60000" operation_key="rabbitmq-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303576" exec-time="76" queue-time="0" op-digest="25cb53ea3977a912bf4c0f70fe9e3c21"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0">
++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1496303576" last-rc-change="1496303576" exec-time="0" queue-time="0" op-digest="ee16760fa8c27e6585c1e35c244d7652" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="rabbitmq-bundle-0_monitor_60000" operation_key="rabbitmq-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303578" exec-time="0" queue-time="0" op-digest="92f557a57dfbb50be88282a4674ea681"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="824" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
++ <lrm_rsc_op id="redis-bundle-docker-0_monitor_60000" operation_key="redis-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="29" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303664" exec-time="74" queue-time="0" op-digest="81e9d490c493f8d22fab2822ec0bed8a"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
++ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="5" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="0" queue-time="0" op-digest="9a6281aab223af75359e5b5a07161162" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="redis-bundle-0_monitor_60000" operation_key="redis-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="6" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303667" exec-time="0" queue-time="0" op-digest="04ffb214f519838e579c97e8aa002054"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.254" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.254_last_0" operation_key="ip-192.168.122.254_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1496303683" last-rc-change="1496303683" exec-time="63" queue-time="0" op-digest="d6875d68a09b5550030b8b6b7cdc9294"/>
++ <lrm_rsc_op id="ip-192.168.122.254_monitor_10000" operation_key="ip-192.168.122.254_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="36" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303683" exec-time="32" queue-time="0" op-digest="5c1c6d0e79751e4002b12de513351a48"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.250" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.250_last_0" operation_key="ip-192.168.122.250_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="41" rc-code="0" op-status="0" interval="0" last-run="1496303696" last-rc-change="1496303696" exec-time="53" queue-time="0" op-digest="b33b44b3e9e535336108cb43f7474ef6"/>
++ <lrm_rsc_op id="ip-192.168.122.250_monitor_10000" operation_key="ip-192.168.122.250_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="42" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303696" exec-time="32" queue-time="0" op-digest="a4aec96753744d49ca9a798d3b0f12aa"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.249" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.249_last_0" operation_key="ip-192.168.122.249_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="47" rc-code="0" op-status="0" interval="0" last-run="1496303710" last-rc-change="1496303710" exec-time="52" queue-time="0" op-digest="22e58cb8beefb7810d8bf03d0d3a906d"/>
++ <lrm_rsc_op id="ip-192.168.122.249_monitor_10000" operation_key="ip-192.168.122.249_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="48" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303710" exec-time="30" queue-time="1" op-digest="860855adc72293b04b1becaeea73ced1"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.253" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.253_last_0" operation_key="ip-192.168.122.253_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1496303723" last-rc-change="1496303723" exec-time="69" queue-time="0" op-digest="f36f4ccbd5f16e4e335c65f0802ddc20"/>
++ <lrm_rsc_op id="ip-192.168.122.253_monitor_10000" operation_key="ip-192.168.122.253_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="54" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303724" exec-time="39" queue-time="0" op-digest="0ee9a624c731239cdbef5c9118cc2bbe"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.247" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.247_last_0" operation_key="ip-192.168.122.247_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="59" rc-code="0" op-status="0" interval="0" last-run="1496303737" last-rc-change="1496303737" exec-time="57" queue-time="0" op-digest="37013238bd139c2b589a9508ff4b5c07"/>
++ <lrm_rsc_op id="ip-192.168.122.247_monitor_10000" operation_key="ip-192.168.122.247_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="60" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303737" exec-time="35" queue-time="0" op-digest="22cd67ba6dcb54e4f4e6ea3450a52b77"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.248" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.248_last_0" operation_key="ip-192.168.122.248_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="65" rc-code="0" op-status="0" interval="0" last-run="1496303750" last-rc-change="1496303750" exec-time="52" queue-time="0" op-digest="7565857faaf5a927e16322d5da3e4e38"/>
++ <lrm_rsc_op id="ip-192.168.122.248_monitor_10000" operation_key="ip-192.168.122.248_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="66" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303750" exec-time="30" queue-time="0" op-digest="2182cf0cd059930b67d43545b4950c6f"/>
++ </lrm_resource>
++ <lrm_resource id="openstack-cinder-volume-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_0" operation_key="openstack-cinder-volume-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="161" rc-code="0" op-status="0" interval="0" last-run="1496305247" last-rc-change="1496305247" exec-time="838" queue-time="0" op-digest="3aa35269c96594499406635b536dcf19"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_monitor_60000" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="162" rc-code="0" op-status="0" interval="60000" last-rc-change="1496305248" exec-time="63" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_failure_0" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:7;14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="158" rc-code="7" op-status="0" interval="60000" last-rc-change="1496305247" exec-time="0" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="1">
++ <instance_attributes id="status-1">
++ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
++ <nvpair id="status-1-fail-count-openstack-cinder-volume-docker-0" name="fail-count-openstack-cinder-volume-docker-0" value="21"/>
++ <nvpair id="status-1-last-failure-openstack-cinder-volume-docker-0" name="last-failure-openstack-cinder-volume-docker-0" value="1496305247"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state remote_node="true" id="redis-bundle-0" uname="redis-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="redis-bundle-0">
++ <instance_attributes id="status-redis-bundle-0">
++ <nvpair id="status-redis-bundle-0-master-redis" name="master-redis" value="3000"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="redis-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="58" rc-code="0" op-status="0" interval="0" last-run="1496303672" last-rc-change="1496303672" exec-time="1583" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ </status>
++</cib>
+diff --git a/pengine/test10/bundle-order-partial-stop.dot b/pengine/test10/bundle-order-partial-stop.dot
+new file mode 100644
+index 0000000..235b634
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-stop.dot
+@@ -0,0 +1,194 @@
++digraph "g" {
++"Cancel galera_monitor_10000 galera-bundle-0" -> "galera_demote_0 galera-bundle-0" [ style = bold]
++"Cancel galera_monitor_10000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"Cancel redis_monitor_20000 redis-bundle-0" -> "redis_demote_0 redis-bundle-0" [ style = bold]
++"Cancel redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"all_stopped" [ style=bold color="green" fontcolor="orange"]
++"do_shutdown undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"galera-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"galera-bundle-0_stop_0 undercloud" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold]
++"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold]
++"galera-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_demoted_0" -> "galera-bundle-master_start_0" [ style = dashed]
++"galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold]
++"galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold]
++"galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed]
++"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed]
++"galera-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold]
++"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
++"galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = dashed]
++"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold]
++"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold]
++"galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold]
++"galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold]
++"galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold]
++"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
++"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_stopped_0" -> "redis-bundle_stop_0" [ style = bold]
++"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold]
++"galera_demote_0 galera-bundle-0" -> "galera-bundle-master_demoted_0" [ style = bold]
++"galera_demote_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed]
++"galera_demote_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed]
++"galera_demote_0 galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
++"galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed]
++"galera_start_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed]
++"galera_start_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed]
++"galera_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera_stop_0 galera-bundle-0" -> "all_stopped" [ style = bold]
++"galera_stop_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold]
++"galera_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold]
++"galera_stop_0 galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = dashed]
++"galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"haproxy-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"haproxy-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle_stopped_0" [ style = bold]
++"haproxy-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle_stop_0" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.247_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.248_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.249_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.250_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.253_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.254_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"ip-192.168.122.247_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.247_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.247_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.247_stop_0 undercloud" -> "ip-192.168.122.247_start_0 <none>" [ style = dashed]
++"ip-192.168.122.247_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.248_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.248_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.248_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.248_stop_0 undercloud" -> "ip-192.168.122.248_start_0 <none>" [ style = dashed]
++"ip-192.168.122.248_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.249_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.249_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.249_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.249_stop_0 undercloud" -> "ip-192.168.122.249_start_0 <none>" [ style = dashed]
++"ip-192.168.122.249_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.250_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.250_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.250_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.250_stop_0 undercloud" -> "ip-192.168.122.250_start_0 <none>" [ style = dashed]
++"ip-192.168.122.250_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.253_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.253_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.253_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.253_stop_0 undercloud" -> "ip-192.168.122.253_start_0 <none>" [ style = dashed]
++"ip-192.168.122.253_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.254_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.254_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.254_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.254_stop_0 undercloud" -> "ip-192.168.122.254_start_0 <none>" [ style = dashed]
++"ip-192.168.122.254_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "openstack-cinder-volume_stopped_0" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume_stop_0" -> "openstack-cinder-volume-docker-0_stop_0 undercloud" [ style = bold]
++"openstack-cinder-volume_stop_0" [ style=bold color="green" fontcolor="orange"]
++"openstack-cinder-volume_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq-bundle-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold]
++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed]
++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold]
++"rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold]
++"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed]
++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "all_stopped" [ style = bold]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-0_stop_0 undercloud" [ style = bold]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"redis-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"redis-bundle-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold]
++"redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_demoted_0" -> "redis-bundle-master_start_0" [ style = dashed]
++"redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold]
++"redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold]
++"redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed]
++"redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed]
++"redis-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"]
++"redis-bundle-master_stop_0" -> "redis-bundle-master_stopped_0" [ style = bold]
++"redis-bundle-master_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_stopped_0" -> "redis-bundle-master_start_0" [ style = dashed]
++"redis-bundle-master_stopped_0" -> "redis-bundle_stopped_0" [ style = bold]
++"redis-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold]
++"redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold]
++"redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold]
++"redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold]
++"redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed]
++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold]
++"redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
++"redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_stopped_0" -> "haproxy-bundle_stop_0" [ style = bold]
++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold]
++"redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"redis_demote_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold]
++"redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold]
++"redis_demote_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed]
++"redis_demote_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed]
++"redis_demote_0 redis-bundle-0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
++"redis_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis_monitor_45000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"redis_monitor_60000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"redis_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = dashed]
++"redis_start_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed]
++"redis_start_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed]
++"redis_start_0 redis-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"redis_stop_0 redis-bundle-0" -> "all_stopped" [ style = bold]
++"redis_stop_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold]
++"redis_stop_0 redis-bundle-0" -> "redis-bundle-master_stopped_0" [ style = bold]
++"redis_stop_0 redis-bundle-0" -> "redis_start_0 redis-bundle-0" [ style = dashed]
++"redis_stop_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++}
+diff --git a/pengine/test10/bundle-order-partial-stop.exp b/pengine/test10/bundle-order-partial-stop.exp
+new file mode 100644
+index 0000000..d085433
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-stop.exp
+@@ -0,0 +1,734 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0">
++ <action_set>
++ <rsc_op id="25" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_timeout="200000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="19" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/rabbitmq-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/rabbitmq/etc/rabbitmq:/etc/rabbitmq:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/rabbitmq:/var/lib/rabbitmq:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/rabbitmq-bundle-0:/var/log -p 3121:3121 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="2">
++ <action_set>
++ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="rabbitmq-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="#uname" port="3121"/>
++ <downed>
++ <node id="rabbitmq-bundle-0"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="25" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="39" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="38" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="44" operation="stop" operation_key="galera-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="4">
++ <action_set>
++ <rsc_op id="38" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="16" operation="cancel" operation_key="galera_monitor_10000" internal_operation_key="galera:0_monitor_10000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="48" operation="demote" operation_key="galera-bundle-master_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="16" operation="cancel" operation_key="galera_monitor_10000" internal_operation_key="galera:0_monitor_10000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_operation="monitor" CRM_meta_role="Master" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="6">
++ <action_set>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="#uname" port="3123"/>
++ <downed>
++ <node id="galera-bundle-0"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="38" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="39" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="8">
++ <action_set>
++ <rsc_op id="62" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="200000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="58" operation="stop" operation_key="redis-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="61" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="67" operation="stop" operation_key="redis-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <rsc_op id="61" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="120000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="17" operation="cancel" operation_key="redis_monitor_20000" internal_operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="71" operation="demote" operation_key="redis-bundle-master_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="10">
++ <action_set>
++ <rsc_op id="17" operation="cancel" operation_key="redis_monitor_20000" internal_operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_operation="monitor" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="12">
++ <action_set>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="#uname" port="3124"/>
++ <downed>
++ <node id="redis-bundle-0"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="61" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="62" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13">
++ <action_set>
++ <rsc_op id="77" operation="stop" operation_key="ip-192.168.122.254_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.254" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.254"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14">
++ <action_set>
++ <rsc_op id="78" operation="stop" operation_key="ip-192.168.122.250_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.250" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.250"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15">
++ <action_set>
++ <rsc_op id="79" operation="stop" operation_key="ip-192.168.122.249_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.249" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.249"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16">
++ <action_set>
++ <rsc_op id="80" operation="stop" operation_key="ip-192.168.122.253_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.253" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.253"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="17">
++ <action_set>
++ <rsc_op id="81" operation="stop" operation_key="ip-192.168.122.247_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.247" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.247"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="18">
++ <action_set>
++ <rsc_op id="82" operation="stop" operation_key="ip-192.168.122.248_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.248" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.248"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="19">
++ <action_set>
++ <rsc_op id="83" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="86" operation="stop" operation_key="haproxy-bundle_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20">
++ <action_set>
++ <rsc_op id="88" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="openstack-cinder-volume-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/cinder/etc/cinder:/etc/cinder:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /dev:/dev:rw -v /run:/run:rw -v /sys:/sys:rw -v /lib/modules:/lib/modules:ro -v /etc/iscsi:/etc/iscsi:rw -v /var/lib/cinder:/var/lib/cinder:rw -v /var/log/containers/cinder:/var/log/cinder:rw --ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="91" operation="stop" operation_key="openstack-cinder-volume_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="21">
++ <action_set>
++ <crm_event id="94" operation="do_shutdown" operation_key="do_shutdown-undercloud" on_node="undercloud" on_node_uuid="1">
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" />
++ <downed>
++ <node id="1"/>
++ </downed>
++ </crm_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="19" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="77" operation="stop" operation_key="ip-192.168.122.254_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="78" operation="stop" operation_key="ip-192.168.122.250_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="79" operation="stop" operation_key="ip-192.168.122.249_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="80" operation="stop" operation_key="ip-192.168.122.253_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="81" operation="stop" operation_key="ip-192.168.122.247_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="82" operation="stop" operation_key="ip-192.168.122.248_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="83" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="88" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="22" priority="1000000">
++ <action_set>
++ <pseudo_event id="92" operation="stopped" operation_key="openstack-cinder-volume_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="88" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="23">
++ <action_set>
++ <pseudo_event id="91" operation="stop" operation_key="openstack-cinder-volume_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="24" priority="1000000">
++ <action_set>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="83" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="25">
++ <action_set>
++ <pseudo_event id="86" operation="stop" operation_key="haproxy-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="59" operation="stopped" operation_key="redis-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="26" priority="1000000">
++ <action_set>
++ <pseudo_event id="76" operation="demoted" operation_key="redis-bundle_demoted_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="72" operation="demoted" operation_key="redis-bundle-master_demoted_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="75" operation="demote" operation_key="redis-bundle_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="27">
++ <action_set>
++ <pseudo_event id="75" operation="demote" operation_key="redis-bundle_demote_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="28" priority="1000000">
++ <action_set>
++ <pseudo_event id="72" operation="demoted" operation_key="redis-bundle-master_demoted_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="61" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="71" operation="demote" operation_key="redis-bundle-master_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="29">
++ <action_set>
++ <pseudo_event id="71" operation="demote" operation_key="redis-bundle-master_demote_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="75" operation="demote" operation_key="redis-bundle_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="30" priority="1000000">
++ <action_set>
++ <pseudo_event id="68" operation="stopped" operation_key="redis-bundle-master_stopped_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="62" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="67" operation="stop" operation_key="redis-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="31">
++ <action_set>
++ <pseudo_event id="67" operation="stop" operation_key="redis-bundle-master_stop_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="58" operation="stop" operation_key="redis-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="72" operation="demoted" operation_key="redis-bundle-master_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="32" priority="1000000">
++ <action_set>
++ <pseudo_event id="59" operation="stopped" operation_key="redis-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="68" operation="stopped" operation_key="redis-bundle-master_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="33">
++ <action_set>
++ <pseudo_event id="58" operation="stop" operation_key="redis-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="36" operation="stopped" operation_key="galera-bundle_stopped_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="76" operation="demoted" operation_key="redis-bundle_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="34">
++ <action_set>
++ <pseudo_event id="56" operation="start" operation_key="redis-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="59" operation="stopped" operation_key="redis-bundle_stopped_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="76" operation="demoted" operation_key="redis-bundle_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="35" priority="1000000">
++ <action_set>
++ <pseudo_event id="53" operation="demoted" operation_key="galera-bundle_demoted_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="49" operation="demoted" operation_key="galera-bundle-master_demoted_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="52" operation="demote" operation_key="galera-bundle_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="36">
++ <action_set>
++ <pseudo_event id="52" operation="demote" operation_key="galera-bundle_demote_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="37" priority="1000000">
++ <action_set>
++ <pseudo_event id="49" operation="demoted" operation_key="galera-bundle-master_demoted_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="38" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="48" operation="demote" operation_key="galera-bundle-master_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="38">
++ <action_set>
++ <pseudo_event id="48" operation="demote" operation_key="galera-bundle-master_demote_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="52" operation="demote" operation_key="galera-bundle_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="39" priority="1000000">
++ <action_set>
++ <pseudo_event id="45" operation="stopped" operation_key="galera-bundle-master_stopped_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="39" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="44" operation="stop" operation_key="galera-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="40">
++ <action_set>
++ <pseudo_event id="44" operation="stop" operation_key="galera-bundle-master_stop_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="49" operation="demoted" operation_key="galera-bundle-master_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="41" priority="1000000">
++ <action_set>
++ <pseudo_event id="36" operation="stopped" operation_key="galera-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="45" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="42">
++ <action_set>
++ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="53" operation="demoted" operation_key="galera-bundle_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="43" priority="1000000">
++ <action_set>
++ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="25" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="44">
++ <action_set>
++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="45" priority="1000000">
++ <action_set>
++ <pseudo_event id="24" operation="stopped" operation_key="rabbitmq-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="46">
++ <action_set>
++ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="47">
++ <action_set>
++ <pseudo_event id="18" operation="all_stopped" operation_key="all_stopped">
++ <attributes />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="19" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="25" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="39" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="62" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="77" operation="stop" operation_key="ip-192.168.122.254_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="78" operation="stop" operation_key="ip-192.168.122.250_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="79" operation="stop" operation_key="ip-192.168.122.249_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="80" operation="stop" operation_key="ip-192.168.122.253_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="81" operation="stop" operation_key="ip-192.168.122.247_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="82" operation="stop" operation_key="ip-192.168.122.248_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="83" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="88" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/bundle-order-partial-stop.scores b/pengine/test10/bundle-order-partial-stop.scores
+new file mode 100644
+index 0000000..a662f42
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-stop.scores
+@@ -0,0 +1,199 @@
++Allocation scores:
++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:0 allocation score on undercloud: -INFINITY
++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY
++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: redis:0 allocation score on undercloud: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on undercloud: 0
++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on undercloud: INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on undercloud: 0
++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera:0 allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on undercloud: 0
++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle allocation score on undercloud: 0
++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0
++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++container_color: rabbitmq:0 allocation score on undercloud: 0
++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on undercloud: 0
++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on undercloud: 0
++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:0 allocation score on undercloud: 0
++galera:0 promotion score on galera-bundle-0: 100
++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on undercloud: INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on undercloud: -INFINITY
++native_color: galera:0 allocation score on galera-bundle-0: INFINITY
++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:0 allocation score on undercloud: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on undercloud: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: -INFINITY
++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: -INFINITY
++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on undercloud: INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on undercloud: -INFINITY
++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: redis:0 allocation score on undercloud: -INFINITY
++redis:0 promotion score on redis-bundle-0: 1
+diff --git a/pengine/test10/bundle-order-partial-stop.summary b/pengine/test10/bundle-order-partial-stop.summary
+new file mode 100644
+index 0000000..bd6f937
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-stop.summary
+@@ -0,0 +1,114 @@
++
++Current cluster status:
++Online: [ undercloud ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Master undercloud
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud
++
++Transition Summary:
++ * Shutdown undercloud
++ * Stop rabbitmq-bundle-docker-0 (undercloud)
++ * Stop rabbitmq-bundle-0 (undercloud)
++ * Stop rabbitmq:0 (Started rabbitmq-bundle-0)
++ * Stop galera-bundle-docker-0 (undercloud)
++ * Stop galera-bundle-0 (undercloud)
++ * Demote galera:0 (Master -> Slave galera-bundle-0)
++ * Restart galera:0 (Slave galera-bundle-0)
++ * Stop redis-bundle-docker-0 (undercloud)
++ * Stop redis-bundle-0 (undercloud)
++ * Demote redis:0 (Master -> Slave redis-bundle-0)
++ * Restart redis:0 (Slave redis-bundle-0)
++ * Stop ip-192.168.122.254 (undercloud)
++ * Stop ip-192.168.122.250 (undercloud)
++ * Stop ip-192.168.122.249 (undercloud)
++ * Stop ip-192.168.122.253 (undercloud)
++ * Stop ip-192.168.122.247 (undercloud)
++ * Stop ip-192.168.122.248 (undercloud)
++ * Stop haproxy-bundle-docker-0 (undercloud)
++ * Stop openstack-cinder-volume-docker-0 (undercloud)
++
++Executing cluster transition:
++ * Resource action: galera cancel=10000 on galera-bundle-0
++ * Resource action: redis cancel=20000 on redis-bundle-0
++ * Pseudo action: openstack-cinder-volume_stop_0
++ * Pseudo action: redis-bundle_demote_0
++ * Pseudo action: redis-bundle-master_demote_0
++ * Pseudo action: galera-bundle_demote_0
++ * Pseudo action: galera-bundle-master_demote_0
++ * Pseudo action: rabbitmq-bundle_stop_0
++ * Resource action: galera demote on galera-bundle-0
++ * Resource action: redis demote on redis-bundle-0
++ * Resource action: openstack-cinder-volume-docker-0 stop on undercloud
++ * Pseudo action: openstack-cinder-volume_stopped_0
++ * Pseudo action: redis-bundle-master_demoted_0
++ * Pseudo action: galera-bundle-master_demoted_0
++ * Pseudo action: rabbitmq-bundle-clone_stop_0
++ * Resource action: rabbitmq stop on rabbitmq-bundle-0
++ * Resource action: rabbitmq-bundle-0 stop on undercloud
++ * Pseudo action: redis-bundle_demoted_0
++ * Pseudo action: galera-bundle_demoted_0
++ * Pseudo action: galera-bundle_stop_0
++ * Pseudo action: rabbitmq-bundle-clone_stopped_0
++ * Pseudo action: rabbitmq-bundle_stopped_0
++ * Resource action: rabbitmq-bundle-docker-0 stop on undercloud
++ * Pseudo action: galera-bundle-master_stop_0
++ * Resource action: galera stop on galera-bundle-0
++ * Resource action: galera-bundle-0 stop on undercloud
++ * Pseudo action: galera-bundle-master_stopped_0
++ * Pseudo action: galera-bundle_stopped_0
++ * Resource action: galera-bundle-docker-0 stop on undercloud
++ * Pseudo action: redis-bundle_stop_0
++ * Pseudo action: redis-bundle-master_stop_0
++ * Resource action: redis stop on redis-bundle-0
++ * Resource action: redis-bundle-0 stop on undercloud
++ * Pseudo action: redis-bundle-master_stopped_0
++ * Pseudo action: redis-bundle_stopped_0
++ * Pseudo action: redis-bundle_start_0
++ * Resource action: redis-bundle-docker-0 stop on undercloud
++ * Pseudo action: haproxy-bundle_stop_0
++ * Resource action: haproxy-bundle-docker-0 stop on undercloud
++ * Pseudo action: haproxy-bundle_stopped_0
++ * Resource action: ip-192.168.122.254 stop on undercloud
++ * Resource action: ip-192.168.122.250 stop on undercloud
++ * Resource action: ip-192.168.122.249 stop on undercloud
++ * Resource action: ip-192.168.122.253 stop on undercloud
++ * Resource action: ip-192.168.122.247 stop on undercloud
++ * Resource action: ip-192.168.122.248 stop on undercloud
++ * Cluster action: do_shutdown on undercloud
++ * Pseudo action: all_stopped
++
++Revised cluster status:
++Online: [ undercloud ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Stopped
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Stopped
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Stopped
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Stopped
++
+diff --git a/pengine/test10/bundle-order-partial-stop.xml b/pengine/test10/bundle-order-partial-stop.xml
+new file mode 100644
+index 0000000..467082a
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial-stop.xml
+@@ -0,0 +1,421 @@
++<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="undercloud">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
++ <instance_attributes id="nodes-rabbitmq-bundle-0">
++ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@undercloud"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <bundle id="rabbitmq-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3121"/>
++ <storage>
++ <storage-mapping id="rabbitmq-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/rabbitmq.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="rabbitmq-cfg-data" options="ro" source-dir="/var/lib/config-data/rabbitmq/etc/rabbitmq" target-dir="/etc/rabbitmq"/>
++ <storage-mapping id="rabbitmq-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="rabbitmq-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="rabbitmq-lib" options="rw" source-dir="/var/lib/rabbitmq" target-dir="/var/lib/rabbitmq"/>
++ <storage-mapping id="rabbitmq-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="rabbitmq-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="rabbitmq-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster">
++ <instance_attributes id="rabbitmq-instance_attributes">
++ <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </instance_attributes>
++ <meta_attributes id="rabbitmq-meta_attributes">
++ <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="rabbitmq-monitor-interval-10" interval="10" name="monitor" timeout="40"/>
++ <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="galera-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
++ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
++ <instance_attributes id="galera-instance_attributes">
++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:undercloud"/>
++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://undercloud"/>
++ </instance_attributes>
++ <meta_attributes id="galera-meta_attributes">
++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
++ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
++ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="redis-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3124"/>
++ <storage>
++ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
++ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
++ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
++ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
++ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
++ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
++ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
++ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
++ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
++ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <primitive class="ocf" id="ip-192.168.122.254" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.254-instance_attributes">
++ <nvpair id="ip-192.168.122.254-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.254-instance_attributes-ip" name="ip" value="192.168.122.254"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.254-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.254-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.254-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.250" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.250-instance_attributes">
++ <nvpair id="ip-192.168.122.250-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.250-instance_attributes-ip" name="ip" value="192.168.122.250"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.250-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.250-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.250-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.249" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.249-instance_attributes">
++ <nvpair id="ip-192.168.122.249-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.249-instance_attributes-ip" name="ip" value="192.168.122.249"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.249-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.249-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.249-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.253" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.253-instance_attributes">
++ <nvpair id="ip-192.168.122.253-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.253-instance_attributes-ip" name="ip" value="192.168.122.253"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.253-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.253-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.253-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.247" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.247-instance_attributes">
++ <nvpair id="ip-192.168.122.247-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.247-instance_attributes-ip" name="ip" value="192.168.122.247"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.247-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.247-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.247-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.248" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.248-instance_attributes">
++ <nvpair id="ip-192.168.122.248-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.248-instance_attributes-ip" name="ip" value="192.168.122.248"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.248-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.248-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.248-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ </bundle>
++ <bundle id="openstack-cinder-volume">
++ <docker run-command="/bin/bash /usr/local/bin/kolla_start" network="host" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" options="--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1"/>
++ <storage>
++ <storage-mapping target-dir="/var/lib/kolla/config_files/config.json" options="ro" id="cinder-volume-cfg-files" source-dir="/var/lib/kolla/config_files/cinder_volume.json"/>
++ <storage-mapping target-dir="/etc/cinder" options="ro" id="cinder-volume-cfg-data" source-dir="/var/lib/config-data/cinder/etc/cinder"/>
++ <storage-mapping target-dir="/etc/hosts" options="ro" id="cinder-volume-hosts" source-dir="/etc/hosts"/>
++ <storage-mapping target-dir="/etc/localtime" options="ro" id="cinder-volume-localtime" source-dir="/etc/localtime"/>
++ <storage-mapping target-dir="/dev" options="rw" id="cinder-volume-dev" source-dir="/dev"/>
++ <storage-mapping target-dir="/run" options="rw" id="cinder-volume-run" source-dir="/run"/>
++ <storage-mapping target-dir="/sys" options="rw" id="cinder-volume-sys" source-dir="/sys"/>
++ <storage-mapping target-dir="/lib/modules" options="ro" id="cinder-lib-modules" source-dir="/lib/modules"/>
++ <storage-mapping target-dir="/etc/iscsi" options="rw" id="cinder-volume-iscsi" source-dir="/etc/iscsi"/>
++ <storage-mapping target-dir="/var/lib/cinder" options="rw" id="cinder-volume-var-lib-cinder" source-dir="/var/lib/cinder"/>
++ <storage-mapping target-dir="/var/log/cinder" options="rw" id="cinder-volume-var-log" source-dir="/var/log/containers/cinder"/>
++ </storage>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-rabbitmq-bundle" resource-discovery="exclusive" rsc="rabbitmq-bundle">
++ <rule id="location-rabbitmq-bundle-rule" score="0">
++ <expression attribute="rabbitmq-role" id="location-rabbitmq-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
++ <rule id="location-galera-bundle-rule" score="0">
++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
++ <rule id="location-redis-bundle-rule" score="0">
++ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.254" resource-discovery="exclusive" rsc="ip-192.168.122.254">
++ <rule id="location-ip-192.168.122.254-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.254-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.250" resource-discovery="exclusive" rsc="ip-192.168.122.250">
++ <rule id="location-ip-192.168.122.250-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.250-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.249" resource-discovery="exclusive" rsc="ip-192.168.122.249">
++ <rule id="location-ip-192.168.122.249-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.249-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.253" resource-discovery="exclusive" rsc="ip-192.168.122.253">
++ <rule id="location-ip-192.168.122.253-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.253-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.247" resource-discovery="exclusive" rsc="ip-192.168.122.247">
++ <rule id="location-ip-192.168.122.247-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.247-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.248" resource-discovery="exclusive" rsc="ip-192.168.122.248">
++ <rule id="location-ip-192.168.122.248-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.248-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_order first="ip-192.168.122.254" first-action="start" id="order-ip-192.168.122.254-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.254-haproxy-bundle-INFINITY" rsc="ip-192.168.122.254" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.250" first-action="start" id="order-ip-192.168.122.250-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.250-haproxy-bundle-INFINITY" rsc="ip-192.168.122.250" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.249" first-action="start" id="order-ip-192.168.122.249-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.249-haproxy-bundle-INFINITY" rsc="ip-192.168.122.249" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.253" first-action="start" id="order-ip-192.168.122.253-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.253-haproxy-bundle-INFINITY" rsc="ip-192.168.122.253" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.247" first-action="start" id="order-ip-192.168.122.247-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.247-haproxy-bundle-INFINITY" rsc="ip-192.168.122.247" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.248" first-action="start" id="order-ip-192.168.122.248-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.248-haproxy-bundle-INFINITY" rsc="ip-192.168.122.248" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_colocation id="colocation-rabbit-with-haproxy-bundle-INFINITY" rsc="rabbitmq-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="haproxy-bundle" first-action="start" id="order-1" kind="Optional" then="redis-bundle" then-action="start"/>
++ <rsc_order first="redis-bundle" first-action="start" id="order-2" kind="Optional" then="galera-bundle" then-action="start"/>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="undercloud" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="1">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="10" rc-code="0" op-status="0" interval="0" last-run="1496303575" last-rc-change="1496303575" exec-time="850" queue-time="0" op-digest="50baee2f0a77a08b0616f0cf07ab724b"/>
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_monitor_60000" operation_key="rabbitmq-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303576" exec-time="76" queue-time="0" op-digest="25cb53ea3977a912bf4c0f70fe9e3c21"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0">
++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1496303576" last-rc-change="1496303576" exec-time="0" queue-time="0" op-digest="ee16760fa8c27e6585c1e35c244d7652" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="rabbitmq-bundle-0_monitor_60000" operation_key="rabbitmq-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303578" exec-time="0" queue-time="0" op-digest="92f557a57dfbb50be88282a4674ea681"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="16:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;16:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="17" rc-code="0" op-status="0" interval="0" last-run="1496303595" last-rc-change="1496303595" exec-time="920" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
++ <lrm_rsc_op id="galera-bundle-docker-0_monitor_60000" operation_key="galera-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="17:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;17:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="18" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303596" exec-time="70" queue-time="0" op-digest="f7e456f82e276e07f726e6e86e93bbb3"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0">
++ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="18:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;18:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1496303596" last-rc-change="1496303596" exec-time="0" queue-time="0" op-digest="edf22e374195d93e82f3c9cc174d519f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="galera-bundle-0_monitor_60000" operation_key="galera-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="20:9:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;20:9:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303600" exec-time="0" queue-time="0" op-digest="40c47455f6993c120f9a064ae51260db"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="824" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
++ <lrm_rsc_op id="redis-bundle-docker-0_monitor_60000" operation_key="redis-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="29" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303664" exec-time="74" queue-time="0" op-digest="81e9d490c493f8d22fab2822ec0bed8a"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
++ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="5" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="0" queue-time="0" op-digest="9a6281aab223af75359e5b5a07161162" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="redis-bundle-0_monitor_60000" operation_key="redis-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="6" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303667" exec-time="0" queue-time="0" op-digest="04ffb214f519838e579c97e8aa002054"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.254" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.254_last_0" operation_key="ip-192.168.122.254_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1496303683" last-rc-change="1496303683" exec-time="63" queue-time="0" op-digest="d6875d68a09b5550030b8b6b7cdc9294"/>
++ <lrm_rsc_op id="ip-192.168.122.254_monitor_10000" operation_key="ip-192.168.122.254_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="36" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303683" exec-time="32" queue-time="0" op-digest="5c1c6d0e79751e4002b12de513351a48"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.250" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.250_last_0" operation_key="ip-192.168.122.250_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="41" rc-code="0" op-status="0" interval="0" last-run="1496303696" last-rc-change="1496303696" exec-time="53" queue-time="0" op-digest="b33b44b3e9e535336108cb43f7474ef6"/>
++ <lrm_rsc_op id="ip-192.168.122.250_monitor_10000" operation_key="ip-192.168.122.250_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="42" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303696" exec-time="32" queue-time="0" op-digest="a4aec96753744d49ca9a798d3b0f12aa"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.249" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.249_last_0" operation_key="ip-192.168.122.249_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="47" rc-code="0" op-status="0" interval="0" last-run="1496303710" last-rc-change="1496303710" exec-time="52" queue-time="0" op-digest="22e58cb8beefb7810d8bf03d0d3a906d"/>
++ <lrm_rsc_op id="ip-192.168.122.249_monitor_10000" operation_key="ip-192.168.122.249_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="48" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303710" exec-time="30" queue-time="1" op-digest="860855adc72293b04b1becaeea73ced1"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.253" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.253_last_0" operation_key="ip-192.168.122.253_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1496303723" last-rc-change="1496303723" exec-time="69" queue-time="0" op-digest="f36f4ccbd5f16e4e335c65f0802ddc20"/>
++ <lrm_rsc_op id="ip-192.168.122.253_monitor_10000" operation_key="ip-192.168.122.253_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="54" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303724" exec-time="39" queue-time="0" op-digest="0ee9a624c731239cdbef5c9118cc2bbe"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.247" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.247_last_0" operation_key="ip-192.168.122.247_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="59" rc-code="0" op-status="0" interval="0" last-run="1496303737" last-rc-change="1496303737" exec-time="57" queue-time="0" op-digest="37013238bd139c2b589a9508ff4b5c07"/>
++ <lrm_rsc_op id="ip-192.168.122.247_monitor_10000" operation_key="ip-192.168.122.247_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="60" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303737" exec-time="35" queue-time="0" op-digest="22cd67ba6dcb54e4f4e6ea3450a52b77"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.248" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.248_last_0" operation_key="ip-192.168.122.248_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="65" rc-code="0" op-status="0" interval="0" last-run="1496303750" last-rc-change="1496303750" exec-time="52" queue-time="0" op-digest="7565857faaf5a927e16322d5da3e4e38"/>
++ <lrm_rsc_op id="ip-192.168.122.248_monitor_10000" operation_key="ip-192.168.122.248_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="66" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303750" exec-time="30" queue-time="0" op-digest="2182cf0cd059930b67d43545b4950c6f"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="72:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;72:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="71" rc-code="0" op-status="0" interval="0" last-run="1496303754" last-rc-change="1496303754" exec-time="862" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ <lrm_rsc_op id="haproxy-bundle-docker-0_monitor_60000" operation_key="haproxy-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="73:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;73:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="72" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303754" exec-time="64" queue-time="0" op-digest="8e98096b4e2923c7be3ce5fa2646c524"/>
++ </lrm_resource>
++ <lrm_resource id="openstack-cinder-volume-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_0" operation_key="openstack-cinder-volume-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="161" rc-code="0" op-status="0" interval="0" last-run="1496305247" last-rc-change="1496305247" exec-time="838" queue-time="0" op-digest="3aa35269c96594499406635b536dcf19"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_monitor_60000" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="162" rc-code="0" op-status="0" interval="60000" last-rc-change="1496305248" exec-time="63" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_failure_0" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:7;14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="158" rc-code="7" op-status="0" interval="60000" last-rc-change="1496305247" exec-time="0" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="1">
++ <instance_attributes id="status-1">
++ <nvpair id="status-1-shutdown" name="shutdown" value="1000"/>
++ <nvpair id="status-1-fail-count-openstack-cinder-volume-docker-0" name="fail-count-openstack-cinder-volume-docker-0" value="21"/>
++ <nvpair id="status-1-last-failure-openstack-cinder-volume-docker-0" name="last-failure-openstack-cinder-volume-docker-0" value="1496305247"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state remote_node="true" id="rabbitmq-bundle-0" uname="rabbitmq-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="rabbitmq-bundle-0">
++ <instance_attributes id="status-rabbitmq-bundle-0">
++ <nvpair id="status-rabbitmq-bundle-0-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@undercloud"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="rabbitmq-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="8:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;8:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="7" rc-code="0" op-status="0" interval="0" last-run="1496303578" last-rc-change="1496303578" exec-time="16270" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/>
++ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="11:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;11:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="49" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303595" exec-time="5628" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="galera-bundle-0" uname="galera-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="galera-bundle-0">
++ <instance_attributes id="status-galera-bundle-0">
++ <nvpair id="status-galera-bundle-0-master-galera" name="master-galera" value="100"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="galera-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="galera_last_0" operation_key="galera_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="23:10:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;23:10:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="86" rc-code="0" op-status="0" interval="0" last-run="1496303611" last-rc-change="1496303611" exec-time="11500" queue-time="0" op-digest="d5f683936a1e6d67cbca8e284acab041" op-secure-params=" user " op-secure-digest="d5f683936a1e6d67cbca8e284acab041"/>
++ <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="25:11:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:8;25:11:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="187" rc-code="8" op-status="0" interval="10000" last-rc-change="1496303623" exec-time="298" queue-time="0" op-digest="a7243ad3ba33c28a04a75e5567c9d73b" op-secure-params=" user " op-secure-digest="d5f683936a1e6d67cbca8e284acab041"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="redis-bundle-0" uname="redis-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="redis-bundle-0">
++ <instance_attributes id="status-redis-bundle-0">
++ <nvpair id="status-redis-bundle-0-master-redis" name="master-redis" value="1"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="redis-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="58" rc-code="0" op-status="0" interval="0" last-run="1496303672" last-rc-change="1496303672" exec-time="1583" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="44:18:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:8;44:18:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="80" rc-code="8" op-status="0" interval="20000" last-rc-change="1496303675" exec-time="771" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ </status>
++</cib>
+diff --git a/pengine/test10/bundle-order-partial.dot b/pengine/test10/bundle-order-partial.dot
+new file mode 100644
+index 0000000..4b30191
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial.dot
+@@ -0,0 +1,2 @@
++digraph "g" {
++}
+diff --git a/pengine/test10/bundle-order-partial.exp b/pengine/test10/bundle-order-partial.exp
+new file mode 100644
+index 0000000..56e315f
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial.exp
+@@ -0,0 +1 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0"/>
+diff --git a/pengine/test10/bundle-order-partial.scores b/pengine/test10/bundle-order-partial.scores
+new file mode 100644
+index 0000000..be91c96
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial.scores
+@@ -0,0 +1,197 @@
++Allocation scores:
++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:0 allocation score on undercloud: -INFINITY
++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY
++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: redis:0 allocation score on undercloud: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on undercloud: 0
++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on undercloud: INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on undercloud: 0
++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera:0 allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on undercloud: 0
++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle allocation score on undercloud: 0
++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0
++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++container_color: rabbitmq:0 allocation score on undercloud: 0
++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on undercloud: 0
++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on undercloud: 0
++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:0 allocation score on undercloud: 0
++galera:0 promotion score on galera-bundle-0: 100
++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on undercloud: INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: galera:0 allocation score on galera-bundle-0: INFINITY
++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:0 allocation score on undercloud: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on undercloud: INFINITY
++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on undercloud: INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on undercloud: INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY
++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: redis:0 allocation score on undercloud: -INFINITY
++redis:0 promotion score on redis-bundle-0: 1
+diff --git a/pengine/test10/bundle-order-partial.summary b/pengine/test10/bundle-order-partial.summary
+new file mode 100644
+index 0000000..6155230
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial.summary
+@@ -0,0 +1,47 @@
++
++Current cluster status:
++Online: [ undercloud ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Master undercloud
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud
++
++Transition Summary:
++
++Executing cluster transition:
++
++Revised cluster status:
++Online: [ undercloud ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Master undercloud
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud
++
+diff --git a/pengine/test10/bundle-order-partial.xml b/pengine/test10/bundle-order-partial.xml
+new file mode 100644
+index 0000000..eda20bb
+--- /dev/null
++++ b/pengine/test10/bundle-order-partial.xml
+@@ -0,0 +1,421 @@
++<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="undercloud">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
++ <instance_attributes id="nodes-rabbitmq-bundle-0">
++ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@undercloud"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <bundle id="rabbitmq-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3121"/>
++ <storage>
++ <storage-mapping id="rabbitmq-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/rabbitmq.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="rabbitmq-cfg-data" options="ro" source-dir="/var/lib/config-data/rabbitmq/etc/rabbitmq" target-dir="/etc/rabbitmq"/>
++ <storage-mapping id="rabbitmq-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="rabbitmq-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="rabbitmq-lib" options="rw" source-dir="/var/lib/rabbitmq" target-dir="/var/lib/rabbitmq"/>
++ <storage-mapping id="rabbitmq-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="rabbitmq-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="rabbitmq-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster">
++ <instance_attributes id="rabbitmq-instance_attributes">
++ <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </instance_attributes>
++ <meta_attributes id="rabbitmq-meta_attributes">
++ <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="rabbitmq-monitor-interval-10" interval="10" name="monitor" timeout="40"/>
++ <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="galera-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
++ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
++ <instance_attributes id="galera-instance_attributes">
++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:undercloud"/>
++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://undercloud"/>
++ </instance_attributes>
++ <meta_attributes id="galera-meta_attributes">
++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
++ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
++ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="redis-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3124"/>
++ <storage>
++ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
++ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
++ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
++ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
++ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
++ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
++ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
++ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
++ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
++ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <primitive class="ocf" id="ip-192.168.122.254" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.254-instance_attributes">
++ <nvpair id="ip-192.168.122.254-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.254-instance_attributes-ip" name="ip" value="192.168.122.254"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.254-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.254-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.254-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.250" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.250-instance_attributes">
++ <nvpair id="ip-192.168.122.250-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.250-instance_attributes-ip" name="ip" value="192.168.122.250"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.250-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.250-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.250-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.249" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.249-instance_attributes">
++ <nvpair id="ip-192.168.122.249-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.249-instance_attributes-ip" name="ip" value="192.168.122.249"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.249-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.249-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.249-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.253" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.253-instance_attributes">
++ <nvpair id="ip-192.168.122.253-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.253-instance_attributes-ip" name="ip" value="192.168.122.253"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.253-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.253-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.253-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.247" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.247-instance_attributes">
++ <nvpair id="ip-192.168.122.247-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.247-instance_attributes-ip" name="ip" value="192.168.122.247"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.247-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.247-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.247-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.248" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.248-instance_attributes">
++ <nvpair id="ip-192.168.122.248-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.248-instance_attributes-ip" name="ip" value="192.168.122.248"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.248-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.248-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.248-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ </bundle>
++ <bundle id="openstack-cinder-volume">
++ <docker run-command="/bin/bash /usr/local/bin/kolla_start" network="host" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" options="--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1"/>
++ <storage>
++ <storage-mapping target-dir="/var/lib/kolla/config_files/config.json" options="ro" id="cinder-volume-cfg-files" source-dir="/var/lib/kolla/config_files/cinder_volume.json"/>
++ <storage-mapping target-dir="/etc/cinder" options="ro" id="cinder-volume-cfg-data" source-dir="/var/lib/config-data/cinder/etc/cinder"/>
++ <storage-mapping target-dir="/etc/hosts" options="ro" id="cinder-volume-hosts" source-dir="/etc/hosts"/>
++ <storage-mapping target-dir="/etc/localtime" options="ro" id="cinder-volume-localtime" source-dir="/etc/localtime"/>
++ <storage-mapping target-dir="/dev" options="rw" id="cinder-volume-dev" source-dir="/dev"/>
++ <storage-mapping target-dir="/run" options="rw" id="cinder-volume-run" source-dir="/run"/>
++ <storage-mapping target-dir="/sys" options="rw" id="cinder-volume-sys" source-dir="/sys"/>
++ <storage-mapping target-dir="/lib/modules" options="ro" id="cinder-lib-modules" source-dir="/lib/modules"/>
++ <storage-mapping target-dir="/etc/iscsi" options="rw" id="cinder-volume-iscsi" source-dir="/etc/iscsi"/>
++ <storage-mapping target-dir="/var/lib/cinder" options="rw" id="cinder-volume-var-lib-cinder" source-dir="/var/lib/cinder"/>
++ <storage-mapping target-dir="/var/log/cinder" options="rw" id="cinder-volume-var-log" source-dir="/var/log/containers/cinder"/>
++ </storage>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-rabbitmq-bundle" resource-discovery="exclusive" rsc="rabbitmq-bundle">
++ <rule id="location-rabbitmq-bundle-rule" score="0">
++ <expression attribute="rabbitmq-role" id="location-rabbitmq-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
++ <rule id="location-galera-bundle-rule" score="0">
++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
++ <rule id="location-redis-bundle-rule" score="0">
++ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.254" resource-discovery="exclusive" rsc="ip-192.168.122.254">
++ <rule id="location-ip-192.168.122.254-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.254-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.250" resource-discovery="exclusive" rsc="ip-192.168.122.250">
++ <rule id="location-ip-192.168.122.250-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.250-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.249" resource-discovery="exclusive" rsc="ip-192.168.122.249">
++ <rule id="location-ip-192.168.122.249-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.249-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.253" resource-discovery="exclusive" rsc="ip-192.168.122.253">
++ <rule id="location-ip-192.168.122.253-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.253-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.247" resource-discovery="exclusive" rsc="ip-192.168.122.247">
++ <rule id="location-ip-192.168.122.247-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.247-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.248" resource-discovery="exclusive" rsc="ip-192.168.122.248">
++ <rule id="location-ip-192.168.122.248-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.248-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_order first="ip-192.168.122.254" first-action="start" id="order-ip-192.168.122.254-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.254-haproxy-bundle-INFINITY" rsc="ip-192.168.122.254" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.250" first-action="start" id="order-ip-192.168.122.250-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.250-haproxy-bundle-INFINITY" rsc="ip-192.168.122.250" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.249" first-action="start" id="order-ip-192.168.122.249-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.249-haproxy-bundle-INFINITY" rsc="ip-192.168.122.249" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.253" first-action="start" id="order-ip-192.168.122.253-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.253-haproxy-bundle-INFINITY" rsc="ip-192.168.122.253" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.247" first-action="start" id="order-ip-192.168.122.247-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.247-haproxy-bundle-INFINITY" rsc="ip-192.168.122.247" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.248" first-action="start" id="order-ip-192.168.122.248-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.248-haproxy-bundle-INFINITY" rsc="ip-192.168.122.248" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_colocation id="colocation-rabbit-with-haproxy-bundle-INFINITY" rsc="rabbitmq-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="haproxy-bundle" first-action="start" id="order-1" kind="Optional" then="redis-bundle" then-action="start"/>
++ <rsc_order first="redis-bundle" first-action="start" id="order-2" kind="Optional" then="galera-bundle" then-action="start"/>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="undercloud" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="1">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="10" rc-code="0" op-status="0" interval="0" last-run="1496303575" last-rc-change="1496303575" exec-time="850" queue-time="0" op-digest="50baee2f0a77a08b0616f0cf07ab724b"/>
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_monitor_60000" operation_key="rabbitmq-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303576" exec-time="76" queue-time="0" op-digest="25cb53ea3977a912bf4c0f70fe9e3c21"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0">
++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1496303576" last-rc-change="1496303576" exec-time="0" queue-time="0" op-digest="ee16760fa8c27e6585c1e35c244d7652" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="rabbitmq-bundle-0_monitor_60000" operation_key="rabbitmq-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303578" exec-time="0" queue-time="0" op-digest="92f557a57dfbb50be88282a4674ea681"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="16:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;16:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="17" rc-code="0" op-status="0" interval="0" last-run="1496303595" last-rc-change="1496303595" exec-time="920" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
++ <lrm_rsc_op id="galera-bundle-docker-0_monitor_60000" operation_key="galera-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="17:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;17:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="18" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303596" exec-time="70" queue-time="0" op-digest="f7e456f82e276e07f726e6e86e93bbb3"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0">
++ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="18:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;18:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1496303596" last-rc-change="1496303596" exec-time="0" queue-time="0" op-digest="edf22e374195d93e82f3c9cc174d519f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="galera-bundle-0_monitor_60000" operation_key="galera-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="20:9:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;20:9:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303600" exec-time="0" queue-time="0" op-digest="40c47455f6993c120f9a064ae51260db"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="824" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
++ <lrm_rsc_op id="redis-bundle-docker-0_monitor_60000" operation_key="redis-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="29" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303664" exec-time="74" queue-time="0" op-digest="81e9d490c493f8d22fab2822ec0bed8a"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
++ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="5" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="0" queue-time="0" op-digest="9a6281aab223af75359e5b5a07161162" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="redis-bundle-0_monitor_60000" operation_key="redis-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="6" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303667" exec-time="0" queue-time="0" op-digest="04ffb214f519838e579c97e8aa002054"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.254" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.254_last_0" operation_key="ip-192.168.122.254_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1496303683" last-rc-change="1496303683" exec-time="63" queue-time="0" op-digest="d6875d68a09b5550030b8b6b7cdc9294"/>
++ <lrm_rsc_op id="ip-192.168.122.254_monitor_10000" operation_key="ip-192.168.122.254_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="36" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303683" exec-time="32" queue-time="0" op-digest="5c1c6d0e79751e4002b12de513351a48"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.250" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.250_last_0" operation_key="ip-192.168.122.250_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="41" rc-code="0" op-status="0" interval="0" last-run="1496303696" last-rc-change="1496303696" exec-time="53" queue-time="0" op-digest="b33b44b3e9e535336108cb43f7474ef6"/>
++ <lrm_rsc_op id="ip-192.168.122.250_monitor_10000" operation_key="ip-192.168.122.250_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="42" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303696" exec-time="32" queue-time="0" op-digest="a4aec96753744d49ca9a798d3b0f12aa"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.249" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.249_last_0" operation_key="ip-192.168.122.249_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="47" rc-code="0" op-status="0" interval="0" last-run="1496303710" last-rc-change="1496303710" exec-time="52" queue-time="0" op-digest="22e58cb8beefb7810d8bf03d0d3a906d"/>
++ <lrm_rsc_op id="ip-192.168.122.249_monitor_10000" operation_key="ip-192.168.122.249_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="48" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303710" exec-time="30" queue-time="1" op-digest="860855adc72293b04b1becaeea73ced1"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.253" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.253_last_0" operation_key="ip-192.168.122.253_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1496303723" last-rc-change="1496303723" exec-time="69" queue-time="0" op-digest="f36f4ccbd5f16e4e335c65f0802ddc20"/>
++ <lrm_rsc_op id="ip-192.168.122.253_monitor_10000" operation_key="ip-192.168.122.253_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="54" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303724" exec-time="39" queue-time="0" op-digest="0ee9a624c731239cdbef5c9118cc2bbe"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.247" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.247_last_0" operation_key="ip-192.168.122.247_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="59" rc-code="0" op-status="0" interval="0" last-run="1496303737" last-rc-change="1496303737" exec-time="57" queue-time="0" op-digest="37013238bd139c2b589a9508ff4b5c07"/>
++ <lrm_rsc_op id="ip-192.168.122.247_monitor_10000" operation_key="ip-192.168.122.247_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="60" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303737" exec-time="35" queue-time="0" op-digest="22cd67ba6dcb54e4f4e6ea3450a52b77"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.248" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.248_last_0" operation_key="ip-192.168.122.248_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="65" rc-code="0" op-status="0" interval="0" last-run="1496303750" last-rc-change="1496303750" exec-time="52" queue-time="0" op-digest="7565857faaf5a927e16322d5da3e4e38"/>
++ <lrm_rsc_op id="ip-192.168.122.248_monitor_10000" operation_key="ip-192.168.122.248_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="66" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303750" exec-time="30" queue-time="0" op-digest="2182cf0cd059930b67d43545b4950c6f"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="72:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;72:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="71" rc-code="0" op-status="0" interval="0" last-run="1496303754" last-rc-change="1496303754" exec-time="862" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ <lrm_rsc_op id="haproxy-bundle-docker-0_monitor_60000" operation_key="haproxy-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="73:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;73:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="72" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303754" exec-time="64" queue-time="0" op-digest="8e98096b4e2923c7be3ce5fa2646c524"/>
++ </lrm_resource>
++ <lrm_resource id="openstack-cinder-volume-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_0" operation_key="openstack-cinder-volume-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="161" rc-code="0" op-status="0" interval="0" last-run="1496305247" last-rc-change="1496305247" exec-time="838" queue-time="0" op-digest="3aa35269c96594499406635b536dcf19"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_monitor_60000" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="162" rc-code="0" op-status="0" interval="60000" last-rc-change="1496305248" exec-time="63" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_failure_0" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:7;14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="158" rc-code="7" op-status="0" interval="60000" last-rc-change="1496305247" exec-time="0" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="1">
++ <instance_attributes id="status-1">
++ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
++ <nvpair id="status-1-fail-count-openstack-cinder-volume-docker-0" name="fail-count-openstack-cinder-volume-docker-0" value="21"/>
++ <nvpair id="status-1-last-failure-openstack-cinder-volume-docker-0" name="last-failure-openstack-cinder-volume-docker-0" value="1496305247"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state remote_node="true" id="rabbitmq-bundle-0" uname="rabbitmq-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="rabbitmq-bundle-0">
++ <instance_attributes id="status-rabbitmq-bundle-0">
++ <nvpair id="status-rabbitmq-bundle-0-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@undercloud"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="rabbitmq-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="8:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;8:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="7" rc-code="0" op-status="0" interval="0" last-run="1496303578" last-rc-change="1496303578" exec-time="16270" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/>
++ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="11:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;11:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="49" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303595" exec-time="5628" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="galera-bundle-0" uname="galera-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="galera-bundle-0">
++ <instance_attributes id="status-galera-bundle-0">
++ <nvpair id="status-galera-bundle-0-master-galera" name="master-galera" value="100"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="galera-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="galera_last_0" operation_key="galera_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="23:10:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;23:10:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="86" rc-code="0" op-status="0" interval="0" last-run="1496303611" last-rc-change="1496303611" exec-time="11500" queue-time="0" op-digest="d5f683936a1e6d67cbca8e284acab041" op-secure-params=" user " op-secure-digest="d5f683936a1e6d67cbca8e284acab041"/>
++ <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="25:11:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:8;25:11:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="187" rc-code="8" op-status="0" interval="10000" last-rc-change="1496303623" exec-time="298" queue-time="0" op-digest="a7243ad3ba33c28a04a75e5567c9d73b" op-secure-params=" user " op-secure-digest="d5f683936a1e6d67cbca8e284acab041"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="redis-bundle-0" uname="redis-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="redis-bundle-0">
++ <instance_attributes id="status-redis-bundle-0">
++ <nvpair id="status-redis-bundle-0-master-redis" name="master-redis" value="1"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="redis-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="58" rc-code="0" op-status="0" interval="0" last-run="1496303672" last-rc-change="1496303672" exec-time="1583" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="44:18:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:8;44:18:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="80" rc-code="8" op-status="0" interval="20000" last-rc-change="1496303675" exec-time="771" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ </status>
++</cib>
+diff --git a/pengine/test10/bundle-order-startup.dot b/pengine/test10/bundle-order-startup.dot
+new file mode 100644
+index 0000000..73947a5
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup.dot
+@@ -0,0 +1,139 @@
++digraph "g" {
++"galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold]
++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
++"galera-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold]
++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold]
++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold]
++"haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle_running_0" -> "redis-bundle_start_0" [ style = bold]
++"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
++"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"ip-192.168.122.247_monitor_0 undercloud" -> "ip-192.168.122.247_start_0 undercloud" [ style = bold]
++"ip-192.168.122.247_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.247_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.247_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold]
++"ip-192.168.122.247_start_0 undercloud" -> "ip-192.168.122.247_monitor_10000 undercloud" [ style = bold]
++"ip-192.168.122.247_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.248_monitor_0 undercloud" -> "ip-192.168.122.248_start_0 undercloud" [ style = bold]
++"ip-192.168.122.248_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.248_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.248_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold]
++"ip-192.168.122.248_start_0 undercloud" -> "ip-192.168.122.248_monitor_10000 undercloud" [ style = bold]
++"ip-192.168.122.248_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.249_monitor_0 undercloud" -> "ip-192.168.122.249_start_0 undercloud" [ style = bold]
++"ip-192.168.122.249_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.249_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.249_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold]
++"ip-192.168.122.249_start_0 undercloud" -> "ip-192.168.122.249_monitor_10000 undercloud" [ style = bold]
++"ip-192.168.122.249_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.250_monitor_0 undercloud" -> "ip-192.168.122.250_start_0 undercloud" [ style = bold]
++"ip-192.168.122.250_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.250_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.250_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold]
++"ip-192.168.122.250_start_0 undercloud" -> "ip-192.168.122.250_monitor_10000 undercloud" [ style = bold]
++"ip-192.168.122.250_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.253_monitor_0 undercloud" -> "ip-192.168.122.253_start_0 undercloud" [ style = bold]
++"ip-192.168.122.253_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.253_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.253_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold]
++"ip-192.168.122.253_start_0 undercloud" -> "ip-192.168.122.253_monitor_10000 undercloud" [ style = bold]
++"ip-192.168.122.253_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.254_monitor_0 undercloud" -> "ip-192.168.122.254_start_0 undercloud" [ style = bold]
++"ip-192.168.122.254_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.254_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.254_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold]
++"ip-192.168.122.254_start_0 undercloud" -> "ip-192.168.122.254_monitor_10000 undercloud" [ style = bold]
++"ip-192.168.122.254_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume-docker-0_monitor_0 undercloud" -> "openstack-cinder-volume-docker-0_start_0 undercloud" [ style = bold]
++"openstack-cinder-volume-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume-docker-0_start_0 undercloud" -> "openstack-cinder-volume-docker-0_monitor_60000 undercloud" [ style = bold]
++"openstack-cinder-volume-docker-0_start_0 undercloud" -> "openstack-cinder-volume_running_0" [ style = bold]
++"openstack-cinder-volume-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume_running_0" [ style=bold color="green" fontcolor="orange"]
++"openstack-cinder-volume_start_0" -> "openstack-cinder-volume-docker-0_start_0 undercloud" [ style = bold]
++"openstack-cinder-volume_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq-bundle-0_monitor_60000 undercloud" [ style = bold]
++"rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold]
++"rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-docker-0_monitor_0 undercloud" -> "rabbitmq-bundle-docker-0_start_0 undercloud" [ style = bold]
++"rabbitmq-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle-0_start_0 undercloud" [ style = bold]
++"rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold]
++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-0_start_0 undercloud" [ style = bold]
++"rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold]
++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-0_start_0 undercloud" -> "redis-bundle-0_monitor_60000 undercloud" [ style = bold]
++"redis-bundle-0_start_0 undercloud" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold]
++"redis-bundle-0_start_0 undercloud" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold]
++"redis-bundle-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
++"redis-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold]
++"redis-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-0_start_0 undercloud" [ style = bold]
++"redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"redis-bundle-docker-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
++"redis-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold]
++"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold]
++"redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_running_0" -> "galera-bundle_start_0" [ style = bold]
++"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_start_0" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold]
++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold]
++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis:0_monitor_45000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis:0_monitor_60000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis:0_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = bold]
++"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold]
++"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold]
++"redis:0_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++}
+diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp
+new file mode 100644
+index 0000000..51d108d
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup.exp
+@@ -0,0 +1,825 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0">
++ <action_set>
++ <rsc_op id="22" operation="monitor" operation_key="rabbitmq:0_monitor_10000" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_timeout="40000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="15" operation="start" operation_key="rabbitmq-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="21" operation="start" operation_key="rabbitmq:0_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="21" operation="start" operation_key="rabbitmq:0_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_timeout="200000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="13" operation="start" operation_key="rabbitmq-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="15" operation="start" operation_key="rabbitmq-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="23" operation="start" operation_key="rabbitmq-bundle-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="2">
++ <action_set>
++ <rsc_op id="14" operation="monitor" operation_key="rabbitmq-bundle-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/rabbitmq-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/rabbitmq/etc/rabbitmq:/etc/rabbitmq:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/rabbitmq:/var/lib/rabbitmq:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/rabbitmq-bundle-0:/var/log -p 3121:3121 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="13" operation="start" operation_key="rabbitmq-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="13" operation="start" operation_key="rabbitmq-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/rabbitmq-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/rabbitmq/etc/rabbitmq:/etc/rabbitmq:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/rabbitmq:/var/lib/rabbitmq:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/rabbitmq-bundle-0:/var/log -p 3121:3121 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="2" operation="monitor" operation_key="rabbitmq-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="17" operation="start" operation_key="rabbitmq-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="4">
++ <action_set>
++ <rsc_op id="2" operation="monitor" operation_key="rabbitmq-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/rabbitmq-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/rabbitmq/etc/rabbitmq:/etc/rabbitmq:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/rabbitmq:/var/lib/rabbitmq:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/rabbitmq-bundle-0:/var/log -p 3121:3121 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="16" operation="monitor" operation_key="rabbitmq-bundle-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="rabbitmq-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3121"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="15" operation="start" operation_key="rabbitmq-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="6">
++ <action_set>
++ <rsc_op id="15" operation="start" operation_key="rabbitmq-bundle-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="rabbitmq-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3121"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="13" operation="start" operation_key="rabbitmq-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <rsc_op id="37" operation="monitor" operation_key="galera:0_monitor_30000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_role="Slave" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="35" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="8">
++ <action_set>
++ <rsc_op id="36" operation="monitor" operation_key="galera:0_monitor_20000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="35" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <rsc_op id="35" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="27" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="38" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="10">
++ <action_set>
++ <rsc_op id="28" operation="monitor" operation_key="galera-bundle-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="27" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <rsc_op id="27" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="31" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="12">
++ <action_set>
++ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="13">
++ <action_set>
++ <rsc_op id="30" operation="monitor" operation_key="galera-bundle-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14">
++ <action_set>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="27" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15">
++ <action_set>
++ <rsc_op id="60" operation="monitor" operation_key="redis:0_monitor_60000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_role="Slave" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="52" operation="start" operation_key="redis-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="58" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16">
++ <action_set>
++ <rsc_op id="59" operation="monitor" operation_key="redis:0_monitor_45000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="45000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="52" operation="start" operation_key="redis-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="58" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="17">
++ <action_set>
++ <rsc_op id="58" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="200000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="50" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="52" operation="start" operation_key="redis-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="61" operation="start" operation_key="redis-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="18">
++ <action_set>
++ <rsc_op id="51" operation="monitor" operation_key="redis-bundle-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="50" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="19">
++ <action_set>
++ <rsc_op id="50" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="4" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="54" operation="start" operation_key="redis-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20">
++ <action_set>
++ <rsc_op id="4" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="21">
++ <action_set>
++ <rsc_op id="53" operation="monitor" operation_key="redis-bundle-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="52" operation="start" operation_key="redis-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="22">
++ <action_set>
++ <rsc_op id="52" operation="start" operation_key="redis-bundle-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="50" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="23">
++ <action_set>
++ <rsc_op id="74" operation="monitor" operation_key="ip-192.168.122.254_monitor_10000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.254" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.254"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="73" operation="start" operation_key="ip-192.168.122.254_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="24">
++ <action_set>
++ <rsc_op id="73" operation="start" operation_key="ip-192.168.122.254_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.254" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="start" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.254"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="5" operation="monitor" operation_key="ip-192.168.122.254_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="25">
++ <action_set>
++ <rsc_op id="5" operation="monitor" operation_key="ip-192.168.122.254_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.254" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.254"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="26">
++ <action_set>
++ <rsc_op id="76" operation="monitor" operation_key="ip-192.168.122.250_monitor_10000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.250" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.250"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="75" operation="start" operation_key="ip-192.168.122.250_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="27">
++ <action_set>
++ <rsc_op id="75" operation="start" operation_key="ip-192.168.122.250_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.250" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="start" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.250"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="6" operation="monitor" operation_key="ip-192.168.122.250_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="28">
++ <action_set>
++ <rsc_op id="6" operation="monitor" operation_key="ip-192.168.122.250_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.250" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.250"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="29">
++ <action_set>
++ <rsc_op id="78" operation="monitor" operation_key="ip-192.168.122.249_monitor_10000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.249" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.249"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="77" operation="start" operation_key="ip-192.168.122.249_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="30">
++ <action_set>
++ <rsc_op id="77" operation="start" operation_key="ip-192.168.122.249_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.249" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="start" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.249"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="7" operation="monitor" operation_key="ip-192.168.122.249_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="31">
++ <action_set>
++ <rsc_op id="7" operation="monitor" operation_key="ip-192.168.122.249_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.249" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.249"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="32">
++ <action_set>
++ <rsc_op id="80" operation="monitor" operation_key="ip-192.168.122.253_monitor_10000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.253" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.253"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="79" operation="start" operation_key="ip-192.168.122.253_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="33">
++ <action_set>
++ <rsc_op id="79" operation="start" operation_key="ip-192.168.122.253_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.253" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="start" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.253"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="8" operation="monitor" operation_key="ip-192.168.122.253_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="34">
++ <action_set>
++ <rsc_op id="8" operation="monitor" operation_key="ip-192.168.122.253_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.253" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.253"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="35">
++ <action_set>
++ <rsc_op id="82" operation="monitor" operation_key="ip-192.168.122.247_monitor_10000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.247" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.247"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="81" operation="start" operation_key="ip-192.168.122.247_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="36">
++ <action_set>
++ <rsc_op id="81" operation="start" operation_key="ip-192.168.122.247_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.247" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="start" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.247"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="9" operation="monitor" operation_key="ip-192.168.122.247_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="37">
++ <action_set>
++ <rsc_op id="9" operation="monitor" operation_key="ip-192.168.122.247_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.247" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.247"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="38">
++ <action_set>
++ <rsc_op id="84" operation="monitor" operation_key="ip-192.168.122.248_monitor_10000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.248" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.248"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="83" operation="start" operation_key="ip-192.168.122.248_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="39">
++ <action_set>
++ <rsc_op id="83" operation="start" operation_key="ip-192.168.122.248_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.248" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="start" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.248"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="10" operation="monitor" operation_key="ip-192.168.122.248_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="40">
++ <action_set>
++ <rsc_op id="10" operation="monitor" operation_key="ip-192.168.122.248_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.248" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.248"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="41">
++ <action_set>
++ <rsc_op id="86" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="85" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="42">
++ <action_set>
++ <rsc_op id="85" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="11" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="87" operation="start" operation_key="haproxy-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="43">
++ <action_set>
++ <rsc_op id="11" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="44">
++ <action_set>
++ <rsc_op id="92" operation="monitor" operation_key="openstack-cinder-volume-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="openstack-cinder-volume-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/cinder/etc/cinder:/etc/cinder:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /dev:/dev:rw -v /run:/run:rw -v /sys:/sys:rw -v /lib/modules:/lib/modules:ro -v /etc/iscsi:/etc/iscsi:rw -v /var/lib/cinder:/var/lib/cinder:rw -v /var/log/containers/cinder:/var/log/cinder:rw --ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="91" operation="start" operation_key="openstack-cinder-volume-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="45">
++ <action_set>
++ <rsc_op id="91" operation="start" operation_key="openstack-cinder-volume-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="openstack-cinder-volume-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/cinder/etc/cinder:/etc/cinder:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /dev:/dev:rw -v /run:/run:rw -v /sys:/sys:rw -v /lib/modules:/lib/modules:ro -v /etc/iscsi:/etc/iscsi:rw -v /var/lib/cinder:/var/lib/cinder:rw -v /var/log/containers/cinder:/var/log/cinder:rw --ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="12" operation="monitor" operation_key="openstack-cinder-volume-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="93" operation="start" operation_key="openstack-cinder-volume_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="46">
++ <action_set>
++ <rsc_op id="12" operation="monitor" operation_key="openstack-cinder-volume-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="openstack-cinder-volume-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/cinder/etc/cinder:/etc/cinder:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /dev:/dev:rw -v /run:/run:rw -v /sys:/sys:rw -v /lib/modules:/lib/modules:ro -v /etc/iscsi:/etc/iscsi:rw -v /var/lib/cinder:/var/lib/cinder:rw -v /var/log/containers/cinder:/var/log/cinder:rw --ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="47" priority="1000000">
++ <action_set>
++ <pseudo_event id="94" operation="running" operation_key="openstack-cinder-volume_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="91" operation="start" operation_key="openstack-cinder-volume-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="48">
++ <action_set>
++ <pseudo_event id="93" operation="start" operation_key="openstack-cinder-volume_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="49" priority="1000000">
++ <action_set>
++ <pseudo_event id="88" operation="running" operation_key="haproxy-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="85" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="50">
++ <action_set>
++ <pseudo_event id="87" operation="start" operation_key="haproxy-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="73" operation="start" operation_key="ip-192.168.122.254_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="75" operation="start" operation_key="ip-192.168.122.250_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="77" operation="start" operation_key="ip-192.168.122.249_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="79" operation="start" operation_key="ip-192.168.122.253_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="81" operation="start" operation_key="ip-192.168.122.247_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="83" operation="start" operation_key="ip-192.168.122.248_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="51" priority="1000000">
++ <action_set>
++ <pseudo_event id="62" operation="running" operation_key="redis-bundle-master_running_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="58" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="61" operation="start" operation_key="redis-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="52">
++ <action_set>
++ <pseudo_event id="61" operation="start" operation_key="redis-bundle-master_start_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="54" operation="start" operation_key="redis-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="53" priority="1000000">
++ <action_set>
++ <pseudo_event id="55" operation="running" operation_key="redis-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="62" operation="running" operation_key="redis-bundle-master_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="54">
++ <action_set>
++ <pseudo_event id="54" operation="start" operation_key="redis-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="88" operation="running" operation_key="haproxy-bundle_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="55" priority="1000000">
++ <action_set>
++ <pseudo_event id="39" operation="running" operation_key="galera-bundle-master_running_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="35" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="38" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="56">
++ <action_set>
++ <pseudo_event id="38" operation="start" operation_key="galera-bundle-master_start_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="31" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="57" priority="1000000">
++ <action_set>
++ <pseudo_event id="32" operation="running" operation_key="galera-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="39" operation="running" operation_key="galera-bundle-master_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="58">
++ <action_set>
++ <pseudo_event id="31" operation="start" operation_key="galera-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="55" operation="running" operation_key="redis-bundle_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="59" priority="1000000">
++ <action_set>
++ <pseudo_event id="24" operation="running" operation_key="rabbitmq-bundle-clone_running_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="21" operation="start" operation_key="rabbitmq:0_start_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="23" operation="start" operation_key="rabbitmq-bundle-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="60">
++ <action_set>
++ <pseudo_event id="23" operation="start" operation_key="rabbitmq-bundle-clone_start_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="17" operation="start" operation_key="rabbitmq-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="61" priority="1000000">
++ <action_set>
++ <pseudo_event id="18" operation="running" operation_key="rabbitmq-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="24" operation="running" operation_key="rabbitmq-bundle-clone_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="62">
++ <action_set>
++ <pseudo_event id="17" operation="start" operation_key="rabbitmq-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/bundle-order-startup.scores b/pengine/test10/bundle-order-startup.scores
+new file mode 100644
+index 0000000..565b47e
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup.scores
+@@ -0,0 +1,197 @@
++Allocation scores:
++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:0 allocation score on undercloud: -INFINITY
++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY
++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: redis:0 allocation score on undercloud: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on undercloud: 0
++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on undercloud: 0
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on undercloud: 0
++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on undercloud: 0
++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera:0 allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on undercloud: 0
++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: 0
++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle allocation score on undercloud: 0
++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-0 allocation score on undercloud: 0
++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0
++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: 0
++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 0
++container_color: rabbitmq:0 allocation score on undercloud: 0
++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on undercloud: 0
++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on undercloud: 0
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on undercloud: 0
++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on undercloud: 0
++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:0 allocation score on undercloud: 0
++galera:0 promotion score on galera-bundle-0: -1
++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on undercloud: 10000
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on undercloud: 0
++native_color: galera:0 allocation score on galera-bundle-0: INFINITY
++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:0 allocation score on undercloud: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on undercloud: 0
++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on undercloud: 0
++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on undercloud: 0
++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on undercloud: 0
++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on undercloud: 0
++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on undercloud: 0
++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on undercloud: 0
++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: 0
++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-0 allocation score on undercloud: 10000
++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: 0
++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on undercloud: 10000
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on undercloud: 0
++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: redis:0 allocation score on undercloud: -INFINITY
++redis:0 promotion score on redis-bundle-0: -1
+diff --git a/pengine/test10/bundle-order-startup.summary b/pengine/test10/bundle-order-startup.summary
+new file mode 100644
+index 0000000..b5c2091
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup.summary
+@@ -0,0 +1,126 @@
++
++Current cluster status:
++Online: [ undercloud ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Stopped
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Stopped
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Stopped
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Stopped
++
++Transition Summary:
++ * Start rabbitmq-bundle-docker-0 (undercloud)
++ * Start rabbitmq-bundle-0 (undercloud)
++ * Start rabbitmq:0 (rabbitmq-bundle-0)
++ * Start galera-bundle-docker-0 (undercloud)
++ * Start galera-bundle-0 (undercloud)
++ * Start galera:0 (galera-bundle-0)
++ * Start redis-bundle-docker-0 (undercloud)
++ * Start redis-bundle-0 (undercloud)
++ * Start redis:0 (redis-bundle-0)
++ * Start ip-192.168.122.254 (undercloud)
++ * Start ip-192.168.122.250 (undercloud)
++ * Start ip-192.168.122.249 (undercloud)
++ * Start ip-192.168.122.253 (undercloud)
++ * Start ip-192.168.122.247 (undercloud)
++ * Start ip-192.168.122.248 (undercloud)
++ * Start haproxy-bundle-docker-0 (undercloud)
++ * Start openstack-cinder-volume-docker-0 (undercloud)
++
++Executing cluster transition:
++ * Resource action: rabbitmq-bundle-docker-0 monitor on undercloud
++ * Resource action: galera-bundle-docker-0 monitor on undercloud
++ * Resource action: redis-bundle-docker-0 monitor on undercloud
++ * Resource action: ip-192.168.122.254 monitor on undercloud
++ * Resource action: ip-192.168.122.250 monitor on undercloud
++ * Resource action: ip-192.168.122.249 monitor on undercloud
++ * Resource action: ip-192.168.122.253 monitor on undercloud
++ * Resource action: ip-192.168.122.247 monitor on undercloud
++ * Resource action: ip-192.168.122.248 monitor on undercloud
++ * Resource action: haproxy-bundle-docker-0 monitor on undercloud
++ * Resource action: openstack-cinder-volume-docker-0 monitor on undercloud
++ * Pseudo action: openstack-cinder-volume_start_0
++ * Pseudo action: rabbitmq-bundle_start_0
++ * Resource action: rabbitmq-bundle-docker-0 start on undercloud
++ * Resource action: rabbitmq-bundle-0 start on undercloud
++ * Resource action: ip-192.168.122.254 start on undercloud
++ * Resource action: ip-192.168.122.250 start on undercloud
++ * Resource action: ip-192.168.122.249 start on undercloud
++ * Resource action: ip-192.168.122.253 start on undercloud
++ * Resource action: ip-192.168.122.247 start on undercloud
++ * Resource action: ip-192.168.122.248 start on undercloud
++ * Resource action: openstack-cinder-volume-docker-0 start on undercloud
++ * Pseudo action: openstack-cinder-volume_running_0
++ * Pseudo action: haproxy-bundle_start_0
++ * Pseudo action: rabbitmq-bundle-clone_start_0
++ * Resource action: rabbitmq:0 start on rabbitmq-bundle-0
++ * Resource action: rabbitmq-bundle-docker-0 monitor=60000 on undercloud
++ * Resource action: rabbitmq-bundle-0 monitor=60000 on undercloud
++ * Resource action: ip-192.168.122.254 monitor=10000 on undercloud
++ * Resource action: ip-192.168.122.250 monitor=10000 on undercloud
++ * Resource action: ip-192.168.122.249 monitor=10000 on undercloud
++ * Resource action: ip-192.168.122.253 monitor=10000 on undercloud
++ * Resource action: ip-192.168.122.247 monitor=10000 on undercloud
++ * Resource action: ip-192.168.122.248 monitor=10000 on undercloud
++ * Resource action: haproxy-bundle-docker-0 start on undercloud
++ * Resource action: openstack-cinder-volume-docker-0 monitor=60000 on undercloud
++ * Pseudo action: haproxy-bundle_running_0
++ * Pseudo action: redis-bundle_start_0
++ * Pseudo action: rabbitmq-bundle-clone_running_0
++ * Pseudo action: rabbitmq-bundle_running_0
++ * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0
++ * Resource action: redis-bundle-docker-0 start on undercloud
++ * Resource action: redis-bundle-0 start on undercloud
++ * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud
++ * Pseudo action: redis-bundle-master_start_0
++ * Resource action: redis:0 start on redis-bundle-0
++ * Resource action: redis-bundle-docker-0 monitor=60000 on undercloud
++ * Resource action: redis-bundle-0 monitor=60000 on undercloud
++ * Pseudo action: redis-bundle-master_running_0
++ * Pseudo action: redis-bundle_running_0
++ * Pseudo action: galera-bundle_start_0
++ * Resource action: galera-bundle-docker-0 start on undercloud
++ * Resource action: galera-bundle-0 start on undercloud
++ * Resource action: redis:0 monitor=60000 on redis-bundle-0
++ * Resource action: redis:0 monitor=45000 on redis-bundle-0
++ * Pseudo action: galera-bundle-master_start_0
++ * Resource action: galera:0 start on galera-bundle-0
++ * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud
++ * Resource action: galera-bundle-0 monitor=60000 on undercloud
++ * Pseudo action: galera-bundle-master_running_0
++ * Pseudo action: galera-bundle_running_0
++ * Resource action: galera:0 monitor=30000 on galera-bundle-0
++ * Resource action: galera:0 monitor=20000 on galera-bundle-0
++
++Revised cluster status:
++Online: [ undercloud ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Slave undercloud
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Slave undercloud
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud
++
+diff --git a/pengine/test10/bundle-order-startup.xml b/pengine/test10/bundle-order-startup.xml
+new file mode 100644
+index 0000000..3e2e8f6
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup.xml
+@@ -0,0 +1,315 @@
++<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="undercloud">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
++ <instance_attributes id="nodes-rabbitmq-bundle-0">
++ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@undercloud"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <bundle id="rabbitmq-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3121"/>
++ <storage>
++ <storage-mapping id="rabbitmq-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/rabbitmq.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="rabbitmq-cfg-data" options="ro" source-dir="/var/lib/config-data/rabbitmq/etc/rabbitmq" target-dir="/etc/rabbitmq"/>
++ <storage-mapping id="rabbitmq-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="rabbitmq-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="rabbitmq-lib" options="rw" source-dir="/var/lib/rabbitmq" target-dir="/var/lib/rabbitmq"/>
++ <storage-mapping id="rabbitmq-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="rabbitmq-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="rabbitmq-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster">
++ <instance_attributes id="rabbitmq-instance_attributes">
++ <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </instance_attributes>
++ <meta_attributes id="rabbitmq-meta_attributes">
++ <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="rabbitmq-monitor-interval-10" interval="10" name="monitor" timeout="40"/>
++ <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="galera-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
++ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
++ <instance_attributes id="galera-instance_attributes">
++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:undercloud"/>
++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://undercloud"/>
++ </instance_attributes>
++ <meta_attributes id="galera-meta_attributes">
++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
++ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
++ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="redis-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3124"/>
++ <storage>
++ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
++ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
++ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
++ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
++ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
++ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
++ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
++ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
++ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
++ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <primitive class="ocf" id="ip-192.168.122.254" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.254-instance_attributes">
++ <nvpair id="ip-192.168.122.254-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.254-instance_attributes-ip" name="ip" value="192.168.122.254"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.254-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.254-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.254-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.250" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.250-instance_attributes">
++ <nvpair id="ip-192.168.122.250-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.250-instance_attributes-ip" name="ip" value="192.168.122.250"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.250-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.250-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.250-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.249" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.249-instance_attributes">
++ <nvpair id="ip-192.168.122.249-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.249-instance_attributes-ip" name="ip" value="192.168.122.249"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.249-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.249-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.249-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.253" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.253-instance_attributes">
++ <nvpair id="ip-192.168.122.253-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.253-instance_attributes-ip" name="ip" value="192.168.122.253"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.253-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.253-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.253-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.247" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.247-instance_attributes">
++ <nvpair id="ip-192.168.122.247-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.247-instance_attributes-ip" name="ip" value="192.168.122.247"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.247-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.247-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.247-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.248" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.248-instance_attributes">
++ <nvpair id="ip-192.168.122.248-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.248-instance_attributes-ip" name="ip" value="192.168.122.248"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.248-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.248-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.248-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ </bundle>
++ <bundle id="openstack-cinder-volume">
++ <docker run-command="/bin/bash /usr/local/bin/kolla_start" network="host" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" options="--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1"/>
++ <storage>
++ <storage-mapping target-dir="/var/lib/kolla/config_files/config.json" options="ro" id="cinder-volume-cfg-files" source-dir="/var/lib/kolla/config_files/cinder_volume.json"/>
++ <storage-mapping target-dir="/etc/cinder" options="ro" id="cinder-volume-cfg-data" source-dir="/var/lib/config-data/cinder/etc/cinder"/>
++ <storage-mapping target-dir="/etc/hosts" options="ro" id="cinder-volume-hosts" source-dir="/etc/hosts"/>
++ <storage-mapping target-dir="/etc/localtime" options="ro" id="cinder-volume-localtime" source-dir="/etc/localtime"/>
++ <storage-mapping target-dir="/dev" options="rw" id="cinder-volume-dev" source-dir="/dev"/>
++ <storage-mapping target-dir="/run" options="rw" id="cinder-volume-run" source-dir="/run"/>
++ <storage-mapping target-dir="/sys" options="rw" id="cinder-volume-sys" source-dir="/sys"/>
++ <storage-mapping target-dir="/lib/modules" options="ro" id="cinder-lib-modules" source-dir="/lib/modules"/>
++ <storage-mapping target-dir="/etc/iscsi" options="rw" id="cinder-volume-iscsi" source-dir="/etc/iscsi"/>
++ <storage-mapping target-dir="/var/lib/cinder" options="rw" id="cinder-volume-var-lib-cinder" source-dir="/var/lib/cinder"/>
++ <storage-mapping target-dir="/var/log/cinder" options="rw" id="cinder-volume-var-log" source-dir="/var/log/containers/cinder"/>
++ </storage>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-rabbitmq-bundle" resource-discovery="exclusive" rsc="rabbitmq-bundle">
++ <rule id="location-rabbitmq-bundle-rule" score="0">
++ <expression attribute="rabbitmq-role" id="location-rabbitmq-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
++ <rule id="location-galera-bundle-rule" score="0">
++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
++ <rule id="location-redis-bundle-rule" score="0">
++ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.254" resource-discovery="exclusive" rsc="ip-192.168.122.254">
++ <rule id="location-ip-192.168.122.254-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.254-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.250" resource-discovery="exclusive" rsc="ip-192.168.122.250">
++ <rule id="location-ip-192.168.122.250-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.250-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.249" resource-discovery="exclusive" rsc="ip-192.168.122.249">
++ <rule id="location-ip-192.168.122.249-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.249-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.253" resource-discovery="exclusive" rsc="ip-192.168.122.253">
++ <rule id="location-ip-192.168.122.253-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.253-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.247" resource-discovery="exclusive" rsc="ip-192.168.122.247">
++ <rule id="location-ip-192.168.122.247-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.247-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.248" resource-discovery="exclusive" rsc="ip-192.168.122.248">
++ <rule id="location-ip-192.168.122.248-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.248-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_order first="ip-192.168.122.254" first-action="start" id="order-ip-192.168.122.254-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.254-haproxy-bundle-INFINITY" rsc="ip-192.168.122.254" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.250" first-action="start" id="order-ip-192.168.122.250-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.250-haproxy-bundle-INFINITY" rsc="ip-192.168.122.250" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.249" first-action="start" id="order-ip-192.168.122.249-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.249-haproxy-bundle-INFINITY" rsc="ip-192.168.122.249" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.253" first-action="start" id="order-ip-192.168.122.253-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.253-haproxy-bundle-INFINITY" rsc="ip-192.168.122.253" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.247" first-action="start" id="order-ip-192.168.122.247-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.247-haproxy-bundle-INFINITY" rsc="ip-192.168.122.247" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.248" first-action="start" id="order-ip-192.168.122.248-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.248-haproxy-bundle-INFINITY" rsc="ip-192.168.122.248" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_colocation id="colocation-rabbit-with-haproxy-bundle-INFINITY" rsc="rabbitmq-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="haproxy-bundle" first-action="start" id="order-1" kind="Optional" then="redis-bundle" then-action="start"/>
++ <rsc_order first="redis-bundle" first-action="start" id="order-2" kind="Optional" then="galera-bundle" then-action="start"/>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="undercloud" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <transient_attributes id="1">
++ <instance_attributes id="status-1">
++ <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
++ <nvpair id="status-1-fail-count-openstack-cinder-volume-docker-0" name="fail-count-openstack-cinder-volume-docker-0" value="21"/>
++ <nvpair id="status-1-last-failure-openstack-cinder-volume-docker-0" name="last-failure-openstack-cinder-volume-docker-0" value="1496305247"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ </status>
++</cib>
+diff --git a/pengine/test10/bundle-order-stop.dot b/pengine/test10/bundle-order-stop.dot
+new file mode 100644
+index 0000000..235b634
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop.dot
+@@ -0,0 +1,194 @@
++digraph "g" {
++"Cancel galera_monitor_10000 galera-bundle-0" -> "galera_demote_0 galera-bundle-0" [ style = bold]
++"Cancel galera_monitor_10000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"Cancel redis_monitor_20000 redis-bundle-0" -> "redis_demote_0 redis-bundle-0" [ style = bold]
++"Cancel redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"all_stopped" [ style=bold color="green" fontcolor="orange"]
++"do_shutdown undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"galera-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"galera-bundle-0_stop_0 undercloud" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold]
++"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold]
++"galera-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_demoted_0" -> "galera-bundle-master_start_0" [ style = dashed]
++"galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold]
++"galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold]
++"galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed]
++"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed]
++"galera-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold]
++"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
++"galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = dashed]
++"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold]
++"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold]
++"galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold]
++"galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold]
++"galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold]
++"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
++"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_stopped_0" -> "redis-bundle_stop_0" [ style = bold]
++"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold]
++"galera_demote_0 galera-bundle-0" -> "galera-bundle-master_demoted_0" [ style = bold]
++"galera_demote_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed]
++"galera_demote_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed]
++"galera_demote_0 galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
++"galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed]
++"galera_start_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed]
++"galera_start_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed]
++"galera_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera_stop_0 galera-bundle-0" -> "all_stopped" [ style = bold]
++"galera_stop_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold]
++"galera_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold]
++"galera_stop_0 galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = dashed]
++"galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"haproxy-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"haproxy-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle_stopped_0" [ style = bold]
++"haproxy-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle_stop_0" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.247_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.248_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.249_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.250_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.253_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" -> "ip-192.168.122.254_stop_0 undercloud" [ style = bold]
++"haproxy-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"ip-192.168.122.247_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.247_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.247_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.247_stop_0 undercloud" -> "ip-192.168.122.247_start_0 <none>" [ style = dashed]
++"ip-192.168.122.247_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.248_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.248_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.248_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.248_stop_0 undercloud" -> "ip-192.168.122.248_start_0 <none>" [ style = dashed]
++"ip-192.168.122.248_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.249_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.249_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.249_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.249_stop_0 undercloud" -> "ip-192.168.122.249_start_0 <none>" [ style = dashed]
++"ip-192.168.122.249_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.250_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.250_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.250_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.250_stop_0 undercloud" -> "ip-192.168.122.250_start_0 <none>" [ style = dashed]
++"ip-192.168.122.250_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.253_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.253_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.253_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.253_stop_0 undercloud" -> "ip-192.168.122.253_start_0 <none>" [ style = dashed]
++"ip-192.168.122.253_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"ip-192.168.122.254_start_0 <none>" [ style=dashed color="red" fontcolor="black"]
++"ip-192.168.122.254_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"ip-192.168.122.254_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"ip-192.168.122.254_stop_0 undercloud" -> "ip-192.168.122.254_start_0 <none>" [ style = dashed]
++"ip-192.168.122.254_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "openstack-cinder-volume_stopped_0" [ style = bold]
++"openstack-cinder-volume-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"openstack-cinder-volume_stop_0" -> "openstack-cinder-volume-docker-0_stop_0 undercloud" [ style = bold]
++"openstack-cinder-volume_stop_0" [ style=bold color="green" fontcolor="orange"]
++"openstack-cinder-volume_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed]
++"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq-bundle-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold]
++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed]
++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold]
++"rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold]
++"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
++"rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed]
++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "all_stopped" [ style = bold]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-0_stop_0 undercloud" [ style = bold]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold]
++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed]
++"rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"redis-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"redis-bundle-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold]
++"redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_demoted_0" -> "redis-bundle-master_start_0" [ style = dashed]
++"redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold]
++"redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold]
++"redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed]
++"redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed]
++"redis-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"]
++"redis-bundle-master_stop_0" -> "redis-bundle-master_stopped_0" [ style = bold]
++"redis-bundle-master_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_stopped_0" -> "redis-bundle-master_start_0" [ style = dashed]
++"redis-bundle-master_stopped_0" -> "redis-bundle_stopped_0" [ style = bold]
++"redis-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold]
++"redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold]
++"redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold]
++"redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold]
++"redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed]
++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold]
++"redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
++"redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_stopped_0" -> "haproxy-bundle_stop_0" [ style = bold]
++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold]
++"redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"redis_demote_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold]
++"redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold]
++"redis_demote_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed]
++"redis_demote_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed]
++"redis_demote_0 redis-bundle-0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
++"redis_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis_monitor_45000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"redis_monitor_60000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"redis_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = dashed]
++"redis_start_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed]
++"redis_start_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed]
++"redis_start_0 redis-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"redis_stop_0 redis-bundle-0" -> "all_stopped" [ style = bold]
++"redis_stop_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold]
++"redis_stop_0 redis-bundle-0" -> "redis-bundle-master_stopped_0" [ style = bold]
++"redis_stop_0 redis-bundle-0" -> "redis_start_0 redis-bundle-0" [ style = dashed]
++"redis_stop_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++}
+diff --git a/pengine/test10/bundle-order-stop.exp b/pengine/test10/bundle-order-stop.exp
+new file mode 100644
+index 0000000..d085433
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop.exp
+@@ -0,0 +1,734 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0">
++ <action_set>
++ <rsc_op id="25" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud">
++ <primitive id="rabbitmq" long-id="rabbitmq:0" class="ocf" provider="heartbeat" type="rabbitmq-cluster"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="rabbitmq-bundle-0" CRM_meta_on_node_uuid="rabbitmq-bundle-0" CRM_meta_timeout="200000" set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="19" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/rabbitmq-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/rabbitmq/etc/rabbitmq:/etc/rabbitmq:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/rabbitmq:/var/lib/rabbitmq:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/rabbitmq-bundle-0:/var/log -p 3121:3121 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="2">
++ <action_set>
++ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="rabbitmq-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="rabbitmq-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="#uname" port="3121"/>
++ <downed>
++ <node id="rabbitmq-bundle-0"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="25" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="39" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="38" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="44" operation="stop" operation_key="galera-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="4">
++ <action_set>
++ <rsc_op id="38" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="16" operation="cancel" operation_key="galera_monitor_10000" internal_operation_key="galera:0_monitor_10000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="48" operation="demote" operation_key="galera-bundle-master_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="16" operation="cancel" operation_key="galera_monitor_10000" internal_operation_key="galera:0_monitor_10000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_operation="monitor" CRM_meta_role="Master" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:undercloud" enable_creation="true" wsrep_cluster_address="gcomm://undercloud"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="6">
++ <action_set>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="#uname" port="3123"/>
++ <downed>
++ <node id="galera-bundle-0"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="38" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="39" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="8">
++ <action_set>
++ <rsc_op id="62" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="200000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="58" operation="stop" operation_key="redis-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="61" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="67" operation="stop" operation_key="redis-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <rsc_op id="61" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="demote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="120000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="17" operation="cancel" operation_key="redis_monitor_20000" internal_operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="71" operation="demote" operation_key="redis-bundle-master_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="10">
++ <action_set>
++ <rsc_op id="17" operation="cancel" operation_key="redis_monitor_20000" internal_operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_operation="monitor" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="12">
++ <action_set>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="#uname" port="3124"/>
++ <downed>
++ <node id="redis-bundle-0"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="61" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="62" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13">
++ <action_set>
++ <rsc_op id="77" operation="stop" operation_key="ip-192.168.122.254_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.254" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.254"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14">
++ <action_set>
++ <rsc_op id="78" operation="stop" operation_key="ip-192.168.122.250_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.250" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.250"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15">
++ <action_set>
++ <rsc_op id="79" operation="stop" operation_key="ip-192.168.122.249_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.249" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.249"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16">
++ <action_set>
++ <rsc_op id="80" operation="stop" operation_key="ip-192.168.122.253_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.253" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.253"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="17">
++ <action_set>
++ <rsc_op id="81" operation="stop" operation_key="ip-192.168.122.247_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.247" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.247"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="18">
++ <action_set>
++ <rsc_op id="82" operation="stop" operation_key="ip-192.168.122.248_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="ip-192.168.122.248" class="ocf" provider="heartbeat" type="IPaddr2"/>
++ <attributes CRM_meta_name="stop" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" cidr_netmask="32" ip="192.168.122.248"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="19">
++ <action_set>
++ <rsc_op id="83" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="86" operation="stop" operation_key="haproxy-bundle_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20">
++ <action_set>
++ <rsc_op id="88" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="openstack-cinder-volume-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/cinder/etc/cinder:/etc/cinder:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /dev:/dev:rw -v /run:/run:rw -v /sys:/sys:rw -v /lib/modules:/lib/modules:ro -v /etc/iscsi:/etc/iscsi:rw -v /var/lib/cinder:/var/lib/cinder:rw -v /var/log/containers/cinder:/var/log/cinder:rw --ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="91" operation="stop" operation_key="openstack-cinder-volume_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="21">
++ <action_set>
++ <crm_event id="94" operation="do_shutdown" operation_key="do_shutdown-undercloud" on_node="undercloud" on_node_uuid="1">
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_op_no_wait="true" />
++ <downed>
++ <node id="1"/>
++ </downed>
++ </crm_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="19" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="77" operation="stop" operation_key="ip-192.168.122.254_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="78" operation="stop" operation_key="ip-192.168.122.250_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="79" operation="stop" operation_key="ip-192.168.122.249_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="80" operation="stop" operation_key="ip-192.168.122.253_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="81" operation="stop" operation_key="ip-192.168.122.247_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="82" operation="stop" operation_key="ip-192.168.122.248_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="83" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="88" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="22" priority="1000000">
++ <action_set>
++ <pseudo_event id="92" operation="stopped" operation_key="openstack-cinder-volume_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="88" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="23">
++ <action_set>
++ <pseudo_event id="91" operation="stop" operation_key="openstack-cinder-volume_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="24" priority="1000000">
++ <action_set>
++ <pseudo_event id="87" operation="stopped" operation_key="haproxy-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="83" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="25">
++ <action_set>
++ <pseudo_event id="86" operation="stop" operation_key="haproxy-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="59" operation="stopped" operation_key="redis-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="26" priority="1000000">
++ <action_set>
++ <pseudo_event id="76" operation="demoted" operation_key="redis-bundle_demoted_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="72" operation="demoted" operation_key="redis-bundle-master_demoted_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="75" operation="demote" operation_key="redis-bundle_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="27">
++ <action_set>
++ <pseudo_event id="75" operation="demote" operation_key="redis-bundle_demote_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="28" priority="1000000">
++ <action_set>
++ <pseudo_event id="72" operation="demoted" operation_key="redis-bundle-master_demoted_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="61" operation="demote" operation_key="redis_demote_0" internal_operation_key="redis:0_demote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="71" operation="demote" operation_key="redis-bundle-master_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="29">
++ <action_set>
++ <pseudo_event id="71" operation="demote" operation_key="redis-bundle-master_demote_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="75" operation="demote" operation_key="redis-bundle_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="30" priority="1000000">
++ <action_set>
++ <pseudo_event id="68" operation="stopped" operation_key="redis-bundle-master_stopped_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="62" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="67" operation="stop" operation_key="redis-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="31">
++ <action_set>
++ <pseudo_event id="67" operation="stop" operation_key="redis-bundle-master_stop_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="58" operation="stop" operation_key="redis-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="72" operation="demoted" operation_key="redis-bundle-master_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="32" priority="1000000">
++ <action_set>
++ <pseudo_event id="59" operation="stopped" operation_key="redis-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="68" operation="stopped" operation_key="redis-bundle-master_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="33">
++ <action_set>
++ <pseudo_event id="58" operation="stop" operation_key="redis-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="36" operation="stopped" operation_key="galera-bundle_stopped_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="76" operation="demoted" operation_key="redis-bundle_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="34">
++ <action_set>
++ <pseudo_event id="56" operation="start" operation_key="redis-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="59" operation="stopped" operation_key="redis-bundle_stopped_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="76" operation="demoted" operation_key="redis-bundle_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="35" priority="1000000">
++ <action_set>
++ <pseudo_event id="53" operation="demoted" operation_key="galera-bundle_demoted_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="49" operation="demoted" operation_key="galera-bundle-master_demoted_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="52" operation="demote" operation_key="galera-bundle_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="36">
++ <action_set>
++ <pseudo_event id="52" operation="demote" operation_key="galera-bundle_demote_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="37" priority="1000000">
++ <action_set>
++ <pseudo_event id="49" operation="demoted" operation_key="galera-bundle-master_demoted_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="38" operation="demote" operation_key="galera_demote_0" internal_operation_key="galera:0_demote_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="48" operation="demote" operation_key="galera-bundle-master_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="38">
++ <action_set>
++ <pseudo_event id="48" operation="demote" operation_key="galera-bundle-master_demote_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="52" operation="demote" operation_key="galera-bundle_demote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="39" priority="1000000">
++ <action_set>
++ <pseudo_event id="45" operation="stopped" operation_key="galera-bundle-master_stopped_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="39" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="44" operation="stop" operation_key="galera-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="40">
++ <action_set>
++ <pseudo_event id="44" operation="stop" operation_key="galera-bundle-master_stop_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="49" operation="demoted" operation_key="galera-bundle-master_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="41" priority="1000000">
++ <action_set>
++ <pseudo_event id="36" operation="stopped" operation_key="galera-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="45" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="42">
++ <action_set>
++ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="53" operation="demoted" operation_key="galera-bundle_demoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="43" priority="1000000">
++ <action_set>
++ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="25" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="44">
++ <action_set>
++ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="45" priority="1000000">
++ <action_set>
++ <pseudo_event id="24" operation="stopped" operation_key="rabbitmq-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="46">
++ <action_set>
++ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="47">
++ <action_set>
++ <pseudo_event id="18" operation="all_stopped" operation_key="all_stopped">
++ <attributes />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="19" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="25" operation="stop" operation_key="rabbitmq_stop_0" internal_operation_key="rabbitmq:0_stop_0" on_node="rabbitmq-bundle-0" on_node_uuid="rabbitmq-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="39" operation="stop" operation_key="galera_stop_0" internal_operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="62" operation="stop" operation_key="redis_stop_0" internal_operation_key="redis:0_stop_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="77" operation="stop" operation_key="ip-192.168.122.254_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="78" operation="stop" operation_key="ip-192.168.122.250_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="79" operation="stop" operation_key="ip-192.168.122.249_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="80" operation="stop" operation_key="ip-192.168.122.253_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="81" operation="stop" operation_key="ip-192.168.122.247_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="82" operation="stop" operation_key="ip-192.168.122.248_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="83" operation="stop" operation_key="haproxy-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="88" operation="stop" operation_key="openstack-cinder-volume-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/bundle-order-stop.scores b/pengine/test10/bundle-order-stop.scores
+new file mode 100644
+index 0000000..a662f42
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop.scores
+@@ -0,0 +1,199 @@
++Allocation scores:
++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:0 allocation score on undercloud: -INFINITY
++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY
++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: redis:0 allocation score on undercloud: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on undercloud: 0
++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on undercloud: INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on undercloud: 0
++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera:0 allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle allocation score on undercloud: 0
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume allocation score on undercloud: 0
++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle allocation score on undercloud: 0
++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0
++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0
++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++container_color: rabbitmq:0 allocation score on undercloud: 0
++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on undercloud: 0
++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on undercloud: 0
++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:0 allocation score on undercloud: 0
++galera:0 promotion score on galera-bundle-0: 100
++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on undercloud: INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on undercloud: -INFINITY
++native_color: galera:0 allocation score on galera-bundle-0: INFINITY
++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:0 allocation score on undercloud: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.247 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.248 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.249 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.250 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.253 allocation score on undercloud: -INFINITY
++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY
++native_color: ip-192.168.122.254 allocation score on undercloud: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: -INFINITY
++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: -INFINITY
++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY
++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on undercloud: INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on undercloud: -INFINITY
++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: redis:0 allocation score on undercloud: -INFINITY
++redis:0 promotion score on redis-bundle-0: 1
+diff --git a/pengine/test10/bundle-order-stop.summary b/pengine/test10/bundle-order-stop.summary
+new file mode 100644
+index 0000000..bd6f937
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop.summary
+@@ -0,0 +1,114 @@
++
++Current cluster status:
++Online: [ undercloud ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Master undercloud
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud
++
++Transition Summary:
++ * Shutdown undercloud
++ * Stop rabbitmq-bundle-docker-0 (undercloud)
++ * Stop rabbitmq-bundle-0 (undercloud)
++ * Stop rabbitmq:0 (Started rabbitmq-bundle-0)
++ * Stop galera-bundle-docker-0 (undercloud)
++ * Stop galera-bundle-0 (undercloud)
++ * Demote galera:0 (Master -> Slave galera-bundle-0)
++ * Restart galera:0 (Slave galera-bundle-0)
++ * Stop redis-bundle-docker-0 (undercloud)
++ * Stop redis-bundle-0 (undercloud)
++ * Demote redis:0 (Master -> Slave redis-bundle-0)
++ * Restart redis:0 (Slave redis-bundle-0)
++ * Stop ip-192.168.122.254 (undercloud)
++ * Stop ip-192.168.122.250 (undercloud)
++ * Stop ip-192.168.122.249 (undercloud)
++ * Stop ip-192.168.122.253 (undercloud)
++ * Stop ip-192.168.122.247 (undercloud)
++ * Stop ip-192.168.122.248 (undercloud)
++ * Stop haproxy-bundle-docker-0 (undercloud)
++ * Stop openstack-cinder-volume-docker-0 (undercloud)
++
++Executing cluster transition:
++ * Resource action: galera cancel=10000 on galera-bundle-0
++ * Resource action: redis cancel=20000 on redis-bundle-0
++ * Pseudo action: openstack-cinder-volume_stop_0
++ * Pseudo action: redis-bundle_demote_0
++ * Pseudo action: redis-bundle-master_demote_0
++ * Pseudo action: galera-bundle_demote_0
++ * Pseudo action: galera-bundle-master_demote_0
++ * Pseudo action: rabbitmq-bundle_stop_0
++ * Resource action: galera demote on galera-bundle-0
++ * Resource action: redis demote on redis-bundle-0
++ * Resource action: openstack-cinder-volume-docker-0 stop on undercloud
++ * Pseudo action: openstack-cinder-volume_stopped_0
++ * Pseudo action: redis-bundle-master_demoted_0
++ * Pseudo action: galera-bundle-master_demoted_0
++ * Pseudo action: rabbitmq-bundle-clone_stop_0
++ * Resource action: rabbitmq stop on rabbitmq-bundle-0
++ * Resource action: rabbitmq-bundle-0 stop on undercloud
++ * Pseudo action: redis-bundle_demoted_0
++ * Pseudo action: galera-bundle_demoted_0
++ * Pseudo action: galera-bundle_stop_0
++ * Pseudo action: rabbitmq-bundle-clone_stopped_0
++ * Pseudo action: rabbitmq-bundle_stopped_0
++ * Resource action: rabbitmq-bundle-docker-0 stop on undercloud
++ * Pseudo action: galera-bundle-master_stop_0
++ * Resource action: galera stop on galera-bundle-0
++ * Resource action: galera-bundle-0 stop on undercloud
++ * Pseudo action: galera-bundle-master_stopped_0
++ * Pseudo action: galera-bundle_stopped_0
++ * Resource action: galera-bundle-docker-0 stop on undercloud
++ * Pseudo action: redis-bundle_stop_0
++ * Pseudo action: redis-bundle-master_stop_0
++ * Resource action: redis stop on redis-bundle-0
++ * Resource action: redis-bundle-0 stop on undercloud
++ * Pseudo action: redis-bundle-master_stopped_0
++ * Pseudo action: redis-bundle_stopped_0
++ * Pseudo action: redis-bundle_start_0
++ * Resource action: redis-bundle-docker-0 stop on undercloud
++ * Pseudo action: haproxy-bundle_stop_0
++ * Resource action: haproxy-bundle-docker-0 stop on undercloud
++ * Pseudo action: haproxy-bundle_stopped_0
++ * Resource action: ip-192.168.122.254 stop on undercloud
++ * Resource action: ip-192.168.122.250 stop on undercloud
++ * Resource action: ip-192.168.122.249 stop on undercloud
++ * Resource action: ip-192.168.122.253 stop on undercloud
++ * Resource action: ip-192.168.122.247 stop on undercloud
++ * Resource action: ip-192.168.122.248 stop on undercloud
++ * Cluster action: do_shutdown on undercloud
++ * Pseudo action: all_stopped
++
++Revised cluster status:
++Online: [ undercloud ]
++
++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest]
++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Stopped
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Stopped
++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Stopped
++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Stopped
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest]
++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Stopped
++
+diff --git a/pengine/test10/bundle-order-stop.xml b/pengine/test10/bundle-order-stop.xml
+new file mode 100644
+index 0000000..467082a
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop.xml
+@@ -0,0 +1,421 @@
++<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="undercloud">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
++ <instance_attributes id="nodes-rabbitmq-bundle-0">
++ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@undercloud"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <bundle id="rabbitmq-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3121"/>
++ <storage>
++ <storage-mapping id="rabbitmq-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/rabbitmq.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="rabbitmq-cfg-data" options="ro" source-dir="/var/lib/config-data/rabbitmq/etc/rabbitmq" target-dir="/etc/rabbitmq"/>
++ <storage-mapping id="rabbitmq-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="rabbitmq-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="rabbitmq-lib" options="rw" source-dir="/var/lib/rabbitmq" target-dir="/var/lib/rabbitmq"/>
++ <storage-mapping id="rabbitmq-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="rabbitmq-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="rabbitmq-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="rabbitmq-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="rabbitmq" provider="heartbeat" type="rabbitmq-cluster">
++ <instance_attributes id="rabbitmq-instance_attributes">
++ <nvpair id="rabbitmq-instance_attributes-set_policy" name="set_policy" value="ha-all ^(?!amq\.).* {"ha-mode":"all"}"/>
++ </instance_attributes>
++ <meta_attributes id="rabbitmq-meta_attributes">
++ <nvpair id="rabbitmq-meta_attributes-notify" name="notify" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="rabbitmq-monitor-interval-10" interval="10" name="monitor" timeout="40"/>
++ <op id="rabbitmq-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="rabbitmq-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="galera-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
++ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
++ <instance_attributes id="galera-instance_attributes">
++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:undercloud"/>
++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://undercloud"/>
++ </instance_attributes>
++ <meta_attributes id="galera-meta_attributes">
++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
++ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
++ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="redis-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3124"/>
++ <storage>
++ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
++ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
++ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
++ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
++ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
++ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
++ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
++ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
++ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
++ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <primitive class="ocf" id="ip-192.168.122.254" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.254-instance_attributes">
++ <nvpair id="ip-192.168.122.254-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.254-instance_attributes-ip" name="ip" value="192.168.122.254"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.254-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.254-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.254-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.250" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.250-instance_attributes">
++ <nvpair id="ip-192.168.122.250-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.250-instance_attributes-ip" name="ip" value="192.168.122.250"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.250-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.250-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.250-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.249" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.249-instance_attributes">
++ <nvpair id="ip-192.168.122.249-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.249-instance_attributes-ip" name="ip" value="192.168.122.249"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.249-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.249-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.249-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.253" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.253-instance_attributes">
++ <nvpair id="ip-192.168.122.253-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.253-instance_attributes-ip" name="ip" value="192.168.122.253"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.253-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.253-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.253-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.247" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.247-instance_attributes">
++ <nvpair id="ip-192.168.122.247-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.247-instance_attributes-ip" name="ip" value="192.168.122.247"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.247-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.247-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.247-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <primitive class="ocf" id="ip-192.168.122.248" provider="heartbeat" type="IPaddr2">
++ <instance_attributes id="ip-192.168.122.248-instance_attributes">
++ <nvpair id="ip-192.168.122.248-instance_attributes-cidr_netmask" name="cidr_netmask" value="32"/>
++ <nvpair id="ip-192.168.122.248-instance_attributes-ip" name="ip" value="192.168.122.248"/>
++ </instance_attributes>
++ <operations>
++ <op id="ip-192.168.122.248-monitor-interval-10s" interval="10s" name="monitor" timeout="20s"/>
++ <op id="ip-192.168.122.248-start-interval-0s" interval="0s" name="start" timeout="20s"/>
++ <op id="ip-192.168.122.248-stop-interval-0s" interval="0s" name="stop" timeout="20s"/>
++ </operations>
++ </primitive>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ </bundle>
++ <bundle id="openstack-cinder-volume">
++ <docker run-command="/bin/bash /usr/local/bin/kolla_start" network="host" image="192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest" options="--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1"/>
++ <storage>
++ <storage-mapping target-dir="/var/lib/kolla/config_files/config.json" options="ro" id="cinder-volume-cfg-files" source-dir="/var/lib/kolla/config_files/cinder_volume.json"/>
++ <storage-mapping target-dir="/etc/cinder" options="ro" id="cinder-volume-cfg-data" source-dir="/var/lib/config-data/cinder/etc/cinder"/>
++ <storage-mapping target-dir="/etc/hosts" options="ro" id="cinder-volume-hosts" source-dir="/etc/hosts"/>
++ <storage-mapping target-dir="/etc/localtime" options="ro" id="cinder-volume-localtime" source-dir="/etc/localtime"/>
++ <storage-mapping target-dir="/dev" options="rw" id="cinder-volume-dev" source-dir="/dev"/>
++ <storage-mapping target-dir="/run" options="rw" id="cinder-volume-run" source-dir="/run"/>
++ <storage-mapping target-dir="/sys" options="rw" id="cinder-volume-sys" source-dir="/sys"/>
++ <storage-mapping target-dir="/lib/modules" options="ro" id="cinder-lib-modules" source-dir="/lib/modules"/>
++ <storage-mapping target-dir="/etc/iscsi" options="rw" id="cinder-volume-iscsi" source-dir="/etc/iscsi"/>
++ <storage-mapping target-dir="/var/lib/cinder" options="rw" id="cinder-volume-var-lib-cinder" source-dir="/var/lib/cinder"/>
++ <storage-mapping target-dir="/var/log/cinder" options="rw" id="cinder-volume-var-log" source-dir="/var/log/containers/cinder"/>
++ </storage>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-rabbitmq-bundle" resource-discovery="exclusive" rsc="rabbitmq-bundle">
++ <rule id="location-rabbitmq-bundle-rule" score="0">
++ <expression attribute="rabbitmq-role" id="location-rabbitmq-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
++ <rule id="location-galera-bundle-rule" score="0">
++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
++ <rule id="location-redis-bundle-rule" score="0">
++ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.254" resource-discovery="exclusive" rsc="ip-192.168.122.254">
++ <rule id="location-ip-192.168.122.254-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.254-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.250" resource-discovery="exclusive" rsc="ip-192.168.122.250">
++ <rule id="location-ip-192.168.122.250-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.250-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.249" resource-discovery="exclusive" rsc="ip-192.168.122.249">
++ <rule id="location-ip-192.168.122.249-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.249-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.253" resource-discovery="exclusive" rsc="ip-192.168.122.253">
++ <rule id="location-ip-192.168.122.253-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.253-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.247" resource-discovery="exclusive" rsc="ip-192.168.122.247">
++ <rule id="location-ip-192.168.122.247-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.247-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-ip-192.168.122.248" resource-discovery="exclusive" rsc="ip-192.168.122.248">
++ <rule id="location-ip-192.168.122.248-rule" score="0">
++ <expression attribute="haproxy-role" id="location-ip-192.168.122.248-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_order first="ip-192.168.122.254" first-action="start" id="order-ip-192.168.122.254-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.254-haproxy-bundle-INFINITY" rsc="ip-192.168.122.254" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.250" first-action="start" id="order-ip-192.168.122.250-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.250-haproxy-bundle-INFINITY" rsc="ip-192.168.122.250" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.249" first-action="start" id="order-ip-192.168.122.249-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.249-haproxy-bundle-INFINITY" rsc="ip-192.168.122.249" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.253" first-action="start" id="order-ip-192.168.122.253-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.253-haproxy-bundle-INFINITY" rsc="ip-192.168.122.253" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.247" first-action="start" id="order-ip-192.168.122.247-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.247-haproxy-bundle-INFINITY" rsc="ip-192.168.122.247" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="ip-192.168.122.248" first-action="start" id="order-ip-192.168.122.248-haproxy-bundle-Optional" kind="Optional" then="haproxy-bundle" then-action="start"/>
++ <rsc_colocation id="colocation-ip-192.168.122.248-haproxy-bundle-INFINITY" rsc="ip-192.168.122.248" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_colocation id="colocation-rabbit-with-haproxy-bundle-INFINITY" rsc="rabbitmq-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_order first="haproxy-bundle" first-action="start" id="order-1" kind="Optional" then="redis-bundle" then-action="start"/>
++ <rsc_order first="redis-bundle" first-action="start" id="order-2" kind="Optional" then="galera-bundle" then-action="start"/>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="undercloud" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="1">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_last_0" operation_key="rabbitmq-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;2:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="10" rc-code="0" op-status="0" interval="0" last-run="1496303575" last-rc-change="1496303575" exec-time="850" queue-time="0" op-digest="50baee2f0a77a08b0616f0cf07ab724b"/>
++ <lrm_rsc_op id="rabbitmq-bundle-docker-0_monitor_60000" operation_key="rabbitmq-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;3:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="11" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303576" exec-time="76" queue-time="0" op-digest="25cb53ea3977a912bf4c0f70fe9e3c21"/>
++ </lrm_resource>
++ <lrm_resource id="rabbitmq-bundle-0" type="remote" class="ocf" provider="pacemaker" container="rabbitmq-bundle-docker-0">
++ <lrm_rsc_op id="rabbitmq-bundle-0_last_0" operation_key="rabbitmq-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;4:6:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1496303576" last-rc-change="1496303576" exec-time="0" queue-time="0" op-digest="ee16760fa8c27e6585c1e35c244d7652" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="rabbitmq-bundle-0_monitor_60000" operation_key="rabbitmq-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;7:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303578" exec-time="0" queue-time="0" op-digest="92f557a57dfbb50be88282a4674ea681"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="16:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;16:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="17" rc-code="0" op-status="0" interval="0" last-run="1496303595" last-rc-change="1496303595" exec-time="920" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
++ <lrm_rsc_op id="galera-bundle-docker-0_monitor_60000" operation_key="galera-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="17:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;17:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="18" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303596" exec-time="70" queue-time="0" op-digest="f7e456f82e276e07f726e6e86e93bbb3"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-0" type="remote" class="ocf" provider="pacemaker" container="galera-bundle-docker-0">
++ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="18:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;18:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1496303596" last-rc-change="1496303596" exec-time="0" queue-time="0" op-digest="edf22e374195d93e82f3c9cc174d519f" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="galera-bundle-0_monitor_60000" operation_key="galera-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="20:9:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;20:9:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="4" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303600" exec-time="0" queue-time="0" op-digest="40c47455f6993c120f9a064ae51260db"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;34:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="28" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="824" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
++ <lrm_rsc_op id="redis-bundle-docker-0_monitor_60000" operation_key="redis-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;35:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="29" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303664" exec-time="74" queue-time="0" op-digest="81e9d490c493f8d22fab2822ec0bed8a"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-0" type="remote" class="ocf" provider="pacemaker" container="redis-bundle-docker-0">
++ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;36:15:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="5" rc-code="0" op-status="0" interval="0" last-run="1496303664" last-rc-change="1496303664" exec-time="0" queue-time="0" op-digest="9a6281aab223af75359e5b5a07161162" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
++ <lrm_rsc_op id="redis-bundle-0_monitor_60000" operation_key="redis-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;39:16:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="6" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303667" exec-time="0" queue-time="0" op-digest="04ffb214f519838e579c97e8aa002054"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.254" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.254_last_0" operation_key="ip-192.168.122.254_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;53:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="35" rc-code="0" op-status="0" interval="0" last-run="1496303683" last-rc-change="1496303683" exec-time="63" queue-time="0" op-digest="d6875d68a09b5550030b8b6b7cdc9294"/>
++ <lrm_rsc_op id="ip-192.168.122.254_monitor_10000" operation_key="ip-192.168.122.254_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;54:21:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="36" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303683" exec-time="32" queue-time="0" op-digest="5c1c6d0e79751e4002b12de513351a48"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.250" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.250_last_0" operation_key="ip-192.168.122.250_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;56:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="41" rc-code="0" op-status="0" interval="0" last-run="1496303696" last-rc-change="1496303696" exec-time="53" queue-time="0" op-digest="b33b44b3e9e535336108cb43f7474ef6"/>
++ <lrm_rsc_op id="ip-192.168.122.250_monitor_10000" operation_key="ip-192.168.122.250_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;57:24:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="42" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303696" exec-time="32" queue-time="0" op-digest="a4aec96753744d49ca9a798d3b0f12aa"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.249" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.249_last_0" operation_key="ip-192.168.122.249_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;59:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="47" rc-code="0" op-status="0" interval="0" last-run="1496303710" last-rc-change="1496303710" exec-time="52" queue-time="0" op-digest="22e58cb8beefb7810d8bf03d0d3a906d"/>
++ <lrm_rsc_op id="ip-192.168.122.249_monitor_10000" operation_key="ip-192.168.122.249_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;60:27:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="48" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303710" exec-time="30" queue-time="1" op-digest="860855adc72293b04b1becaeea73ced1"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.253" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.253_last_0" operation_key="ip-192.168.122.253_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;62:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="53" rc-code="0" op-status="0" interval="0" last-run="1496303723" last-rc-change="1496303723" exec-time="69" queue-time="0" op-digest="f36f4ccbd5f16e4e335c65f0802ddc20"/>
++ <lrm_rsc_op id="ip-192.168.122.253_monitor_10000" operation_key="ip-192.168.122.253_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;63:30:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="54" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303724" exec-time="39" queue-time="0" op-digest="0ee9a624c731239cdbef5c9118cc2bbe"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.247" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.247_last_0" operation_key="ip-192.168.122.247_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;65:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="59" rc-code="0" op-status="0" interval="0" last-run="1496303737" last-rc-change="1496303737" exec-time="57" queue-time="0" op-digest="37013238bd139c2b589a9508ff4b5c07"/>
++ <lrm_rsc_op id="ip-192.168.122.247_monitor_10000" operation_key="ip-192.168.122.247_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;66:33:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="60" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303737" exec-time="35" queue-time="0" op-digest="22cd67ba6dcb54e4f4e6ea3450a52b77"/>
++ </lrm_resource>
++ <lrm_resource id="ip-192.168.122.248" type="IPaddr2" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="ip-192.168.122.248_last_0" operation_key="ip-192.168.122.248_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;68:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="65" rc-code="0" op-status="0" interval="0" last-run="1496303750" last-rc-change="1496303750" exec-time="52" queue-time="0" op-digest="7565857faaf5a927e16322d5da3e4e38"/>
++ <lrm_rsc_op id="ip-192.168.122.248_monitor_10000" operation_key="ip-192.168.122.248_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;69:36:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="66" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303750" exec-time="30" queue-time="0" op-digest="2182cf0cd059930b67d43545b4950c6f"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="72:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;72:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="71" rc-code="0" op-status="0" interval="0" last-run="1496303754" last-rc-change="1496303754" exec-time="862" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ <lrm_rsc_op id="haproxy-bundle-docker-0_monitor_60000" operation_key="haproxy-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="73:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;73:37:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="72" rc-code="0" op-status="0" interval="60000" last-rc-change="1496303754" exec-time="64" queue-time="0" op-digest="8e98096b4e2923c7be3ce5fa2646c524"/>
++ </lrm_resource>
++ <lrm_resource id="openstack-cinder-volume-docker-0" type="docker" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_0" operation_key="openstack-cinder-volume-docker-0_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;76:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="161" rc-code="0" op-status="0" interval="0" last-run="1496305247" last-rc-change="1496305247" exec-time="838" queue-time="0" op-digest="3aa35269c96594499406635b536dcf19"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_monitor_60000" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;14:93:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="162" rc-code="0" op-status="0" interval="60000" last-rc-change="1496305248" exec-time="63" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ <lrm_rsc_op id="openstack-cinder-volume-docker-0_last_failure_0" operation_key="openstack-cinder-volume-docker-0_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:7;14:91:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="158" rc-code="7" op-status="0" interval="60000" last-rc-change="1496305247" exec-time="0" queue-time="0" op-digest="8bf789d58c162762d2d6f44701785901"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ <transient_attributes id="1">
++ <instance_attributes id="status-1">
++ <nvpair id="status-1-shutdown" name="shutdown" value="1000"/>
++ <nvpair id="status-1-fail-count-openstack-cinder-volume-docker-0" name="fail-count-openstack-cinder-volume-docker-0" value="21"/>
++ <nvpair id="status-1-last-failure-openstack-cinder-volume-docker-0" name="last-failure-openstack-cinder-volume-docker-0" value="1496305247"/>
++ </instance_attributes>
++ </transient_attributes>
++ </node_state>
++ <node_state remote_node="true" id="rabbitmq-bundle-0" uname="rabbitmq-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="rabbitmq-bundle-0">
++ <instance_attributes id="status-rabbitmq-bundle-0">
++ <nvpair id="status-rabbitmq-bundle-0-rmq-node-attr-rabbitmq" name="rmq-node-attr-rabbitmq" value="rabbit@undercloud"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="rabbitmq-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="rabbitmq" type="rabbitmq-cluster" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="rabbitmq_last_0" operation_key="rabbitmq_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="8:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;8:7:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="7" rc-code="0" op-status="0" interval="0" last-run="1496303578" last-rc-change="1496303578" exec-time="16270" queue-time="0" op-digest="780d433233eb4f94c1a151623d002e84"/>
++ <lrm_rsc_op id="rabbitmq_monitor_10000" operation_key="rabbitmq_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="11:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;11:8:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="49" rc-code="0" op-status="0" interval="10000" last-rc-change="1496303595" exec-time="5628" queue-time="0" op-digest="6b46cdf9111345cbd0460b2540d3b2c7"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="galera-bundle-0" uname="galera-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="galera-bundle-0">
++ <instance_attributes id="status-galera-bundle-0">
++ <nvpair id="status-galera-bundle-0-master-galera" name="master-galera" value="100"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="galera-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="galera" type="galera" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="galera_last_0" operation_key="galera_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="23:10:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;23:10:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="86" rc-code="0" op-status="0" interval="0" last-run="1496303611" last-rc-change="1496303611" exec-time="11500" queue-time="0" op-digest="d5f683936a1e6d67cbca8e284acab041" op-secure-params=" user " op-secure-digest="d5f683936a1e6d67cbca8e284acab041"/>
++ <lrm_rsc_op id="galera_monitor_10000" operation_key="galera_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="25:11:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:8;25:11:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="187" rc-code="8" op-status="0" interval="10000" last-rc-change="1496303623" exec-time="298" queue-time="0" op-digest="a7243ad3ba33c28a04a75e5567c9d73b" op-secure-params=" user " op-secure-digest="d5f683936a1e6d67cbca8e284acab041"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state remote_node="true" id="redis-bundle-0" uname="redis-bundle-0" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
++ <transient_attributes id="redis-bundle-0">
++ <instance_attributes id="status-redis-bundle-0">
++ <nvpair id="status-redis-bundle-0-master-redis" name="master-redis" value="1"/>
++ </instance_attributes>
++ </transient_attributes>
++ <lrm id="redis-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="redis" type="redis" class="ocf" provider="heartbeat">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:0;42:17:0:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="58" rc-code="0" op-status="0" interval="0" last-run="1496303672" last-rc-change="1496303672" exec-time="1583" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.12" transition-key="44:18:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" transition-magic="0:8;44:18:8:14f1d42b-7a22-4dc5-987e-d4fc94449aea" on_node="undercloud" call-id="80" rc-code="8" op-status="0" interval="20000" last-rc-change="1496303675" exec-time="771" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e" op-secure-params=" user " op-secure-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ </status>
++</cib>
+--
+1.8.3.1
+
+
+From 00c0cda9226b04a9a95e78b9bff63419bf3b4e47 Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Thu, 15 Jun 2017 14:05:45 +1000
+Subject: [PATCH 3/8] PE: Basic inter-bundle ordering when both sides have
+ children
+
+---
+ pengine/container.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++-
+ 1 file changed, 62 insertions(+), 1 deletion(-)
+
+diff --git a/pengine/container.c b/pengine/container.c
+index d648e0f..a39aebf 100644
+--- a/pengine/container.c
++++ b/pengine/container.c
+@@ -396,15 +396,76 @@ container_action_flags(action_t * action, node_t * node)
+ return flags;
+ }
+
+-
+ enum pe_graph_flags
+ container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
+ enum pe_action_flags filter, enum pe_ordering type)
+ {
++ gboolean current = FALSE;
+ enum pe_graph_flags changed = pe_graph_none;
++ container_variant_data_t *first_data = NULL;
++ container_variant_data_t *then_data = NULL;
++
+
+ // At the point we need to force container X to stop because
+ // resource Y needs to stop, here is where we'd implement that
++ crm_trace("%s -> %s", first->uuid, then->uuid);
++ if(first->rsc == NULL || then->rsc == NULL) {
++ return changed;
++
++ } else if(first->rsc->variant != then->rsc->variant) {
++ return changed; // For now
++ }
++
++ /* Fix this - lazy */
++ if (crm_ends_with(first->uuid, "_stopped_0")
++ || crm_ends_with(first->uuid, "_demoted_0")) {
++ current = TRUE;
++ }
++
++ get_container_variant_data(first_data, first->rsc);
++ get_container_variant_data(then_data, then->rsc);
++
++ if(first_data->child == NULL || then_data->child == NULL) {
++ return changed; // For now
++ }
++
++ for (GListPtr gIter = then_data->tuples; gIter != NULL; gIter = gIter->next) {
++ container_grouping_t *tuple = (container_grouping_t *)gIter->data;
++
++ resource_t *first_child = find_compatible_child(tuple->docker, first_data->child, RSC_ROLE_UNKNOWN, current);
++ if (first_child == NULL && current) {
++ crm_trace("Ignore");
++
++ } else if (first_child == NULL) {
++ crm_debug("No match found for %s (%d / %s / %s)", tuple->child->id, current, first->uuid, then->uuid);
++
++ /* Me no like this hack - but what else can we do?
++ *
++ * If there is no-one active or about to be active
++ * on the same node as then_child, then they must
++ * not be allowed to start
++ */
++ if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) {
++ pe_rsc_info(then->rsc, "Inhibiting %s from being active", tuple->child->id);
++ if(assign_node(tuple->child, NULL, TRUE)) {
++ changed |= pe_graph_updated_then;
++ }
++ }
++
++ } else {
++ enum action_tasks task = get_complex_task(first_child, first->task, TRUE);
++ pe_action_t *first_action = find_first_action(first_child->actions, NULL, task2text(task), node);
++ pe_action_t *then_action = find_first_action(tuple->child->actions, NULL, then->task, node);
++
++ if (order_actions(first_action, then_action, type)) {
++ crm_debug("Created constraint for %s -> %s", first_action->uuid, then_action->uuid);
++ changed |= (pe_graph_updated_first | pe_graph_updated_then);
++ }
++ changed |= tuple->child->cmds->update_actions(first_action, then_action, node,
++ first_child->cmds->action_flags(first_action, node),
++ filter, type);
++ }
++ }
+
+ return changed;
+ }
+--
+1.8.3.1
+
+
+From 45fa91e049629a22fd69310e34235b5422b87146 Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Fri, 16 Jun 2017 13:33:55 +1000
+Subject: [PATCH 4/8] Fix: PE: Functional inter-bundle ordering when either
+ side has no child
+
+Also, force containers to restart if they are found active before one of
+their dependancies.
+---
+ pengine/clone.c | 34 ++---
+ pengine/container.c | 85 ++++++++----
+ pengine/graph.c | 11 +-
+ pengine/master.c | 5 +-
+ pengine/test10/bundle-order-partial-start-2.dot | 21 +++
+ pengine/test10/bundle-order-partial-start-2.exp | 149 ++++++++++++++++++---
+ .../test10/bundle-order-partial-start-2.summary | 9 ++
+ pengine/test10/bundle-order-partial-start.dot | 1 +
+ pengine/test10/bundle-order-partial-start.exp | 3 +
+ pengine/test10/bundle-order-partial-stop.dot | 19 ++-
+ pengine/test10/bundle-order-partial-stop.exp | 47 +++----
+ pengine/test10/bundle-order-partial-stop.summary | 1 -
+ pengine/test10/bundle-order-startup.dot | 2 +
+ pengine/test10/bundle-order-startup.exp | 6 +
+ pengine/test10/bundle-order-stop.dot | 19 ++-
+ pengine/test10/bundle-order-stop.exp | 47 +++----
+ pengine/test10/bundle-order-stop.summary | 1 -
+ pengine/utils.h | 4 +-
+ 18 files changed, 331 insertions(+), 133 deletions(-)
+
+diff --git a/pengine/clone.c b/pengine/clone.c
+index a51677a..a79271e 100644
+--- a/pengine/clone.c
++++ b/pengine/clone.c
+@@ -946,7 +946,7 @@ assign_node(resource_t * rsc, node_t * node, gboolean force)
+
+ static resource_t *
+ find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc,
+- enum rsc_role_e filter, gboolean current)
++ GListPtr children, enum rsc_role_e filter, gboolean current)
+ {
+ GListPtr gIter = NULL;
+
+@@ -958,8 +958,7 @@ find_compatible_child_by_node(resource_t * local_child, node_t * local_node, res
+ crm_trace("Looking for compatible child from %s for %s on %s",
+ local_child->id, rsc->id, local_node->details->uname);
+
+- gIter = rsc->children;
+- for (; gIter != NULL; gIter = gIter->next) {
++ for (gIter = children; gIter != NULL; gIter = gIter->next) {
+ resource_t *child_rsc = (resource_t *) gIter->data;
+
+ if(is_child_compatible(child_rsc, local_node, filter, current)) {
+@@ -1003,8 +1002,7 @@ is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e
+ }
+
+ resource_t *
+-find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_e filter,
+- gboolean current)
++find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr children, enum rsc_role_e filter, gboolean current)
+ {
+ resource_t *pair = NULL;
+ GListPtr gIter = NULL;
+@@ -1013,7 +1011,7 @@ find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_
+
+ local_node = local_child->fns->location(local_child, NULL, current);
+ if (local_node) {
+- return find_compatible_child_by_node(local_child, local_node, rsc, filter, current);
++ return find_compatible_child_by_node(local_child, local_node, rsc, children, filter, current);
+ }
+
+ scratch = g_hash_table_get_values(local_child->allowed_nodes);
+@@ -1023,7 +1021,7 @@ find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_
+ for (; gIter != NULL; gIter = gIter->next) {
+ node_t *node = (node_t *) gIter->data;
+
+- pair = find_compatible_child_by_node(local_child, node, rsc, filter, current);
++ pair = find_compatible_child_by_node(local_child, node, rsc, children, filter, current);
+ if (pair) {
+ goto done;
+ }
+@@ -1084,7 +1082,7 @@ clone_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation
+ } else if (do_interleave) {
+ resource_t *rh_child = NULL;
+
+- rh_child = find_compatible_child(rsc_lh, rsc_rh, RSC_ROLE_UNKNOWN, FALSE);
++ rh_child = find_compatible_child(rsc_lh, rsc_rh, rsc_rh->children, RSC_ROLE_UNKNOWN, FALSE);
+
+ if (rh_child) {
+ pe_rsc_debug(rsc_rh, "Pairing %s with %s", rsc_lh->id, rh_child->id);
+@@ -1167,7 +1165,7 @@ clone_child_action(action_t * action)
+ }
+
+ enum pe_action_flags
+-clone_action_flags(action_t * action, node_t * node)
++summary_action_flags(action_t * action, GListPtr children, node_t * node)
+ {
+ GListPtr gIter = NULL;
+ gboolean any_runnable = FALSE;
+@@ -1176,15 +1174,13 @@ clone_action_flags(action_t * action, node_t * node)
+ enum pe_action_flags flags = (pe_action_optional | pe_action_runnable | pe_action_pseudo);
+ const char *task_s = task2text(task);
+
+- gIter = action->rsc->children;
+- for (; gIter != NULL; gIter = gIter->next) {
++ for (gIter = children; gIter != NULL; gIter = gIter->next) {
+ action_t *child_action = NULL;
+ resource_t *child = (resource_t *) gIter->data;
+
+- child_action =
+- find_first_action(child->actions, NULL, task_s, child->children ? NULL : node);
+- pe_rsc_trace(action->rsc, "Checking for %s in %s on %s", task_s, child->id,
+- node ? node->details->uname : "none");
++ child_action = find_first_action(child->actions, NULL, task_s, child->children ? NULL : node);
++ pe_rsc_trace(action->rsc, "Checking for %s in %s on %s (%s)", task_s, child->id,
++ node ? node->details->uname : "none", child_action?child_action->uuid:"NA");
+ if (child_action) {
+ enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);
+
+@@ -1223,6 +1219,12 @@ clone_action_flags(action_t * action, node_t * node)
+ return flags;
+ }
+
++enum pe_action_flags
++clone_action_flags(action_t * action, node_t * node)
++{
++ return summary_action_flags(action, action->rsc->children, node);
++}
++
+ static enum pe_graph_flags
+ clone_update_actions_interleave(action_t * first, action_t * then, node_t * node,
+ enum pe_action_flags flags, enum pe_action_flags filter,
+@@ -1246,7 +1248,7 @@ clone_update_actions_interleave(action_t * first, action_t * then, node_t * node
+ resource_t *then_child = (resource_t *) gIter->data;
+
+ CRM_ASSERT(then_child != NULL);
+- first_child = find_compatible_child(then_child, first->rsc, RSC_ROLE_UNKNOWN, current);
++ first_child = find_compatible_child(then_child, first->rsc, first->rsc->children, RSC_ROLE_UNKNOWN, current);
+ if (first_child == NULL && current) {
+ crm_trace("Ignore");
+
+diff --git a/pengine/container.c b/pengine/container.c
+index a39aebf..f3ea797 100644
+--- a/pengine/container.c
++++ b/pengine/container.c
+@@ -42,6 +42,19 @@ gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set);
+ void distribute_children(resource_t *rsc, GListPtr children, GListPtr nodes,
+ int max, int per_host_max, pe_working_set_t * data_set);
+
++static GListPtr get_container_list(resource_t *rsc)
++{
++ GListPtr containers = NULL;
++ container_variant_data_t *data = NULL;
++
++ get_container_variant_data(data, rsc);
++ for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
++ container_grouping_t *tuple = (container_grouping_t *)gIter->data;
++ containers = g_list_append(containers, tuple->docker);
++ }
++ return containers;
++}
++
+ node_t *
+ container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
+ {
+@@ -54,11 +67,7 @@ container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
+ get_container_variant_data(container_data, rsc);
+
+ set_bit(rsc->flags, pe_rsc_allocating);
+-
+- for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
+- container_grouping_t *tuple = (container_grouping_t *)gIter->data;
+- containers = g_list_append(containers, tuple->docker);
+- }
++ containers = get_container_list(rsc);
+
+ dump_node_scores(show_scores ? 0 : scores_log_level, rsc, __FUNCTION__, rsc->allowed_nodes);
+
+@@ -128,6 +137,7 @@ container_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+
+ CRM_CHECK(rsc != NULL, return);
+
++ containers = get_container_list(rsc);
+ get_container_variant_data(container_data, rsc);
+ for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) {
+ container_grouping_t *tuple = (container_grouping_t *)gIter->data;
+@@ -138,7 +148,6 @@ container_create_actions(resource_t * rsc, pe_working_set_t * data_set)
+ }
+ if(tuple->docker) {
+ tuple->docker->cmds->create_actions(tuple->docker, data_set);
+- containers = g_list_append(containers, tuple->docker);
+ }
+ if(tuple->remote) {
+ tuple->remote->cmds->create_actions(tuple->remote, data_set);
+@@ -392,7 +401,18 @@ container_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc, rsc_colocatio
+ enum pe_action_flags
+ container_action_flags(action_t * action, node_t * node)
+ {
+- enum pe_action_flags flags = (pe_action_optional | pe_action_runnable | pe_action_pseudo);
++ enum pe_action_flags flags = 0;
++ container_variant_data_t *data = NULL;
++
++ get_container_variant_data(data, action->rsc);
++ if(data->child) {
++ flags = summary_action_flags(action, data->child->children, node);
++
++ } else {
++ GListPtr containers = get_container_list(action->rsc);
++ flags = summary_action_flags(action, containers, node);
++ g_list_free(containers);
++ }
+ return flags;
+ }
+
+@@ -402,9 +422,8 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum
+ {
+ gboolean current = FALSE;
+ enum pe_graph_flags changed = pe_graph_none;
+- container_variant_data_t *first_data = NULL;
+ container_variant_data_t *then_data = NULL;
+-
++ GListPtr containers = NULL;
+
+ // At the point we need to force container X to stop because
+ // resource Y needs to stop, here is where we'd implement that
+@@ -422,22 +441,21 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum
+ current = TRUE;
+ }
+
+- get_container_variant_data(first_data, first->rsc);
+ get_container_variant_data(then_data, then->rsc);
+-
+- if(first_data->child == NULL || then_data->child == NULL) {
+- return changed; // For now
+- }
++ containers = get_container_list(first->rsc);
+
+ for (GListPtr gIter = then_data->tuples; gIter != NULL; gIter = gIter->next) {
+ container_grouping_t *tuple = (container_grouping_t *)gIter->data;
+
+- resource_t *first_child = find_compatible_child(tuple->docker, first_data->child, RSC_ROLE_UNKNOWN, current);
++ /* We can't do the then_data->child->children trick here,
++ * since the node's wont match
++ */
++ resource_t *first_child = find_compatible_child(tuple->docker, first->rsc, containers, RSC_ROLE_UNKNOWN, current);
+ if (first_child == NULL && current) {
+ crm_trace("Ignore");
+
+ } else if (first_child == NULL) {
+- crm_debug("No match found for %s (%d / %s / %s)", tuple->child->id, current, first->uuid, then->uuid);
++ crm_debug("No match found for %s (%d / %s / %s)", tuple->docker->id, current, first->uuid, then->uuid);
+
+ /* Me no like this hack - but what else can we do?
+ *
+@@ -446,27 +464,45 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum
+ * not be allowed to start
+ */
+ if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) {
+- pe_rsc_info(then->rsc, "Inhibiting %s from being active", tuple->child->id);
+- if(assign_node(tuple->child, NULL, TRUE)) {
++ pe_rsc_info(then->rsc, "Inhibiting %s from being active", tuple->docker->id);
++ if(assign_node(tuple->docker, NULL, TRUE)) {
+ changed |= pe_graph_updated_then;
+ }
+ }
+
+ } else {
+ enum action_tasks task = get_complex_task(first_child, first->task, TRUE);
++
++ /* Potentially we might want to invovle first_data->child
++ * if present, however we mostly just need the "you need
++ * to stop" signal to flow back up the ordering chain via
++ * the docker resources which are always present
++ *
++ * Almost certain to break if first->task or then->task is
++ * promote or demote
++ */
+ pe_action_t *first_action = find_first_action(first_child->actions, NULL, task2text(task), node);
+- pe_action_t *then_action = find_first_action(tuple->child->actions, NULL, then->task, node);
++ pe_action_t *then_action = find_first_action(tuple->docker->actions, NULL, then->task, node);
+
+ if (order_actions(first_action, then_action, type)) {
+- crm_debug("Created constraint for %s -> %s", first_action->uuid, then_action->uuid);
++ crm_debug("Created constraint for %s (%d) -> %s (%d) %.6x",
++ first_action->uuid, is_set(first_action->flags, pe_action_optional),
++ then_action->uuid, is_set(then_action->flags, pe_action_optional), type);
+ changed |= (pe_graph_updated_first | pe_graph_updated_then);
+ }
+- changed |= tuple->child->cmds->update_actions(first_action, then_action, node,
+- first_child->cmds->action_flags(first_action, node),
+- filter, type);
++ if(first_action && then_action) {
++ changed |= tuple->docker->cmds->update_actions(first_action, then_action, node,
++ first_child->cmds->action_flags(first_action, node),
++ filter, type);
++ } else {
++ crm_err("Nothing found either for %s (%p) or %s (%p) %s",
++ first_child->id, first_action,
++ tuple->docker->id, then_action, task2text(task));
++ }
+ }
+ }
+
++ g_list_free(containers);
+ return changed;
+ }
+
+@@ -492,11 +528,8 @@ container_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
+ }
+
+ if(container_data->child && (constraint->role_filter == RSC_ROLE_SLAVE || constraint->role_filter == RSC_ROLE_MASTER)) {
+- // Translate the node into container names running on that node
+- crm_err("Applying constraint %s", constraint->id);
+ container_data->child->cmds->rsc_location(container_data->child, constraint);
+ container_data->child->rsc_location = g_list_prepend(container_data->child->rsc_location, constraint);
+- crm_err("Added %d location constraints to %s", g_list_length(container_data->child->rsc_location), container_data->child->id);
+ }
+ }
+
+diff --git a/pengine/graph.c b/pengine/graph.c
+index c93745b..b774c71 100644
+--- a/pengine/graph.c
++++ b/pengine/graph.c
+@@ -231,7 +231,9 @@ graph_update_action(action_t * first, action_t * then, node_t * node,
+ pe_action_optional, pe_order_implies_first);
+
+ } else if (is_set(first_flags, pe_action_optional) == FALSE) {
+- pe_rsc_trace(first->rsc, "first unrunnable: %s then %s", first->uuid, then->uuid);
++ pe_rsc_trace(first->rsc, "first unrunnable: %s (%d) then %s (%d)",
++ first->uuid, is_set(first_flags, pe_action_optional),
++ then->uuid, is_set(then_flags, pe_action_optional));
+ if (update_action_flags(first, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__)) {
+ changed |= pe_graph_updated_first;
+ }
+@@ -240,7 +242,9 @@ graph_update_action(action_t * first, action_t * then, node_t * node,
+ if (changed) {
+ pe_rsc_trace(then->rsc, "implies left: %s then %s: changed", first->uuid, then->uuid);
+ } else {
+- crm_trace("implies left: %s then %s", first->uuid, then->uuid);
++ crm_trace("implies left: %s (%d) then %s (%d)",
++ first->uuid, is_set(first_flags, pe_action_optional),
++ then->uuid, is_set(then_flags, pe_action_optional));
+ }
+ }
+
+@@ -607,7 +611,8 @@ update_action(action_t * then)
+ }
+
+ if (changed & pe_graph_disable) {
+- crm_trace("Disabled constraint %s -> %s", other->action->uuid, then->uuid);
++ crm_trace("Disabled constraint %s -> %s in favor of %s -> %s",
++ other->action->uuid, then->uuid, first->uuid, then->uuid);
+ clear_bit(changed, pe_graph_disable);
+ other->type = pe_order_none;
+ }
+diff --git a/pengine/master.c b/pengine/master.c
+index 93e5186..c15e740 100644
+--- a/pengine/master.c
++++ b/pengine/master.c
+@@ -153,8 +153,6 @@ static void apply_master_location(resource_t *child, GListPtr location_constrain
+ int new_priority = merge_weights(child->priority, cons_node->weight);
+ pe_rsc_trace(child, "\t%s[%s]: %d -> %d (%d)", child->id, cons_node->details->uname,
+ child->priority, new_priority, cons_node->weight);
+- crm_err("\t%s[%s]: %d -> %d (%d)", child->id, cons_node->details->uname,
+- child->priority, new_priority, cons_node->weight);
+ child->priority = new_priority;
+ }
+ }
+@@ -720,7 +718,6 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
+ }
+
+ apply_master_location(child_rsc, child_rsc->rsc_location, chosen);
+- crm_err("Applying %d location constraints for %s", g_list_length(rsc->rsc_location), rsc->id);
+ apply_master_location(child_rsc, rsc->rsc_location, chosen);
+
+ for (gIter2 = child_rsc->rsc_cons; gIter2 != NULL; gIter2 = gIter2->next) {
+@@ -1025,7 +1022,7 @@ master_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocatio
+ g_list_free(rhs);
+
+ } else if (constraint->role_lh == RSC_ROLE_MASTER) {
+- resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE);
++ resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, rsc_rh->children, constraint->role_rh, FALSE);
+
+ if (rh_child == NULL && constraint->score >= INFINITY) {
+ pe_rsc_trace(rsc_lh, "%s can't be promoted %s", rsc_lh->id, constraint->id);
+diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot
+index 59a8b15..20afbe6 100644
+--- a/pengine/test10/bundle-order-partial-start-2.dot
++++ b/pengine/test10/bundle-order-partial-start-2.dot
+@@ -1,10 +1,30 @@
+ digraph "g" {
++"all_stopped" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold]
++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"galera-bundle-0_stop_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold]
++"galera-bundle-0_stop_0 undercloud" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
+ "galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
+ "galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold]
+ "galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
+ "galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
+ "galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold]
+ "galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
+ "galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
+@@ -16,6 +36,7 @@ digraph "g" {
+ "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp
+index 621332c..08f5084 100644
+--- a/pengine/test10/bundle-order-partial-start-2.exp
++++ b/pengine/test10/bundle-order-partial-start-2.exp
+@@ -34,6 +34,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="33" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <rsc_op id="38" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+@@ -47,6 +50,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="33" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <rsc_op id="38" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="undercloud"/>
+ </trigger>
+ </inputs>
+@@ -60,12 +66,104 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="33" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="41" operation="start" operation_key="galera-bundle-master_start_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="5">
+ <action_set>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="30" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="34" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="6">
++ <action_set>
++ <rsc_op id="30" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="8">
++ <action_set>
++ <rsc_op id="33" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3123"/>
++ <downed>
++ <node id="galera-bundle-0"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="10">
++ <action_set>
++ <rsc_op id="4" operation="monitor" operation_key="galera-bundle-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="undercloud" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="undercloud" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="33" operation="start" operation_key="galera-bundle-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="11">
++ <action_set>
+ <rsc_op id="64" operation="monitor" operation_key="redis_monitor_20000" internal_operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
+@@ -77,7 +175,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="6">
++ <synapse id="12">
+ <action_set>
+ <rsc_op id="63" operation="promote" operation_key="redis_promote_0" internal_operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="undercloud">
+ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
+@@ -90,7 +188,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="7">
++ <synapse id="13">
+ <action_set>
+ <rsc_op id="90" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_60000" on_node="undercloud" on_node_uuid="1">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+@@ -103,7 +201,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="8">
++ <synapse id="14">
+ <action_set>
+ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+@@ -119,7 +217,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="9">
++ <synapse id="15">
+ <action_set>
+ <rsc_op id="15" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="undercloud" on_node_uuid="1">
+ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
+@@ -128,7 +226,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="10" priority="1000000">
++ <synapse id="16" priority="1000000">
+ <action_set>
+ <pseudo_event id="92" operation="running" operation_key="haproxy-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -140,7 +238,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="11">
++ <synapse id="17">
+ <action_set>
+ <pseudo_event id="91" operation="start" operation_key="haproxy-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -148,7 +246,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="12" priority="1000000">
++ <synapse id="18" priority="1000000">
+ <action_set>
+ <pseudo_event id="74" operation="promoted" operation_key="redis-bundle_promoted_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -160,7 +258,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="13">
++ <synapse id="19">
+ <action_set>
+ <pseudo_event id="73" operation="promote" operation_key="redis-bundle_promote_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -168,7 +266,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="14" priority="1000000">
++ <synapse id="20" priority="1000000">
+ <action_set>
+ <pseudo_event id="70" operation="promoted" operation_key="redis-bundle-master_promoted_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -180,7 +278,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="15">
++ <synapse id="21">
+ <action_set>
+ <pseudo_event id="69" operation="promote" operation_key="redis-bundle-master_promote_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -192,7 +290,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="16" priority="1000000">
++ <synapse id="22" priority="1000000">
+ <action_set>
+ <pseudo_event id="42" operation="running" operation_key="galera-bundle-master_running_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -207,7 +305,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="17">
++ <synapse id="23">
+ <action_set>
+ <pseudo_event id="41" operation="start" operation_key="galera-bundle-master_start_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -219,7 +317,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="18" priority="1000000">
++ <synapse id="24" priority="1000000">
+ <action_set>
+ <pseudo_event id="35" operation="running" operation_key="galera-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -231,7 +329,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="19">
++ <synapse id="25">
+ <action_set>
+ <pseudo_event id="34" operation="start" operation_key="galera-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -249,7 +347,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="20" priority="1000000">
++ <synapse id="26" priority="1000000">
+ <action_set>
+ <pseudo_event id="27" operation="running" operation_key="rabbitmq-bundle-clone_running_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -264,7 +362,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="21">
++ <synapse id="27">
+ <action_set>
+ <pseudo_event id="26" operation="start" operation_key="rabbitmq-bundle-clone_start_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -276,7 +374,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="22" priority="1000000">
++ <synapse id="28" priority="1000000">
+ <action_set>
+ <pseudo_event id="21" operation="running" operation_key="rabbitmq-bundle_running_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -288,7 +386,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="23">
++ <synapse id="29">
+ <action_set>
+ <pseudo_event id="20" operation="start" operation_key="rabbitmq-bundle_start_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -296,4 +394,19 @@
+ </action_set>
+ <inputs/>
+ </synapse>
++ <synapse id="30">
++ <action_set>
++ <pseudo_event id="14" operation="all_stopped" operation_key="all_stopped">
++ <attributes />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="30" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
+ </transition_graph>
+diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary
+index 859ca25..5e3927c 100644
+--- a/pengine/test10/bundle-order-partial-start-2.summary
++++ b/pengine/test10/bundle-order-partial-start-2.summary
+@@ -22,21 +22,26 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-
+
+ Transition Summary:
+ * Start rabbitmq:0 (rabbitmq-bundle-0)
++ * Restart galera-bundle-docker-0 (Started undercloud)
++ * Restart galera-bundle-0 (Started undercloud)
+ * Start galera:0 (galera-bundle-0)
+ * Promote redis:0 (Slave -> Master redis-bundle-0)
+ * Start haproxy-bundle-docker-0 (undercloud)
+
+ Executing cluster transition:
++ * Resource action: galera-bundle-0 stop on undercloud
+ * Resource action: haproxy-bundle-docker-0 monitor on undercloud
+ * Pseudo action: haproxy-bundle_start_0
+ * Pseudo action: redis-bundle_promote_0
+ * Pseudo action: redis-bundle-master_promote_0
+ * Pseudo action: rabbitmq-bundle_start_0
++ * Resource action: galera-bundle-docker-0 stop on undercloud
+ * Resource action: redis promote on redis-bundle-0
+ * Resource action: haproxy-bundle-docker-0 start on undercloud
+ * Pseudo action: haproxy-bundle_running_0
+ * Pseudo action: redis-bundle-master_promoted_0
+ * Pseudo action: rabbitmq-bundle-clone_start_0
++ * Pseudo action: all_stopped
+ * Resource action: rabbitmq:0 start on rabbitmq-bundle-0
+ * Resource action: redis monitor=20000 on redis-bundle-0
+ * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud
+@@ -45,6 +50,10 @@ Executing cluster transition:
+ * Pseudo action: rabbitmq-bundle_running_0
+ * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0
+ * Pseudo action: galera-bundle_start_0
++ * Resource action: galera-bundle-docker-0 start on undercloud
++ * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud
++ * Resource action: galera-bundle-0 start on undercloud
++ * Resource action: galera-bundle-0 monitor=60000 on undercloud
+ * Pseudo action: galera-bundle-master_start_0
+ * Resource action: galera:0 start on galera-bundle-0
+ * Pseudo action: galera-bundle-master_running_0
+diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot
+index 06c3620..c36ff04 100644
+--- a/pengine/test10/bundle-order-partial-start.dot
++++ b/pengine/test10/bundle-order-partial-start.dot
+@@ -30,6 +30,7 @@ digraph "g" {
+ "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp
+index b48bb03..4a5c01c 100644
+--- a/pengine/test10/bundle-order-partial-start.exp
++++ b/pengine/test10/bundle-order-partial-start.exp
+@@ -103,6 +103,9 @@
+ <trigger>
+ <pseudo_event id="33" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
++ <trigger>
++ <rsc_op id="88" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+diff --git a/pengine/test10/bundle-order-partial-stop.dot b/pengine/test10/bundle-order-partial-stop.dot
+index 235b634..af2493c 100644
+--- a/pengine/test10/bundle-order-partial-stop.dot
++++ b/pengine/test10/bundle-order-partial-stop.dot
+@@ -11,6 +11,7 @@ digraph "g" {
+ "galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold]
+ "galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold]
+@@ -19,6 +20,7 @@ digraph "g" {
+ "galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold]
+ "galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold]
+ "galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = dashed]
+ "galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed]
+ "galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed]
+@@ -32,11 +34,16 @@ digraph "g" {
+ "galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold]
+ "galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold]
+ "galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_demoted_0" -> "galera-bundle_start_0" [ style = dashed]
+ "galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold]
+ "galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed]
++"galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
+ "galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold]
+ "galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
+ "galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = dashed]
+ "galera-bundle_stopped_0" -> "redis-bundle_stop_0" [ style = bold]
+ "galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+ "galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold]
+@@ -110,6 +117,7 @@ digraph "g" {
+ "rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
+ "rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed]
+ "rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed]
+ "rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed]
+@@ -123,6 +131,7 @@ digraph "g" {
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold]
+ "rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
+ "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+@@ -142,6 +151,7 @@ digraph "g" {
+ "redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"redis-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold]
+ "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold]
+@@ -150,6 +160,7 @@ digraph "g" {
+ "redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold]
+ "redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold]
+ "redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = dashed]
+ "redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed]
+ "redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed]
+@@ -163,16 +174,18 @@ digraph "g" {
+ "redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold]
+ "redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold]
+ "redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
+-"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold]
++"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = dashed]
+ "redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold]
+ "redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_running_0" -> "galera-bundle_start_0" [ style = dashed]
++"redis-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed]
+-"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
+ "redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold]
+ "redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
+ "redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+ "redis-bundle_stopped_0" -> "haproxy-bundle_stop_0" [ style = bold]
+-"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold]
++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = dashed]
+ "redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+ "redis_demote_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold]
+ "redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold]
+diff --git a/pengine/test10/bundle-order-partial-stop.exp b/pengine/test10/bundle-order-partial-stop.exp
+index d085433..02a7372 100644
+--- a/pengine/test10/bundle-order-partial-stop.exp
++++ b/pengine/test10/bundle-order-partial-stop.exp
+@@ -173,6 +173,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+@@ -283,6 +286,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="86" operation="stop" operation_key="haproxy-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+@@ -505,22 +511,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="34">
+- <action_set>
+- <pseudo_event id="56" operation="start" operation_key="redis-bundle_start_0">
+- <attributes CRM_meta_timeout="20000" />
+- </pseudo_event>
+- </action_set>
+- <inputs>
+- <trigger>
+- <pseudo_event id="59" operation="stopped" operation_key="redis-bundle_stopped_0"/>
+- </trigger>
+- <trigger>
+- <pseudo_event id="76" operation="demoted" operation_key="redis-bundle_demoted_0"/>
+- </trigger>
+- </inputs>
+- </synapse>
+- <synapse id="35" priority="1000000">
++ <synapse id="34" priority="1000000">
+ <action_set>
+ <pseudo_event id="53" operation="demoted" operation_key="galera-bundle_demoted_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -535,7 +526,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="36">
++ <synapse id="35">
+ <action_set>
+ <pseudo_event id="52" operation="demote" operation_key="galera-bundle_demote_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -543,7 +534,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="37" priority="1000000">
++ <synapse id="36" priority="1000000">
+ <action_set>
+ <pseudo_event id="49" operation="demoted" operation_key="galera-bundle-master_demoted_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -558,7 +549,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="38">
++ <synapse id="37">
+ <action_set>
+ <pseudo_event id="48" operation="demote" operation_key="galera-bundle-master_demote_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -570,7 +561,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="39" priority="1000000">
++ <synapse id="38" priority="1000000">
+ <action_set>
+ <pseudo_event id="45" operation="stopped" operation_key="galera-bundle-master_stopped_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -585,7 +576,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="40">
++ <synapse id="39">
+ <action_set>
+ <pseudo_event id="44" operation="stop" operation_key="galera-bundle-master_stop_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -600,7 +591,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="41" priority="1000000">
++ <synapse id="40" priority="1000000">
+ <action_set>
+ <pseudo_event id="36" operation="stopped" operation_key="galera-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -612,7 +603,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="42">
++ <synapse id="41">
+ <action_set>
+ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -624,7 +615,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="43" priority="1000000">
++ <synapse id="42" priority="1000000">
+ <action_set>
+ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -639,7 +630,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="44">
++ <synapse id="43">
+ <action_set>
+ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -651,7 +642,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="45" priority="1000000">
++ <synapse id="44" priority="1000000">
+ <action_set>
+ <pseudo_event id="24" operation="stopped" operation_key="rabbitmq-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -663,7 +654,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="46">
++ <synapse id="45">
+ <action_set>
+ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -671,7 +662,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="47">
++ <synapse id="46">
+ <action_set>
+ <pseudo_event id="18" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+diff --git a/pengine/test10/bundle-order-partial-stop.summary b/pengine/test10/bundle-order-partial-stop.summary
+index bd6f937..c341f4c 100644
+--- a/pengine/test10/bundle-order-partial-stop.summary
++++ b/pengine/test10/bundle-order-partial-stop.summary
+@@ -78,7 +78,6 @@ Executing cluster transition:
+ * Resource action: redis-bundle-0 stop on undercloud
+ * Pseudo action: redis-bundle-master_stopped_0
+ * Pseudo action: redis-bundle_stopped_0
+- * Pseudo action: redis-bundle_start_0
+ * Resource action: redis-bundle-docker-0 stop on undercloud
+ * Pseudo action: haproxy-bundle_stop_0
+ * Resource action: haproxy-bundle-docker-0 stop on undercloud
+diff --git a/pengine/test10/bundle-order-startup.dot b/pengine/test10/bundle-order-startup.dot
+index 73947a5..4e60d79 100644
+--- a/pengine/test10/bundle-order-startup.dot
++++ b/pengine/test10/bundle-order-startup.dot
+@@ -32,6 +32,7 @@ digraph "g" {
+ "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold]
++"haproxy-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "haproxy-bundle_running_0" -> "redis-bundle_start_0" [ style = bold]
+ "haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+@@ -116,6 +117,7 @@ digraph "g" {
+ "redis-bundle-docker-0_monitor_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "redis-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-0_start_0 undercloud" [ style = bold]
+ "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
+ "redis-bundle-docker-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
+diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp
+index 51d108d..66ecb08 100644
+--- a/pengine/test10/bundle-order-startup.exp
++++ b/pengine/test10/bundle-order-startup.exp
+@@ -176,6 +176,9 @@
+ <trigger>
+ <pseudo_event id="31" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
++ <trigger>
++ <rsc_op id="50" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="12">
+@@ -291,6 +294,9 @@
+ <trigger>
+ <pseudo_event id="54" operation="start" operation_key="redis-bundle_start_0"/>
+ </trigger>
++ <trigger>
++ <rsc_op id="85" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="20">
+diff --git a/pengine/test10/bundle-order-stop.dot b/pengine/test10/bundle-order-stop.dot
+index 235b634..af2493c 100644
+--- a/pengine/test10/bundle-order-stop.dot
++++ b/pengine/test10/bundle-order-stop.dot
+@@ -11,6 +11,7 @@ digraph "g" {
+ "galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold]
+ "galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold]
+@@ -19,6 +20,7 @@ digraph "g" {
+ "galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold]
+ "galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold]
+ "galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = dashed]
+ "galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed]
+ "galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed]
+@@ -32,11 +34,16 @@ digraph "g" {
+ "galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold]
+ "galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold]
+ "galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_demoted_0" -> "galera-bundle_start_0" [ style = dashed]
+ "galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold]
+ "galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed]
++"galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
+ "galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold]
+ "galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
+ "galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = dashed]
+ "galera-bundle_stopped_0" -> "redis-bundle_stop_0" [ style = bold]
+ "galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+ "galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold]
+@@ -110,6 +117,7 @@ digraph "g" {
+ "rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
+ "rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed]
+ "rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed]
+ "rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed]
+@@ -123,6 +131,7 @@ digraph "g" {
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
++"rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold]
+ "rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
+ "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+@@ -142,6 +151,7 @@ digraph "g" {
+ "redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"redis-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold]
+ "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold]
+@@ -150,6 +160,7 @@ digraph "g" {
+ "redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold]
+ "redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold]
+ "redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = dashed]
+ "redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed]
+ "redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed]
+@@ -163,16 +174,18 @@ digraph "g" {
+ "redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold]
+ "redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold]
+ "redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"]
+-"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold]
++"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = dashed]
+ "redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold]
+ "redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_running_0" -> "galera-bundle_start_0" [ style = dashed]
++"redis-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed]
+-"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
+ "redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold]
+ "redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
+ "redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+ "redis-bundle_stopped_0" -> "haproxy-bundle_stop_0" [ style = bold]
+-"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold]
++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = dashed]
+ "redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+ "redis_demote_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold]
+ "redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold]
+diff --git a/pengine/test10/bundle-order-stop.exp b/pengine/test10/bundle-order-stop.exp
+index d085433..02a7372 100644
+--- a/pengine/test10/bundle-order-stop.exp
++++ b/pengine/test10/bundle-order-stop.exp
+@@ -173,6 +173,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+ </trigger>
+ </inputs>
+@@ -283,6 +286,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="86" operation="stop" operation_key="haproxy-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+@@ -505,22 +511,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="34">
+- <action_set>
+- <pseudo_event id="56" operation="start" operation_key="redis-bundle_start_0">
+- <attributes CRM_meta_timeout="20000" />
+- </pseudo_event>
+- </action_set>
+- <inputs>
+- <trigger>
+- <pseudo_event id="59" operation="stopped" operation_key="redis-bundle_stopped_0"/>
+- </trigger>
+- <trigger>
+- <pseudo_event id="76" operation="demoted" operation_key="redis-bundle_demoted_0"/>
+- </trigger>
+- </inputs>
+- </synapse>
+- <synapse id="35" priority="1000000">
++ <synapse id="34" priority="1000000">
+ <action_set>
+ <pseudo_event id="53" operation="demoted" operation_key="galera-bundle_demoted_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -535,7 +526,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="36">
++ <synapse id="35">
+ <action_set>
+ <pseudo_event id="52" operation="demote" operation_key="galera-bundle_demote_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -543,7 +534,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="37" priority="1000000">
++ <synapse id="36" priority="1000000">
+ <action_set>
+ <pseudo_event id="49" operation="demoted" operation_key="galera-bundle-master_demoted_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -558,7 +549,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="38">
++ <synapse id="37">
+ <action_set>
+ <pseudo_event id="48" operation="demote" operation_key="galera-bundle-master_demote_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -570,7 +561,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="39" priority="1000000">
++ <synapse id="38" priority="1000000">
+ <action_set>
+ <pseudo_event id="45" operation="stopped" operation_key="galera-bundle-master_stopped_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -585,7 +576,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="40">
++ <synapse id="39">
+ <action_set>
+ <pseudo_event id="44" operation="stop" operation_key="galera-bundle-master_stop_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -600,7 +591,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="41" priority="1000000">
++ <synapse id="40" priority="1000000">
+ <action_set>
+ <pseudo_event id="36" operation="stopped" operation_key="galera-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -612,7 +603,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="42">
++ <synapse id="41">
+ <action_set>
+ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -624,7 +615,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="43" priority="1000000">
++ <synapse id="42" priority="1000000">
+ <action_set>
+ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -639,7 +630,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="44">
++ <synapse id="43">
+ <action_set>
+ <pseudo_event id="29" operation="stop" operation_key="rabbitmq-bundle-clone_stop_0">
+ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
+@@ -651,7 +642,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="45" priority="1000000">
++ <synapse id="44" priority="1000000">
+ <action_set>
+ <pseudo_event id="24" operation="stopped" operation_key="rabbitmq-bundle_stopped_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -663,7 +654,7 @@
+ </trigger>
+ </inputs>
+ </synapse>
+- <synapse id="46">
++ <synapse id="45">
+ <action_set>
+ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0">
+ <attributes CRM_meta_timeout="20000" />
+@@ -671,7 +662,7 @@
+ </action_set>
+ <inputs/>
+ </synapse>
+- <synapse id="47">
++ <synapse id="46">
+ <action_set>
+ <pseudo_event id="18" operation="all_stopped" operation_key="all_stopped">
+ <attributes />
+diff --git a/pengine/test10/bundle-order-stop.summary b/pengine/test10/bundle-order-stop.summary
+index bd6f937..c341f4c 100644
+--- a/pengine/test10/bundle-order-stop.summary
++++ b/pengine/test10/bundle-order-stop.summary
+@@ -78,7 +78,6 @@ Executing cluster transition:
+ * Resource action: redis-bundle-0 stop on undercloud
+ * Pseudo action: redis-bundle-master_stopped_0
+ * Pseudo action: redis-bundle_stopped_0
+- * Pseudo action: redis-bundle_start_0
+ * Resource action: redis-bundle-docker-0 stop on undercloud
+ * Pseudo action: haproxy-bundle_stop_0
+ * Resource action: haproxy-bundle-docker-0 stop on undercloud
+diff --git a/pengine/utils.h b/pengine/utils.h
+index 10e7201..1421166 100644
+--- a/pengine/utils.h
++++ b/pengine/utils.h
+@@ -50,11 +50,11 @@ extern void log_action(unsigned int log_level, const char *pre_text,
+ action_t * action, gboolean details);
+
+ extern gboolean can_run_any(GHashTable * nodes);
+-extern resource_t *find_compatible_child(resource_t * local_child, resource_t * rsc,
++extern resource_t *find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr children,
+ enum rsc_role_e filter, gboolean current);
+ gboolean is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e filter, gboolean current);
+ bool assign_node(resource_t * rsc, node_t * node, gboolean force);
+-
++enum pe_action_flags summary_action_flags(action_t * action, GListPtr children, node_t * node);
+
+ enum filter_colocation_res {
+ influence_nothing = 0,
+--
+1.8.3.1
+
+
+From e083fa923274c908a8ef4b124a5d7e53caabd7f5 Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Fri, 16 Jun 2017 14:47:37 +1000
+Subject: [PATCH 5/8] Fix: PE: Default to non-interleaved bundle ordering for
+ safety
+
+---
+ pengine/container.c | 89 +++++++++++++++++++-----
+ pengine/test10/bundle-order-partial-start-2.dot | 2 +-
+ pengine/test10/bundle-order-partial-start-2.exp | 6 +-
+ pengine/test10/bundle-order-partial-start.dot | 2 +-
+ pengine/test10/bundle-order-partial-start.exp | 6 +-
+ pengine/test10/bundle-order-partial-stop.dot | 8 ++-
+ pengine/test10/bundle-order-partial-stop.exp | 22 ++++--
+ pengine/test10/bundle-order-partial-stop.summary | 6 +-
+ pengine/test10/bundle-order-startup.dot | 5 +-
+ pengine/test10/bundle-order-startup.exp | 15 ++--
+ pengine/test10/bundle-order-stop.dot | 8 ++-
+ pengine/test10/bundle-order-stop.exp | 22 ++++--
+ pengine/test10/bundle-order-stop.summary | 6 +-
+ 13 files changed, 143 insertions(+), 54 deletions(-)
+
+diff --git a/pengine/container.c b/pengine/container.c
+index f3ea797..adee92a 100644
+--- a/pengine/container.c
++++ b/pengine/container.c
+@@ -209,13 +209,10 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)
+
+ if(tuple->child) {
+ order_stop_stop(rsc, tuple->child, pe_order_implies_first_printed);
+-
+- } else {
+- order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed);
+- new_rsc_order(tuple->docker, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
+- new_rsc_order(tuple->docker, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed,
+- data_set);
+ }
++ order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed);
++ new_rsc_order(tuple->docker, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set);
++ new_rsc_order(tuple->docker, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set);
+
+ if(tuple->ip) {
+ tuple->ip->cmds->internal_constraints(tuple->ip, data_set);
+@@ -416,8 +413,8 @@ container_action_flags(action_t * action, node_t * node)
+ return flags;
+ }
+
+-enum pe_graph_flags
+-container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
++static enum pe_graph_flags
++container_update_interleave_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
+ enum pe_action_flags filter, enum pe_ordering type)
+ {
+ gboolean current = FALSE;
+@@ -425,22 +422,18 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum
+ container_variant_data_t *then_data = NULL;
+ GListPtr containers = NULL;
+
+- // At the point we need to force container X to stop because
+- // resource Y needs to stop, here is where we'd implement that
+- crm_trace("%s -> %s", first->uuid, then->uuid);
+- if(first->rsc == NULL || then->rsc == NULL) {
+- return changed;
+-
+- } else if(first->rsc->variant != then->rsc->variant) {
+- return changed; // For now
+- }
+-
+ /* Fix this - lazy */
+ if (crm_ends_with(first->uuid, "_stopped_0")
+ || crm_ends_with(first->uuid, "_demoted_0")) {
+ current = TRUE;
+ }
+
++ /* Eventually we may want to allow interleaving between bundles
++ * and clones, but for now assert both sides are bundles
++ */
++ CRM_ASSERT(first->rsc->variant == pe_container);
++ CRM_ASSERT(then->rsc->variant == pe_container);
++
+ get_container_variant_data(then_data, then->rsc);
+ containers = get_container_list(first->rsc);
+
+@@ -506,6 +499,66 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum
+ return changed;
+ }
+
++enum pe_graph_flags
++container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
++ enum pe_action_flags filter, enum pe_ordering type)
++{
++ bool interleave = FALSE;
++ enum pe_graph_flags changed = pe_graph_none;
++
++ crm_trace("%s -> %s", first->uuid, then->uuid);
++
++ if(first->rsc == NULL || then->rsc == NULL) {
++ return changed;
++
++ } else if(first->rsc->variant == then->rsc->variant) {
++ // When and how to turn on interleaving?
++ // interleave = TRUE;
++ }
++
++ if(interleave) {
++ changed = container_update_interleave_actions(first, then, node, flags, filter, type);
++
++ } else {
++ GListPtr gIter = then->rsc->children;
++ GListPtr containers = NULL;
++
++ // Handle the 'primitive' ordering case
++ changed |= native_update_actions(first, then, node, flags, filter, type);
++
++ // Now any children (or containers in the case of a bundle)
++ if(then->rsc->variant == pe_container) {
++ containers = get_container_list(then->rsc);
++ gIter = containers;
++ }
++
++ for (; gIter != NULL; gIter = gIter->next) {
++ resource_t *then_child = (resource_t *) gIter->data;
++ enum pe_graph_flags then_child_changed = pe_graph_none;
++ action_t *then_child_action = find_first_action(then_child->actions, NULL, then->task, node);
++
++ if (then_child_action) {
++ enum pe_action_flags then_child_flags = then_child->cmds->action_flags(then_child_action, node);
++
++ if (is_set(then_child_flags, pe_action_runnable)) {
++ then_child_changed |=
++ then_child->cmds->update_actions(first, then_child_action, node, flags, filter, type);
++ }
++ changed |= then_child_changed;
++ if (then_child_changed & pe_graph_updated_then) {
++ for (GListPtr lpc = then_child_action->actions_after; lpc != NULL; lpc = lpc->next) {
++ action_wrapper_t *next = (action_wrapper_t *) lpc->data;
++ update_action(next->action);
++ }
++ }
++ }
++ }
++
++ g_list_free(containers);
++ }
++ return changed;
++}
++
+ void
+ container_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
+ {
+diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot
+index 20afbe6..b5ecaf5 100644
+--- a/pengine/test10/bundle-order-partial-start-2.dot
++++ b/pengine/test10/bundle-order-partial-start-2.dot
+@@ -13,6 +13,7 @@ digraph "g" {
+ "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold]
+ "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle_running_0" [ style = bold]
+ "galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
+ "galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+@@ -36,7 +37,6 @@ digraph "g" {
+ "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
+-"haproxy-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp
+index 08f5084..a2740d7 100644
+--- a/pengine/test10/bundle-order-partial-start-2.exp
++++ b/pengine/test10/bundle-order-partial-start-2.exp
+@@ -90,9 +90,6 @@
+ <trigger>
+ <pseudo_event id="34" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
+- <trigger>
+- <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
+- </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="6">
+@@ -325,6 +322,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="31" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="42" operation="running" operation_key="galera-bundle-master_running_0"/>
+ </trigger>
+ </inputs>
+diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot
+index c36ff04..756acfd 100644
+--- a/pengine/test10/bundle-order-partial-start.dot
++++ b/pengine/test10/bundle-order-partial-start.dot
+@@ -10,6 +10,7 @@ digraph "g" {
+ "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold]
+ "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle_running_0" [ style = bold]
+ "galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
+ "galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
+@@ -30,7 +31,6 @@ digraph "g" {
+ "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
+-"haproxy-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp
+index 4a5c01c..ebd5785 100644
+--- a/pengine/test10/bundle-order-partial-start.exp
++++ b/pengine/test10/bundle-order-partial-start.exp
+@@ -103,9 +103,6 @@
+ <trigger>
+ <pseudo_event id="33" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
+- <trigger>
+- <rsc_op id="88" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
+- </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+@@ -306,6 +303,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="29" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="41" operation="running" operation_key="galera-bundle-master_running_0"/>
+ </trigger>
+ </inputs>
+diff --git a/pengine/test10/bundle-order-partial-stop.dot b/pengine/test10/bundle-order-partial-stop.dot
+index af2493c..c0e6616 100644
+--- a/pengine/test10/bundle-order-partial-stop.dot
++++ b/pengine/test10/bundle-order-partial-stop.dot
+@@ -11,7 +11,7 @@ digraph "g" {
+ "galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
+-"galera-bundle-docker-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" -> "galera-bundle_stopped_0" [ style = bold]
+ "galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold]
+ "galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold]
+@@ -40,6 +40,7 @@ digraph "g" {
+ "galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed]
+ "galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold]
+ "galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
+ "galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+@@ -130,9 +131,11 @@ digraph "g" {
+ "rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "rabbitmq-bundle_stopped_0" [ style = bold]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold]
++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
+ "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+ "rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+@@ -151,7 +154,7 @@ digraph "g" {
+ "redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
+-"redis-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"redis-bundle-docker-0_stop_0 undercloud" -> "redis-bundle_stopped_0" [ style = bold]
+ "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold]
+ "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold]
+@@ -181,6 +184,7 @@ digraph "g" {
+ "redis-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed]
+ "redis-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
++"redis-bundle_stop_0" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold]
+ "redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
+ "redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+diff --git a/pengine/test10/bundle-order-partial-stop.exp b/pengine/test10/bundle-order-partial-stop.exp
+index 02a7372..27ae320 100644
+--- a/pengine/test10/bundle-order-partial-stop.exp
++++ b/pengine/test10/bundle-order-partial-stop.exp
+@@ -26,6 +26,9 @@
+ <trigger>
+ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+ </trigger>
++ <trigger>
++ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0"/>
++ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+@@ -99,6 +102,9 @@
+ <trigger>
+ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+ </trigger>
++ <trigger>
++ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0"/>
++ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+@@ -173,10 +179,10 @@
+ </action_set>
+ <inputs>
+ <trigger>
+- <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ <pseudo_event id="58" operation="stop" operation_key="redis-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -286,9 +292,6 @@
+ </action_set>
+ <inputs>
+ <trigger>
+- <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+- </trigger>
+- <trigger>
+ <pseudo_event id="86" operation="stop" operation_key="haproxy-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+@@ -492,6 +495,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="68" operation="stopped" operation_key="redis-bundle-master_stopped_0"/>
+ </trigger>
+ </inputs>
+@@ -599,6 +605,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="45" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>
+ </trigger>
+ </inputs>
+@@ -650,6 +659,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="19" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+diff --git a/pengine/test10/bundle-order-partial-stop.summary b/pengine/test10/bundle-order-partial-stop.summary
+index c341f4c..b30a237 100644
+--- a/pengine/test10/bundle-order-partial-stop.summary
++++ b/pengine/test10/bundle-order-partial-stop.summary
+@@ -64,21 +64,21 @@ Executing cluster transition:
+ * Pseudo action: galera-bundle_demoted_0
+ * Pseudo action: galera-bundle_stop_0
+ * Pseudo action: rabbitmq-bundle-clone_stopped_0
+- * Pseudo action: rabbitmq-bundle_stopped_0
+ * Resource action: rabbitmq-bundle-docker-0 stop on undercloud
+ * Pseudo action: galera-bundle-master_stop_0
++ * Pseudo action: rabbitmq-bundle_stopped_0
+ * Resource action: galera stop on galera-bundle-0
+ * Resource action: galera-bundle-0 stop on undercloud
+ * Pseudo action: galera-bundle-master_stopped_0
+- * Pseudo action: galera-bundle_stopped_0
+ * Resource action: galera-bundle-docker-0 stop on undercloud
++ * Pseudo action: galera-bundle_stopped_0
+ * Pseudo action: redis-bundle_stop_0
+ * Pseudo action: redis-bundle-master_stop_0
+ * Resource action: redis stop on redis-bundle-0
+ * Resource action: redis-bundle-0 stop on undercloud
+ * Pseudo action: redis-bundle-master_stopped_0
+- * Pseudo action: redis-bundle_stopped_0
+ * Resource action: redis-bundle-docker-0 stop on undercloud
++ * Pseudo action: redis-bundle_stopped_0
+ * Pseudo action: haproxy-bundle_stop_0
+ * Resource action: haproxy-bundle-docker-0 stop on undercloud
+ * Pseudo action: haproxy-bundle_stopped_0
+diff --git a/pengine/test10/bundle-order-startup.dot b/pengine/test10/bundle-order-startup.dot
+index 4e60d79..66053b9 100644
+--- a/pengine/test10/bundle-order-startup.dot
++++ b/pengine/test10/bundle-order-startup.dot
+@@ -10,6 +10,7 @@ digraph "g" {
+ "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold]
+ "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle_running_0" [ style = bold]
+ "galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
+ "galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
+@@ -32,7 +33,6 @@ digraph "g" {
+ "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold]
+-"haproxy-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "haproxy-bundle_running_0" -> "redis-bundle_start_0" [ style = bold]
+ "haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+@@ -98,6 +98,7 @@ digraph "g" {
+ "rabbitmq-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
+ "rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle-0_start_0 undercloud" [ style = bold]
+ "rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle_running_0" [ style = bold]
+ "rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold]
+ "rabbitmq-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
+@@ -117,9 +118,9 @@ digraph "g" {
+ "redis-bundle-docker-0_monitor_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "redis-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"]
+-"redis-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold]
+ "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-0_start_0 undercloud" [ style = bold]
+ "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_monitor_60000 undercloud" [ style = bold]
++"redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle_running_0" [ style = bold]
+ "redis-bundle-docker-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
+ "redis-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold]
+diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp
+index 66ecb08..2128d7c 100644
+--- a/pengine/test10/bundle-order-startup.exp
++++ b/pengine/test10/bundle-order-startup.exp
+@@ -176,9 +176,6 @@
+ <trigger>
+ <pseudo_event id="31" operation="start" operation_key="galera-bundle_start_0"/>
+ </trigger>
+- <trigger>
+- <rsc_op id="50" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
+- </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="12">
+@@ -294,9 +291,6 @@
+ <trigger>
+ <pseudo_event id="54" operation="start" operation_key="redis-bundle_start_0"/>
+ </trigger>
+- <trigger>
+- <rsc_op id="85" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
+- </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="20">
+@@ -714,6 +708,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="50" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="62" operation="running" operation_key="redis-bundle-master_running_0"/>
+ </trigger>
+ </inputs>
+@@ -765,6 +762,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="27" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="39" operation="running" operation_key="galera-bundle-master_running_0"/>
+ </trigger>
+ </inputs>
+@@ -816,6 +816,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="13" operation="start" operation_key="rabbitmq-bundle-docker-0_start_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="24" operation="running" operation_key="rabbitmq-bundle-clone_running_0"/>
+ </trigger>
+ </inputs>
+diff --git a/pengine/test10/bundle-order-stop.dot b/pengine/test10/bundle-order-stop.dot
+index af2493c..c0e6616 100644
+--- a/pengine/test10/bundle-order-stop.dot
++++ b/pengine/test10/bundle-order-stop.dot
+@@ -11,7 +11,7 @@ digraph "g" {
+ "galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
+-"galera-bundle-docker-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"galera-bundle-docker-0_stop_0 undercloud" -> "galera-bundle_stopped_0" [ style = bold]
+ "galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold]
+ "galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold]
+@@ -40,6 +40,7 @@ digraph "g" {
+ "galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed]
+ "galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold]
+ "galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold]
+ "galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+@@ -130,9 +131,11 @@ digraph "g" {
+ "rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "rabbitmq-bundle_stopped_0" [ style = bold]
+ "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold]
++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold]
+ "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+ "rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
+@@ -151,7 +154,7 @@ digraph "g" {
+ "redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold]
+ "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold]
+-"redis-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold]
++"redis-bundle-docker-0_stop_0 undercloud" -> "redis-bundle_stopped_0" [ style = bold]
+ "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"]
+ "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold]
+ "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold]
+@@ -181,6 +184,7 @@ digraph "g" {
+ "redis-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
+ "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed]
+ "redis-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
++"redis-bundle_stop_0" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold]
+ "redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold]
+ "redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold]
+ "redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
+diff --git a/pengine/test10/bundle-order-stop.exp b/pengine/test10/bundle-order-stop.exp
+index 02a7372..27ae320 100644
+--- a/pengine/test10/bundle-order-stop.exp
++++ b/pengine/test10/bundle-order-stop.exp
+@@ -26,6 +26,9 @@
+ <trigger>
+ <rsc_op id="20" operation="stop" operation_key="rabbitmq-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+ </trigger>
++ <trigger>
++ <pseudo_event id="23" operation="stop" operation_key="rabbitmq-bundle_stop_0"/>
++ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="2">
+@@ -99,6 +102,9 @@
+ <trigger>
+ <rsc_op id="32" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+ </trigger>
++ <trigger>
++ <pseudo_event id="35" operation="stop" operation_key="galera-bundle_stop_0"/>
++ </trigger>
+ </inputs>
+ </synapse>
+ <synapse id="7">
+@@ -173,10 +179,10 @@
+ </action_set>
+ <inputs>
+ <trigger>
+- <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+ </trigger>
+ <trigger>
+- <rsc_op id="55" operation="stop" operation_key="redis-bundle-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ <pseudo_event id="58" operation="stop" operation_key="redis-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+ </synapse>
+@@ -286,9 +292,6 @@
+ </action_set>
+ <inputs>
+ <trigger>
+- <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
+- </trigger>
+- <trigger>
+ <pseudo_event id="86" operation="stop" operation_key="haproxy-bundle_stop_0"/>
+ </trigger>
+ </inputs>
+@@ -492,6 +495,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="54" operation="stop" operation_key="redis-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="68" operation="stopped" operation_key="redis-bundle-master_stopped_0"/>
+ </trigger>
+ </inputs>
+@@ -599,6 +605,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="31" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="45" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>
+ </trigger>
+ </inputs>
+@@ -650,6 +659,9 @@
+ </action_set>
+ <inputs>
+ <trigger>
++ <rsc_op id="19" operation="stop" operation_key="rabbitmq-bundle-docker-0_stop_0" on_node="undercloud" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
+ <pseudo_event id="30" operation="stopped" operation_key="rabbitmq-bundle-clone_stopped_0"/>
+ </trigger>
+ </inputs>
+diff --git a/pengine/test10/bundle-order-stop.summary b/pengine/test10/bundle-order-stop.summary
+index c341f4c..b30a237 100644
+--- a/pengine/test10/bundle-order-stop.summary
++++ b/pengine/test10/bundle-order-stop.summary
+@@ -64,21 +64,21 @@ Executing cluster transition:
+ * Pseudo action: galera-bundle_demoted_0
+ * Pseudo action: galera-bundle_stop_0
+ * Pseudo action: rabbitmq-bundle-clone_stopped_0
+- * Pseudo action: rabbitmq-bundle_stopped_0
+ * Resource action: rabbitmq-bundle-docker-0 stop on undercloud
+ * Pseudo action: galera-bundle-master_stop_0
++ * Pseudo action: rabbitmq-bundle_stopped_0
+ * Resource action: galera stop on galera-bundle-0
+ * Resource action: galera-bundle-0 stop on undercloud
+ * Pseudo action: galera-bundle-master_stopped_0
+- * Pseudo action: galera-bundle_stopped_0
+ * Resource action: galera-bundle-docker-0 stop on undercloud
++ * Pseudo action: galera-bundle_stopped_0
+ * Pseudo action: redis-bundle_stop_0
+ * Pseudo action: redis-bundle-master_stop_0
+ * Resource action: redis stop on redis-bundle-0
+ * Resource action: redis-bundle-0 stop on undercloud
+ * Pseudo action: redis-bundle-master_stopped_0
+- * Pseudo action: redis-bundle_stopped_0
+ * Resource action: redis-bundle-docker-0 stop on undercloud
++ * Pseudo action: redis-bundle_stopped_0
+ * Pseudo action: haproxy-bundle_stop_0
+ * Resource action: haproxy-bundle-docker-0 stop on undercloud
+ * Pseudo action: haproxy-bundle_stopped_0
+--
+1.8.3.1
+
+
+From a189db3a8f42ee4817b9b0e1fc128ea64e945130 Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Tue, 20 Jun 2017 16:19:02 +1000
+Subject: [PATCH 6/8] Feature: PE: Generic ordering for anything with bundles
+
+---
+ pengine/allocate.c | 4 +-
+ pengine/allocate.h | 3 -
+ pengine/clone.c | 220 +++-------------------------------------
+ pengine/container.c | 287 +++++++++++++++++++++++++++++++++++++++++-----------
+ pengine/master.c | 2 +-
+ pengine/utils.h | 10 +-
+ 6 files changed, 250 insertions(+), 276 deletions(-)
+
+diff --git a/pengine/allocate.c b/pengine/allocate.c
+index 8f15282..3a95be6 100644
+--- a/pengine/allocate.c
++++ b/pengine/allocate.c
+@@ -79,7 +79,7 @@ resource_alloc_functions_t resource_class_alloc_functions[] = {
+ clone_rsc_colocation_rh,
+ clone_rsc_location,
+ clone_action_flags,
+- clone_update_actions,
++ container_update_actions,
+ clone_expand,
+ clone_append_meta,
+ },
+@@ -93,7 +93,7 @@ resource_alloc_functions_t resource_class_alloc_functions[] = {
+ master_rsc_colocation_rh,
+ clone_rsc_location,
+ clone_action_flags,
+- clone_update_actions,
++ container_update_actions,
+ clone_expand,
+ master_append_meta,
+ },
+diff --git a/pengine/allocate.h b/pengine/allocate.h
+index 9a30b80..4ca5d65 100644
+--- a/pengine/allocate.h
++++ b/pengine/allocate.h
+@@ -168,9 +168,6 @@ extern enum pe_graph_flags native_update_actions(action_t * first, action_t * th
+ extern enum pe_graph_flags group_update_actions(action_t * first, action_t * then, node_t * node,
+ enum pe_action_flags flags,
+ enum pe_action_flags filter, enum pe_ordering type);
+-extern enum pe_graph_flags clone_update_actions(action_t * first, action_t * then, node_t * node,
+- enum pe_action_flags flags,
+- enum pe_action_flags filter, enum pe_ordering type);
+ extern enum pe_graph_flags container_update_actions(action_t * first, action_t * then, node_t * node,
+ enum pe_action_flags flags,
+ enum pe_action_flags filter, enum pe_ordering type);
+diff --git a/pengine/clone.c b/pengine/clone.c
+index a79271e..28f368e 100644
+--- a/pengine/clone.c
++++ b/pengine/clone.c
+@@ -944,34 +944,6 @@ assign_node(resource_t * rsc, node_t * node, gboolean force)
+ return changed;
+ }
+
+-static resource_t *
+-find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc,
+- GListPtr children, enum rsc_role_e filter, gboolean current)
+-{
+- GListPtr gIter = NULL;
+-
+- if (local_node == NULL) {
+- crm_err("Can't colocate unrunnable child %s with %s", local_child->id, rsc->id);
+- return NULL;
+- }
+-
+- crm_trace("Looking for compatible child from %s for %s on %s",
+- local_child->id, rsc->id, local_node->details->uname);
+-
+- for (gIter = children; gIter != NULL; gIter = gIter->next) {
+- resource_t *child_rsc = (resource_t *) gIter->data;
+-
+- if(is_child_compatible(child_rsc, local_node, filter, current)) {
+- crm_trace("Pairing %s with %s on %s",
+- local_child->id, child_rsc->id, local_node->details->uname);
+- return child_rsc;
+- }
+- }
+-
+- crm_trace("Can't pair %s with %s", local_child->id, rsc->id);
+- return NULL;
+-}
+-
+ gboolean
+ is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e filter, gboolean current)
+ {
+@@ -1002,7 +974,7 @@ is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e
+ }
+
+ resource_t *
+-find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr children, enum rsc_role_e filter, gboolean current)
++find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_e filter, gboolean current)
+ {
+ resource_t *pair = NULL;
+ GListPtr gIter = NULL;
+@@ -1011,7 +983,7 @@ find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr child
+
+ local_node = local_child->fns->location(local_child, NULL, current);
+ if (local_node) {
+- return find_compatible_child_by_node(local_child, local_node, rsc, children, filter, current);
++ return find_compatible_child_by_node(local_child, local_node, rsc, filter, current);
+ }
+
+ scratch = g_hash_table_get_values(local_child->allowed_nodes);
+@@ -1021,7 +993,7 @@ find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr child
+ for (; gIter != NULL; gIter = gIter->next) {
+ node_t *node = (node_t *) gIter->data;
+
+- pair = find_compatible_child_by_node(local_child, node, rsc, children, filter, current);
++ pair = find_compatible_child_by_node(local_child, node, rsc, filter, current);
+ if (pair) {
+ goto done;
+ }
+@@ -1048,29 +1020,26 @@ clone_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation
+ {
+ GListPtr gIter = NULL;
+ gboolean do_interleave = FALSE;
+- clone_variant_data_t *clone_data = NULL;
+- clone_variant_data_t *clone_data_lh = NULL;
++ const char *interleave_s = NULL;
+
+ CRM_CHECK(constraint != NULL, return);
+ CRM_CHECK(rsc_lh != NULL, pe_err("rsc_lh was NULL for %s", constraint->id); return);
+ CRM_CHECK(rsc_rh != NULL, pe_err("rsc_rh was NULL for %s", constraint->id); return);
+ CRM_CHECK(rsc_lh->variant == pe_native, return);
+
+- get_clone_variant_data(clone_data, constraint->rsc_rh);
+ pe_rsc_trace(rsc_rh, "Processing constraint %s: %s -> %s %d",
+ constraint->id, rsc_lh->id, rsc_rh->id, constraint->score);
+
+- if (pe_rsc_is_clone(constraint->rsc_lh)) {
+-
+- get_clone_variant_data(clone_data_lh, constraint->rsc_lh);
+- if (clone_data_lh->interleave
+- && clone_data->clone_node_max != clone_data_lh->clone_node_max) {
+- crm_config_err("Cannot interleave " XML_CIB_TAG_INCARNATION " %s and %s because"
+- " they do not support the same number of" " resources per node",
++ /* only the LHS side needs to be labeled as interleave */
++ interleave_s = g_hash_table_lookup(constraint->rsc_lh->meta, XML_RSC_ATTR_INTERLEAVE);
++ if(crm_is_true(interleave_s) && constraint->rsc_lh->variant > pe_group) {
++ // TODO: Do we actually care about multiple RH copies sharing a LH copy anymore?
++ if (copies_per_node(constraint->rsc_lh) != copies_per_node(constraint->rsc_rh)) {
++ crm_config_err("Cannot interleave %s and %s because"
++ " they do not support the same number of copies per node",
+ constraint->rsc_lh->id, constraint->rsc_rh->id);
+
+- /* only the LHS side needs to be labeled as interleave */
+- } else if (clone_data_lh->interleave) {
++ } else {
+ do_interleave = TRUE;
+ }
+ }
+@@ -1082,7 +1051,7 @@ clone_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation
+ } else if (do_interleave) {
+ resource_t *rh_child = NULL;
+
+- rh_child = find_compatible_child(rsc_lh, rsc_rh, rsc_rh->children, RSC_ROLE_UNKNOWN, FALSE);
++ rh_child = find_compatible_child(rsc_lh, rsc_rh, RSC_ROLE_UNKNOWN, FALSE);
+
+ if (rh_child) {
+ pe_rsc_debug(rsc_rh, "Pairing %s with %s", rsc_lh->id, rh_child->id);
+@@ -1125,7 +1094,7 @@ clone_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation
+ }
+ }
+
+-static enum action_tasks
++enum action_tasks
+ clone_child_action(action_t * action)
+ {
+ enum action_tasks result = no_action;
+@@ -1225,167 +1194,6 @@ clone_action_flags(action_t * action, node_t * node)
+ return summary_action_flags(action, action->rsc->children, node);
+ }
+
+-static enum pe_graph_flags
+-clone_update_actions_interleave(action_t * first, action_t * then, node_t * node,
+- enum pe_action_flags flags, enum pe_action_flags filter,
+- enum pe_ordering type)
+-{
+- gboolean current = FALSE;
+- resource_t *first_child = NULL;
+- GListPtr gIter = then->rsc->children;
+- enum pe_graph_flags changed = pe_graph_none; /*pe_graph_disable */
+-
+- enum action_tasks task = clone_child_action(first);
+- const char *first_task = task2text(task);
+-
+- /* Fix this - lazy */
+- if (crm_ends_with(first->uuid, "_stopped_0")
+- || crm_ends_with(first->uuid, "_demoted_0")) {
+- current = TRUE;
+- }
+-
+- for (; gIter != NULL; gIter = gIter->next) {
+- resource_t *then_child = (resource_t *) gIter->data;
+-
+- CRM_ASSERT(then_child != NULL);
+- first_child = find_compatible_child(then_child, first->rsc, first->rsc->children, RSC_ROLE_UNKNOWN, current);
+- if (first_child == NULL && current) {
+- crm_trace("Ignore");
+-
+- } else if (first_child == NULL) {
+- crm_debug("No match found for %s (%d / %s / %s)", then_child->id, current, first->uuid,
+- then->uuid);
+-
+- /* Me no like this hack - but what else can we do?
+- *
+- * If there is no-one active or about to be active
+- * on the same node as then_child, then they must
+- * not be allowed to start
+- */
+- if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) {
+- pe_rsc_info(then->rsc, "Inhibiting %s from being active", then_child->id);
+- if(assign_node(then_child, NULL, TRUE)) {
+- changed |= pe_graph_updated_then;
+- }
+- }
+-
+- } else {
+- action_t *first_action = NULL;
+- action_t *then_action = NULL;
+-
+- pe_rsc_debug(then->rsc, "Pairing %s with %s", first_child->id, then_child->id);
+-
+- first_action = find_first_action(first_child->actions, NULL, first_task, node);
+- then_action = find_first_action(then_child->actions, NULL, then->task, node);
+-
+- if (first_action == NULL) {
+- if (is_not_set(first_child->flags, pe_rsc_orphan)
+- && crm_str_eq(first_task, RSC_STOP, TRUE) == FALSE
+- && crm_str_eq(first_task, RSC_DEMOTE, TRUE) == FALSE) {
+- crm_err("Internal error: No action found for %s in %s (first)",
+- first_task, first_child->id);
+-
+- } else {
+- crm_trace("No action found for %s in %s%s (first)",
+- first_task, first_child->id,
+- is_set(first_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
+- }
+- continue;
+- }
+-
+- /* We're only interested if 'then' is neither stopping nor being demoted */
+- if (then_action == NULL) {
+- if (is_not_set(then_child->flags, pe_rsc_orphan)
+- && crm_str_eq(then->task, RSC_STOP, TRUE) == FALSE
+- && crm_str_eq(then->task, RSC_DEMOTE, TRUE) == FALSE) {
+- crm_err("Internal error: No action found for %s in %s (then)",
+- then->task, then_child->id);
+-
+- } else {
+- crm_trace("No action found for %s in %s%s (then)",
+- then->task, then_child->id,
+- is_set(then_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
+- }
+- continue;
+- }
+-
+- if (order_actions(first_action, then_action, type)) {
+- crm_debug("Created constraint for %s -> %s", first_action->uuid, then_action->uuid);
+- changed |= (pe_graph_updated_first | pe_graph_updated_then);
+- }
+- changed |=
+- then_child->cmds->update_actions(first_action, then_action, node,
+- first_child->cmds->action_flags(first_action,
+- node), filter,
+- type);
+- }
+- }
+- return changed;
+-}
+-
+-enum pe_graph_flags
+-clone_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
+- enum pe_action_flags filter, enum pe_ordering type)
+-{
+- const char *rsc = "none";
+- gboolean interleave = FALSE;
+- enum pe_graph_flags changed = pe_graph_none;
+-
+- if (first->rsc != then->rsc
+- && pe_rsc_is_clone(first->rsc)
+- && pe_rsc_is_clone(then->rsc)) {
+- clone_variant_data_t *clone_data = NULL;
+-
+- if (crm_ends_with(then->uuid, "_stop_0")
+- || crm_ends_with(then->uuid, "_demote_0")) {
+- get_clone_variant_data(clone_data, first->rsc);
+- rsc = first->rsc->id;
+- } else {
+- get_clone_variant_data(clone_data, then->rsc);
+- rsc = then->rsc->id;
+- }
+- interleave = clone_data->interleave;
+- }
+-
+- crm_trace("Interleave %s -> %s: %s (based on %s)",
+- first->uuid, then->uuid, interleave ? "yes" : "no", rsc);
+-
+- if (interleave) {
+- changed = clone_update_actions_interleave(first, then, node, flags, filter, type);
+-
+- } else if (then->rsc) {
+- GListPtr gIter = then->rsc->children;
+-
+- changed |= native_update_actions(first, then, node, flags, filter, type);
+-
+- for (; gIter != NULL; gIter = gIter->next) {
+- enum pe_graph_flags child_changed = pe_graph_none;
+- GListPtr lpc = NULL;
+- resource_t *child = (resource_t *) gIter->data;
+- action_t *child_action = find_first_action(child->actions, NULL, then->task, node);
+-
+- if (child_action) {
+- enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);
+-
+- if (is_set(child_flags, pe_action_runnable)) {
+-
+- child_changed |=
+- child->cmds->update_actions(first, child_action, node, flags, filter, type);
+- }
+- changed |= child_changed;
+- if (child_changed & pe_graph_updated_then) {
+- for (lpc = child_action->actions_after; lpc != NULL; lpc = lpc->next) {
+- action_wrapper_t *other = (action_wrapper_t *) lpc->data;
+- update_action(other->action);
+- }
+- }
+- }
+- }
+- }
+-
+- return changed;
+-}
+-
+ void
+ clone_rsc_location(resource_t * rsc, rsc_to_node_t * constraint)
+ {
+diff --git a/pengine/container.c b/pengine/container.c
+index adee92a..f742660 100644
+--- a/pengine/container.c
++++ b/pengine/container.c
+@@ -47,14 +47,33 @@ static GListPtr get_container_list(resource_t *rsc)
+ GListPtr containers = NULL;
+ container_variant_data_t *data = NULL;
+
+- get_container_variant_data(data, rsc);
+- for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
+- container_grouping_t *tuple = (container_grouping_t *)gIter->data;
+- containers = g_list_append(containers, tuple->docker);
++ if(rsc->variant == pe_container) {
++ get_container_variant_data(data, rsc);
++ for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
++ container_grouping_t *tuple = (container_grouping_t *)gIter->data;
++ containers = g_list_append(containers, tuple->docker);
++ }
+ }
+ return containers;
+ }
+
++static GListPtr get_containers_or_children(resource_t *rsc)
++{
++ GListPtr containers = NULL;
++ container_variant_data_t *data = NULL;
++
++ if(rsc->variant == pe_container) {
++ get_container_variant_data(data, rsc);
++ for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
++ container_grouping_t *tuple = (container_grouping_t *)gIter->data;
++ containers = g_list_append(containers, tuple->docker);
++ }
++ return containers;
++ } else {
++ return rsc->children;
++ }
++}
++
+ node_t *
+ container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set)
+ {
+@@ -337,6 +356,39 @@ container_rsc_colocation_lh(resource_t * rsc, resource_t * rsc_rh, rsc_colocatio
+ CRM_ASSERT(FALSE);
+ }
+
++int copies_per_node(resource_t * rsc)
++{
++ /* Strictly speaking, there should be a 'copies_per_node' addition
++ * to the resource function table and each case would be a
++ * function. However that would be serious overkill to return an
++ * int. In fact, it seems to me that both function tables
++ * could/should be replaced by resources.{c,h} full of
++ * rsc_{some_operation} functions containing a switch as below
++ * which calls out to functions named {variant}_{some_operation}
++ * as needed.
++ */
++ switch(rsc->variant) {
++ case pe_unknown:
++ return 0;
++ case pe_native:
++ case pe_group:
++ return 1;
++ case pe_clone:
++ case pe_master:
++ {
++ const char *max_clones_node = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_NODEMAX);
++ return crm_parse_int(max_clones_node, "1");
++ }
++ case pe_container:
++ {
++ container_variant_data_t *data = NULL;
++ get_container_variant_data(data, rsc);
++ return data->replicas_per_host;
++ }
++ }
++ return 0;
++}
++
+ void
+ container_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc, rsc_colocation_t * constraint)
+ {
+@@ -346,6 +398,7 @@ container_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc, rsc_colocatio
+ CRM_CHECK(constraint != NULL, return);
+ CRM_CHECK(rsc_lh != NULL, pe_err("rsc_lh was NULL for %s", constraint->id); return);
+ CRM_CHECK(rsc != NULL, pe_err("rsc was NULL for %s", constraint->id); return);
++ CRM_ASSERT(rsc_lh->variant == pe_native);
+
+ if (is_set(rsc->flags, pe_rsc_provisional)) {
+ pe_rsc_trace(rsc, "%s is still provisional", rsc->id);
+@@ -413,14 +466,65 @@ container_action_flags(action_t * action, node_t * node)
+ return flags;
+ }
+
++resource_t *
++find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc,
++ enum rsc_role_e filter, gboolean current)
++{
++ GListPtr gIter = NULL;
++ GListPtr children = NULL;
++
++ if (local_node == NULL) {
++ crm_err("Can't colocate unrunnable child %s with %s", local_child->id, rsc->id);
++ return NULL;
++ }
++
++ crm_trace("Looking for compatible child from %s for %s on %s",
++ local_child->id, rsc->id, local_node->details->uname);
++
++ children = get_containers_or_children(rsc);
++ for (gIter = children; gIter != NULL; gIter = gIter->next) {
++ resource_t *child_rsc = (resource_t *) gIter->data;
++
++ if(is_child_compatible(child_rsc, local_node, filter, current)) {
++ crm_trace("Pairing %s with %s on %s",
++ local_child->id, child_rsc->id, local_node->details->uname);
++ return child_rsc;
++ }
++ }
++
++ crm_trace("Can't pair %s with %s", local_child->id, rsc->id);
++ if(children != rsc->children) {
++ g_list_free(children);
++ }
++ return NULL;
++}
++
++static container_grouping_t *
++tuple_for_docker(resource_t *rsc, resource_t *docker, node_t *node)
++{
++ if(rsc->variant == pe_container) {
++ container_variant_data_t *data = NULL;
++ get_container_variant_data(data, rsc);
++ for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) {
++ container_grouping_t *tuple = (container_grouping_t *)gIter->data;
++ if(tuple->child
++ && docker == tuple->docker
++ && node->details == tuple->node->details) {
++ return tuple;
++ }
++ }
++ }
++ return NULL;
++}
++
+ static enum pe_graph_flags
+ container_update_interleave_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
+ enum pe_action_flags filter, enum pe_ordering type)
+ {
++ GListPtr gIter = NULL;
++ GListPtr children = NULL;
+ gboolean current = FALSE;
+ enum pe_graph_flags changed = pe_graph_none;
+- container_variant_data_t *then_data = NULL;
+- GListPtr containers = NULL;
+
+ /* Fix this - lazy */
+ if (crm_ends_with(first->uuid, "_stopped_0")
+@@ -428,27 +532,15 @@ container_update_interleave_actions(action_t * first, action_t * then, node_t *
+ current = TRUE;
+ }
+
+- /* Eventually we may want to allow interleaving between bundles
+- * and clones, but for now assert both sides are bundles
+- */
+- CRM_ASSERT(first->rsc->variant == pe_container);
+- CRM_ASSERT(then->rsc->variant == pe_container);
+-
+- get_container_variant_data(then_data, then->rsc);
+- containers = get_container_list(first->rsc);
+-
+- for (GListPtr gIter = then_data->tuples; gIter != NULL; gIter = gIter->next) {
+- container_grouping_t *tuple = (container_grouping_t *)gIter->data;
+-
+- /* We can't do the then_data->child->children trick here,
+- * since the node's wont match
+- */
+- resource_t *first_child = find_compatible_child(tuple->docker, first->rsc, containers, RSC_ROLE_UNKNOWN, current);
++ children = get_containers_or_children(then->rsc);
++ for (gIter = children; gIter != NULL; gIter = gIter->next) {
++ resource_t *then_child = (resource_t *) gIter->data;
++ resource_t *first_child = find_compatible_child(then_child, first->rsc, RSC_ROLE_UNKNOWN, current);
+ if (first_child == NULL && current) {
+ crm_trace("Ignore");
+
+ } else if (first_child == NULL) {
+- crm_debug("No match found for %s (%d / %s / %s)", tuple->docker->id, current, first->uuid, then->uuid);
++ crm_debug("No match found for %s (%d / %s / %s)", then_child->id, current, first->uuid, then->uuid);
+
+ /* Me no like this hack - but what else can we do?
+ *
+@@ -457,25 +549,76 @@ container_update_interleave_actions(action_t * first, action_t * then, node_t *
+ * not be allowed to start
+ */
+ if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) {
+- pe_rsc_info(then->rsc, "Inhibiting %s from being active", tuple->docker->id);
+- if(assign_node(tuple->docker, NULL, TRUE)) {
++ pe_rsc_info(then->rsc, "Inhibiting %s from being active", then_child->id);
++ if(assign_node(then_child, NULL, TRUE)) {
+ changed |= pe_graph_updated_then;
+ }
+ }
+
+ } else {
+- enum action_tasks task = get_complex_task(first_child, first->task, TRUE);
++ pe_action_t *first_action = NULL;
++ pe_action_t *then_action = NULL;
+
+- /* Potentially we might want to invovle first_data->child
+- * if present, however we mostly just need the "you need
+- * to stop" signal to flow back up the ordering chain via
+- * the docker resources which are always present
+- *
+- * Almost certain to break if first->task or then->task is
+- * promote or demote
+- */
+- pe_action_t *first_action = find_first_action(first_child->actions, NULL, task2text(task), node);
+- pe_action_t *then_action = find_first_action(tuple->docker->actions, NULL, then->task, node);
++ enum action_tasks task = clone_child_action(first);
++ const char *first_task = task2text(task);
++
++ container_grouping_t *first_tuple = tuple_for_docker(first->rsc, first_child, node);
++ container_grouping_t *then_tuple = tuple_for_docker(then->rsc, then_child, node);
++
++ if(strstr(first->task, "stop") && first_tuple && first_tuple->child) {
++ /* Except for 'stopped' we should be looking at the
++ * in-container resource, actions for the child will
++ * happen later and are therefor more likely to align
++ * with the user's intent.
++ */
++ first_action = find_first_action(first_tuple->child->actions, NULL, task2text(task), node);
++ } else {
++ first_action = find_first_action(first_child->actions, NULL, task2text(task), node);
++ }
++
++ if(strstr(then->task, "mote") && then_tuple && then_tuple->child) {
++ /* Promote/demote actions will never be found for the
++ * docker resource, look in the child instead
++ *
++ * Alternatively treat:
++ * 'XXXX then promote YYYY' as 'XXXX then start container for YYYY', and
++ * 'demote XXXX then stop YYYY' as 'stop container for XXXX then stop YYYY'
++ */
++ then_action = find_first_action(then_tuple->child->actions, NULL, then->task, node);
++ } else {
++ then_action = find_first_action(then_child->actions, NULL, then->task, node);
++ }
++
++ if (first_action == NULL) {
++ if (is_not_set(first_child->flags, pe_rsc_orphan)
++ && crm_str_eq(first_task, RSC_STOP, TRUE) == FALSE
++ && crm_str_eq(first_task, RSC_DEMOTE, TRUE) == FALSE) {
++ crm_err("Internal error: No action found for %s in %s (first)",
++ first_task, first_child->id);
++
++ } else {
++ crm_trace("No action found for %s in %s%s (first)",
++ first_task, first_child->id,
++ is_set(first_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
++ }
++ continue;
++ }
++
++ /* We're only interested if 'then' is neither stopping nor being demoted */
++ if (then_action == NULL) {
++ if (is_not_set(then_child->flags, pe_rsc_orphan)
++ && crm_str_eq(then->task, RSC_STOP, TRUE) == FALSE
++ && crm_str_eq(then->task, RSC_DEMOTE, TRUE) == FALSE) {
++ crm_err("Internal error: No action found for %s in %s (then)",
++ then->task, then_child->id);
++
++ } else {
++ crm_trace("No action found for %s in %s%s (then)",
++ then->task, then_child->id,
++ is_set(then_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : "");
++ }
++ continue;
++ }
+
+ if (order_actions(first_action, then_action, type)) {
+ crm_debug("Created constraint for %s (%d) -> %s (%d) %.6x",
+@@ -484,55 +627,75 @@ container_update_interleave_actions(action_t * first, action_t * then, node_t *
+ changed |= (pe_graph_updated_first | pe_graph_updated_then);
+ }
+ if(first_action && then_action) {
+- changed |= tuple->docker->cmds->update_actions(first_action, then_action, node,
+- first_child->cmds->action_flags(first_action, node),
+- filter, type);
++ changed |= then_child->cmds->update_actions(first_action, then_action, node,
++ first_child->cmds->action_flags(first_action, node),
++ filter, type);
+ } else {
+ crm_err("Nothing found either for %s (%p) or %s (%p) %s",
+ first_child->id, first_action,
+- tuple->docker->id, then_action, task2text(task));
++ then_child->id, then_action, task2text(task));
+ }
+ }
+ }
+
+- g_list_free(containers);
++ if(children != then->rsc->children) {
++ g_list_free(children);
++ }
+ return changed;
+ }
+
++bool can_interleave_actions(pe_action_t *first, pe_action_t *then)
++{
++ bool interleave = FALSE;
++ resource_t *rsc = NULL;
++ const char *interleave_s = NULL;
++
++ if(first->rsc == NULL || then->rsc == NULL) {
++ crm_trace("Not interleaving %s with %s (both must be resources)", first->uuid, then->uuid);
++ return FALSE;
++ } else if(first->rsc == then->rsc) {
++ crm_trace("Not interleaving %s with %s (must belong to different resources)", first->uuid, then->uuid);
++ return FALSE;
++ } else if(first->rsc->variant < pe_clone || then->rsc->variant < pe_clone) {
++ crm_trace("Not interleaving %s with %s (both sides must be clones, masters, or bundles)", first->uuid, then->uuid);
++ return FALSE;
++ }
++
++ if (crm_ends_with(then->uuid, "_stop_0") || crm_ends_with(then->uuid, "_demote_0")) {
++ rsc = first->rsc;
++ } else {
++ rsc = then->rsc;
++ }
++
++ interleave_s = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERLEAVE);
++ interleave = crm_is_true(interleave_s);
++ crm_trace("Interleave %s -> %s: %s (based on %s)",
++ first->uuid, then->uuid, interleave ? "yes" : "no", rsc->id);
++
++ return interleave;
++}
++
+ enum pe_graph_flags
+ container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags,
+ enum pe_action_flags filter, enum pe_ordering type)
+ {
+- bool interleave = FALSE;
+ enum pe_graph_flags changed = pe_graph_none;
+
+ crm_trace("%s -> %s", first->uuid, then->uuid);
+
+- if(first->rsc == NULL || then->rsc == NULL) {
+- return changed;
+-
+- } else if(first->rsc->variant == then->rsc->variant) {
+- // When and how to turn on interleaving?
+- // interleave = TRUE;
+- }
+-
+- if(interleave) {
++ if(can_interleave_actions(first, then)) {
+ changed = container_update_interleave_actions(first, then, node, flags, filter, type);
+
+- } else {
+- GListPtr gIter = then->rsc->children;
+- GListPtr containers = NULL;
++ } else if(then->rsc) {
++ GListPtr gIter = NULL;
++ GListPtr children = NULL;
+
+ // Handle the 'primitive' ordering case
+ changed |= native_update_actions(first, then, node, flags, filter, type);
+
+ // Now any children (or containers in the case of a bundle)
+- if(then->rsc->variant == pe_container) {
+- containers = get_container_list(then->rsc);
+- gIter = containers;
+- }
+-
+- for (; gIter != NULL; gIter = gIter->next) {
++ children = get_containers_or_children(then->rsc);
++ for (gIter = children; gIter != NULL; gIter = gIter->next) {
+ resource_t *then_child = (resource_t *) gIter->data;
+ enum pe_graph_flags then_child_changed = pe_graph_none;
+ action_t *then_child_action = find_first_action(then_child->actions, NULL, then->task, node);
+@@ -554,7 +717,9 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum
+ }
+ }
+
+- g_list_free(containers);
++ if(children != then->rsc->children) {
++ g_list_free(children);
++ }
+ }
+ return changed;
+ }
+diff --git a/pengine/master.c b/pengine/master.c
+index c15e740..8c39f49 100644
+--- a/pengine/master.c
++++ b/pengine/master.c
+@@ -1022,7 +1022,7 @@ master_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocatio
+ g_list_free(rhs);
+
+ } else if (constraint->role_lh == RSC_ROLE_MASTER) {
+- resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, rsc_rh->children, constraint->role_rh, FALSE);
++ resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE);
+
+ if (rh_child == NULL && constraint->score >= INFINITY) {
+ pe_rsc_trace(rsc_lh, "%s can't be promoted %s", rsc_lh->id, constraint->id);
+diff --git a/pengine/utils.h b/pengine/utils.h
+index 1421166..7a788f7 100644
+--- a/pengine/utils.h
++++ b/pengine/utils.h
+@@ -49,12 +49,16 @@ void native_deallocate(resource_t * rsc);
+ extern void log_action(unsigned int log_level, const char *pre_text,
+ action_t * action, gboolean details);
+
+-extern gboolean can_run_any(GHashTable * nodes);
+-extern resource_t *find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr children,
+- enum rsc_role_e filter, gboolean current);
++gboolean can_run_any(GHashTable * nodes);
++bool can_interleave_actions(pe_action_t *first, pe_action_t *then);
++resource_t *find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_e filter, gboolean current);
++resource_t *find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc,
++ enum rsc_role_e filter, gboolean current);
+ gboolean is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e filter, gboolean current);
+ bool assign_node(resource_t * rsc, node_t * node, gboolean force);
+ enum pe_action_flags summary_action_flags(action_t * action, GListPtr children, node_t * node);
++enum action_tasks clone_child_action(action_t * action);
++int copies_per_node(resource_t * rsc);
+
+ enum filter_colocation_res {
+ influence_nothing = 0,
+--
+1.8.3.1
+
+
+From 879a52ce9f4288fde5c19d2fb677e5ad51e6c4af Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Fri, 23 Jun 2017 13:16:57 +1000
+Subject: [PATCH 7/8] Test: PE: Generic ordering for anything with bundles
+
+---
+ lib/pengine/container.c | 14 +-
+ pengine/regression.sh | 5 +-
+ pengine/test10/bundle-order-startup-clone-2.dot | 335 ++++
+ pengine/test10/bundle-order-startup-clone-2.exp | 1697 ++++++++++++++++++++
+ pengine/test10/bundle-order-startup-clone-2.scores | 584 +++++++
+ .../test10/bundle-order-startup-clone-2.summary | 179 +++
+ pengine/test10/bundle-order-startup-clone-2.xml | 190 +++
+ pengine/test10/bundle-order-startup-clone.dot | 119 ++
+ pengine/test10/bundle-order-startup-clone.exp | 327 ++++
+ pengine/test10/bundle-order-startup-clone.scores | 174 ++
+ pengine/test10/bundle-order-startup-clone.summary | 69 +
+ pengine/test10/bundle-order-startup-clone.xml | 187 +++
+ pengine/test10/bundle-order-stop-clone.dot | 80 +
+ pengine/test10/bundle-order-stop-clone.exp | 345 ++++
+ pengine/test10/bundle-order-stop-clone.scores | 591 +++++++
+ pengine/test10/bundle-order-stop-clone.summary | 75 +
+ pengine/test10/bundle-order-stop-clone.xml | 398 +++++
+ 17 files changed, 5366 insertions(+), 3 deletions(-)
+ create mode 100644 pengine/test10/bundle-order-startup-clone-2.dot
+ create mode 100644 pengine/test10/bundle-order-startup-clone-2.exp
+ create mode 100644 pengine/test10/bundle-order-startup-clone-2.scores
+ create mode 100644 pengine/test10/bundle-order-startup-clone-2.summary
+ create mode 100644 pengine/test10/bundle-order-startup-clone-2.xml
+ create mode 100644 pengine/test10/bundle-order-startup-clone.dot
+ create mode 100644 pengine/test10/bundle-order-startup-clone.exp
+ create mode 100644 pengine/test10/bundle-order-startup-clone.scores
+ create mode 100644 pengine/test10/bundle-order-startup-clone.summary
+ create mode 100644 pengine/test10/bundle-order-startup-clone.xml
+ create mode 100644 pengine/test10/bundle-order-stop-clone.dot
+ create mode 100644 pengine/test10/bundle-order-stop-clone.exp
+ create mode 100644 pengine/test10/bundle-order-stop-clone.scores
+ create mode 100644 pengine/test10/bundle-order-stop-clone.summary
+ create mode 100644 pengine/test10/bundle-order-stop-clone.xml
+
+diff --git a/lib/pengine/container.c b/lib/pengine/container.c
+index cf81253..6e98e6f 100644
+--- a/lib/pengine/container.c
++++ b/lib/pengine/container.c
+@@ -727,8 +727,6 @@ container_unpack(resource_t * rsc, pe_working_set_t * data_set)
+ }
+
+ container_data->child = new_rsc;
+- container_data->child->orig_xml = xml_obj; // Also the trigger for common_free()
+- // to free xml_resource as container_data->child->xml
+
+ mount = calloc(1, sizeof(container_mount_t));
+ mount->source = strdup(DEFAULT_REMOTE_KEY_LOCATION);
+@@ -1035,17 +1033,25 @@ tuple_free(container_grouping_t *tuple)
+
+ if(tuple->ip) {
+ tuple->ip->fns->free(tuple->ip);
++ tuple->ip->xml = NULL;
++ free_xml(tuple->ip->xml);
+ tuple->ip = NULL;
+ }
+ if(tuple->child) {
++ free_xml(tuple->child->xml);
++ tuple->child->xml = NULL;
+ tuple->child->fns->free(tuple->child);
+ tuple->child = NULL;
+ }
+ if(tuple->docker) {
++ free_xml(tuple->docker->xml);
++ tuple->docker->xml = NULL;
+ tuple->docker->fns->free(tuple->docker);
+ tuple->docker = NULL;
+ }
+ if(tuple->remote) {
++ free_xml(tuple->remote->xml);
++ tuple->remote->xml = NULL;
+ tuple->remote->fns->free(tuple->remote);
+ tuple->remote = NULL;
+ }
+@@ -1073,9 +1079,13 @@ container_free(resource_t * rsc)
+ free(container_data->docker_run_command);
+ free(container_data->docker_host_options);
+
++ if(container_data->child) {
++ free_xml(container_data->child->xml);
++ }
+ g_list_free_full(container_data->tuples, (GDestroyNotify)tuple_free);
+ g_list_free_full(container_data->mounts, (GDestroyNotify)mount_free);
+ g_list_free_full(container_data->ports, (GDestroyNotify)port_free);
++ g_list_free(rsc->children);
+ common_free(rsc);
+ }
+
+diff --git a/pengine/regression.sh b/pengine/regression.sh
+index e97b54b..b844bb9 100755
+--- a/pengine/regression.sh
++++ b/pengine/regression.sh
+@@ -23,7 +23,6 @@ core=`dirname $0`
+ create_mode="true"
+ info Generating test outputs for these tests...
+ # do_test file description
+-
+ info Done.
+ echo ""
+
+@@ -808,6 +807,10 @@ do_test bundle-order-partial-start-2 "Bundle startup ordering when some dependan
+ do_test bundle-order-stop "Bundle stop ordering"
+ do_test bundle-order-partial-stop "Bundle startup ordering when some dependancies are already stopped"
+
++do_test bundle-order-startup-clone "Prevent startup because bundle isn't promoted"
++do_test bundle-order-startup-clone-2 "Bundle startup with clones"
++do_test bundle-order-stop-clone "Stop bundle because clone is stopping"
++
+ echo ""
+ do_test whitebox-fail1 "Fail whitebox container rsc."
+ do_test whitebox-fail2 "Fail whitebox container rsc lrmd connection."
+diff --git a/pengine/test10/bundle-order-startup-clone-2.dot b/pengine/test10/bundle-order-startup-clone-2.dot
+new file mode 100644
+index 0000000..af90261
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone-2.dot
+@@ -0,0 +1,335 @@
++digraph "g" {
++"galera-bundle-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = bold]
++"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-1_start_0 metal-2" -> "galera-bundle-1_monitor_60000 metal-2" [ style = bold]
++"galera-bundle-1_start_0 metal-2" -> "galera:1_monitor_20000 galera-bundle-1" [ style = bold]
++"galera-bundle-1_start_0 metal-2" -> "galera:1_monitor_30000 galera-bundle-1" [ style = bold]
++"galera-bundle-1_start_0 metal-2" -> "galera:1_start_0 galera-bundle-1" [ style = bold]
++"galera-bundle-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-2_start_0 metal-3" -> "galera-bundle-2_monitor_60000 metal-3" [ style = bold]
++"galera-bundle-2_start_0 metal-3" -> "galera:2_monitor_20000 galera-bundle-2" [ style = bold]
++"galera-bundle-2_start_0 metal-3" -> "galera:2_monitor_30000 galera-bundle-2" [ style = bold]
++"galera-bundle-2_start_0 metal-3" -> "galera:2_start_0 galera-bundle-2" [ style = bold]
++"galera-bundle-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-docker-0_monitor_60000 metal-1" [ style = bold]
++"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle_running_0" [ style = bold]
++"galera-bundle-docker-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-docker-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle-docker-1_monitor_60000 metal-2" [ style = bold]
++"galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle_running_0" [ style = bold]
++"galera-bundle-docker-1_start_0 metal-2" -> "galera:1_start_0 galera-bundle-1" [ style = bold]
++"galera-bundle-docker-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle-2_start_0 metal-3" [ style = bold]
++"galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle-docker-2_monitor_60000 metal-3" [ style = bold]
++"galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle_running_0" [ style = bold]
++"galera-bundle-docker-2_start_0 metal-3" -> "galera:2_start_0 galera-bundle-2" [ style = bold]
++"galera-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold]
++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold]
++"galera-bundle-master_start_0" -> "galera:1_start_0 galera-bundle-1" [ style = bold]
++"galera-bundle-master_start_0" -> "galera:2_start_0 galera-bundle-2" [ style = bold]
++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold]
++"galera-bundle_start_0" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold]
++"galera-bundle_start_0" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold]
++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold]
++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold]
++"galera:0_start_0 galera-bundle-0" -> "galera:1_start_0 galera-bundle-1" [ style = bold]
++"galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"galera:1_monitor_20000 galera-bundle-1" [ style=bold color="green" fontcolor="black"]
++"galera:1_monitor_30000 galera-bundle-1" [ style=bold color="green" fontcolor="black"]
++"galera:1_start_0 galera-bundle-1" -> "galera-bundle-master_running_0" [ style = bold]
++"galera:1_start_0 galera-bundle-1" -> "galera:1_monitor_20000 galera-bundle-1" [ style = bold]
++"galera:1_start_0 galera-bundle-1" -> "galera:1_monitor_30000 galera-bundle-1" [ style = bold]
++"galera:1_start_0 galera-bundle-1" -> "galera:2_start_0 galera-bundle-2" [ style = bold]
++"galera:1_start_0 galera-bundle-1" [ style=bold color="green" fontcolor="black"]
++"galera:2_monitor_20000 galera-bundle-2" [ style=bold color="green" fontcolor="black"]
++"galera:2_monitor_30000 galera-bundle-2" [ style=bold color="green" fontcolor="black"]
++"galera:2_start_0 galera-bundle-2" -> "galera-bundle-master_running_0" [ style = bold]
++"galera:2_start_0 galera-bundle-2" -> "galera:2_monitor_20000 galera-bundle-2" [ style = bold]
++"galera:2_start_0 galera-bundle-2" -> "galera:2_monitor_30000 galera-bundle-2" [ style = bold]
++"galera:2_start_0 galera-bundle-2" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_start_0 metal-1" -> "haproxy-bundle-docker-0_monitor_60000 metal-1" [ style = bold]
++"haproxy-bundle-docker-0_start_0 metal-1" -> "haproxy-bundle_running_0" [ style = bold]
++"haproxy-bundle-docker-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-1_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle-docker-1_monitor_0 metal-1" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-1_monitor_0 metal-1" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-1_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle-docker-1_monitor_0 metal-2" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-1_monitor_0 metal-2" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-1_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle-docker-1_monitor_0 metal-3" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-1_monitor_0 metal-3" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-1_start_0 metal-2" -> "haproxy-bundle-docker-1_monitor_60000 metal-2" [ style = bold]
++"haproxy-bundle-docker-1_start_0 metal-2" -> "haproxy-bundle_running_0" [ style = bold]
++"haproxy-bundle-docker-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-2_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle-docker-2_monitor_0 metal-1" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-2_monitor_0 metal-1" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-2_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle-docker-2_monitor_0 metal-2" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-2_monitor_0 metal-2" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-2_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle-docker-2_monitor_0 metal-3" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-2_monitor_0 metal-3" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-2_start_0 metal-3" -> "haproxy-bundle-docker-2_monitor_60000 metal-3" [ style = bold]
++"haproxy-bundle-docker-2_start_0 metal-3" -> "haproxy-bundle_running_0" [ style = bold]
++"haproxy-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle_running_0" -> "storage-clone_start_0" [ style = bold]
++"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold]
++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold]
++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold]
++"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-0_start_0 metal-1" -> "redis-bundle-0_monitor_60000 metal-1" [ style = bold]
++"redis-bundle-0_start_0 metal-1" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold]
++"redis-bundle-0_start_0 metal-1" -> "redis:0_promote_0 redis-bundle-0" [ style = bold]
++"redis-bundle-0_start_0 metal-1" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
++"redis-bundle-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-1_start_0 metal-2" -> "redis-bundle-1_monitor_60000 metal-2" [ style = bold]
++"redis-bundle-1_start_0 metal-2" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold]
++"redis-bundle-1_start_0 metal-2" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
++"redis-bundle-1_start_0 metal-2" -> "redis:1_start_0 redis-bundle-1" [ style = bold]
++"redis-bundle-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-2_start_0 metal-3" -> "redis-bundle-2_monitor_60000 metal-3" [ style = bold]
++"redis-bundle-2_start_0 metal-3" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold]
++"redis-bundle-2_start_0 metal-3" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
++"redis-bundle-2_start_0 metal-3" -> "redis:2_start_0 redis-bundle-2" [ style = bold]
++"redis-bundle-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle-docker-0_monitor_60000 metal-1" [ style = bold]
++"redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle_running_0" [ style = bold]
++"redis-bundle-docker-0_start_0 metal-1" -> "redis:0_promote_0 redis-bundle-0" [ style = bold]
++"redis-bundle-docker-0_start_0 metal-1" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
++"redis-bundle-docker-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle-docker-1_monitor_60000 metal-2" [ style = bold]
++"redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle_running_0" [ style = bold]
++"redis-bundle-docker-1_start_0 metal-2" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
++"redis-bundle-docker-1_start_0 metal-2" -> "redis:1_start_0 redis-bundle-1" [ style = bold]
++"redis-bundle-docker-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle-2_start_0 metal-3" [ style = bold]
++"redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle-docker-2_monitor_60000 metal-3" [ style = bold]
++"redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle_running_0" [ style = bold]
++"redis-bundle-docker-2_start_0 metal-3" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
++"redis-bundle-docker-2_start_0 metal-3" -> "redis:2_start_0 redis-bundle-2" [ style = bold]
++"redis-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-master_promote_0" -> "redis:0_promote_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_promote_0" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
++"redis-bundle-master_promote_0" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
++"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold]
++"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_running_0" -> "redis-bundle-master_promote_0" [ style = bold]
++"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold]
++"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold]
++"redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_start_0" -> "redis:1_start_0 redis-bundle-1" [ style = bold]
++"redis-bundle-master_start_0" -> "redis:2_start_0 redis-bundle-2" [ style = bold]
++"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold]
++"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_promoted_0" -> "storage-clone_start_0" [ style = bold]
++"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_running_0" -> "redis-bundle_promote_0" [ style = bold]
++"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_start_0" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold]
++"redis-bundle_start_0" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold]
++"redis-bundle_start_0" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold]
++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold]
++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis:0_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis:0_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold]
++"redis:0_promote_0 redis-bundle-0" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold]
++"redis:0_promote_0 redis-bundle-0" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
++"redis:0_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis:0_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = bold]
++"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold]
++"redis:0_start_0 redis-bundle-0" -> "redis:0_promote_0 redis-bundle-0" [ style = bold]
++"redis:0_start_0 redis-bundle-0" -> "redis:1_start_0 redis-bundle-1" [ style = bold]
++"redis:0_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis:1_monitor_20000 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
++"redis:1_promote_0 redis-bundle-1" -> "redis-bundle-master_promoted_0" [ style = bold]
++"redis:1_promote_0 redis-bundle-1" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold]
++"redis:1_promote_0 redis-bundle-1" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
++"redis:1_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
++"redis:1_start_0 redis-bundle-1" -> "redis-bundle-master_running_0" [ style = bold]
++"redis:1_start_0 redis-bundle-1" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold]
++"redis:1_start_0 redis-bundle-1" -> "redis:1_promote_0 redis-bundle-1" [ style = bold]
++"redis:1_start_0 redis-bundle-1" -> "redis:2_start_0 redis-bundle-2" [ style = bold]
++"redis:1_start_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"]
++"redis:2_monitor_20000 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
++"redis:2_promote_0 redis-bundle-2" -> "redis-bundle-master_promoted_0" [ style = bold]
++"redis:2_promote_0 redis-bundle-2" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold]
++"redis:2_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
++"redis:2_start_0 redis-bundle-2" -> "redis-bundle-master_running_0" [ style = bold]
++"redis:2_start_0 redis-bundle-2" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold]
++"redis:2_start_0 redis-bundle-2" -> "redis:2_promote_0 redis-bundle-2" [ style = bold]
++"redis:2_start_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"]
++"storage-clone_confirmed-post_notify_running_0" -> "galera-bundle_start_0" [ style = bold]
++"storage-clone_confirmed-post_notify_running_0" -> "storage:0_monitor_30000 metal-1" [ style = bold]
++"storage-clone_confirmed-post_notify_running_0" -> "storage:1_monitor_30000 metal-2" [ style = bold]
++"storage-clone_confirmed-post_notify_running_0" -> "storage:2_monitor_30000 metal-3" [ style = bold]
++"storage-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_post_notify_running_0" [ style = bold]
++"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_start_0" [ style = bold]
++"storage-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_post_notify_running_0" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold]
++"storage-clone_post_notify_running_0" -> "storage:0_post_notify_start_0 metal-1" [ style = bold]
++"storage-clone_post_notify_running_0" -> "storage:1_post_notify_start_0 metal-2" [ style = bold]
++"storage-clone_post_notify_running_0" -> "storage:2_post_notify_start_0 metal-3" [ style = bold]
++"storage-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_pre_notify_start_0" -> "storage-clone_confirmed-pre_notify_start_0" [ style = bold]
++"storage-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_running_0" -> "storage-clone_post_notify_running_0" [ style = bold]
++"storage-clone_running_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_start_0" -> "storage-clone_running_0" [ style = bold]
++"storage-clone_start_0" -> "storage:0_start_0 metal-1" [ style = bold]
++"storage-clone_start_0" -> "storage:1_start_0 metal-2" [ style = bold]
++"storage-clone_start_0" -> "storage:2_start_0 metal-3" [ style = bold]
++"storage-clone_start_0" [ style=bold color="green" fontcolor="orange"]
++"storage:0_monitor_0 metal-1" -> "storage-clone_start_0" [ style = bold]
++"storage:0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"storage:0_monitor_30000 metal-1" [ style=bold color="green" fontcolor="black"]
++"storage:0_post_notify_start_0 metal-1" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold]
++"storage:0_post_notify_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"storage:0_start_0 metal-1" -> "storage-clone_running_0" [ style = bold]
++"storage:0_start_0 metal-1" -> "storage:0_monitor_30000 metal-1" [ style = bold]
++"storage:0_start_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"storage:1_monitor_0 metal-2" -> "storage-clone_start_0" [ style = bold]
++"storage:1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"storage:1_monitor_30000 metal-2" [ style=bold color="green" fontcolor="black"]
++"storage:1_post_notify_start_0 metal-2" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold]
++"storage:1_post_notify_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"storage:1_start_0 metal-2" -> "storage-clone_running_0" [ style = bold]
++"storage:1_start_0 metal-2" -> "storage:1_monitor_30000 metal-2" [ style = bold]
++"storage:1_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"storage:2_monitor_0 metal-3" -> "storage-clone_start_0" [ style = bold]
++"storage:2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"storage:2_monitor_30000 metal-3" [ style=bold color="green" fontcolor="black"]
++"storage:2_post_notify_start_0 metal-3" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold]
++"storage:2_post_notify_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"storage:2_start_0 metal-3" -> "storage-clone_running_0" [ style = bold]
++"storage:2_start_0 metal-3" -> "storage:2_monitor_30000 metal-3" [ style = bold]
++"storage:2_start_0 metal-3" [ style=bold color="green" fontcolor="black"]
++}
+diff --git a/pengine/test10/bundle-order-startup-clone-2.exp b/pengine/test10/bundle-order-startup-clone-2.exp
+new file mode 100644
+index 0000000..d05eb96
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone-2.exp
+@@ -0,0 +1,1697 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0" priority="1000000">
++ <action_set>
++ <rsc_op id="145" operation="notify" operation_key="storage:0_post_notify_start_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="33" operation="monitor" operation_key="storage:0_monitor_30000" on_node="metal-1" on_node_uuid="1">
++ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="32" operation="start" operation_key="storage:0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="2">
++ <action_set>
++ <rsc_op id="32" operation="start" operation_key="storage:0_start_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="2" operation="monitor" operation_key="storage:0_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="4" priority="1000000">
++ <action_set>
++ <rsc_op id="146" operation="notify" operation_key="storage:1_post_notify_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="35" operation="monitor" operation_key="storage:1_monitor_30000" on_node="metal-2" on_node_uuid="2">
++ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="34" operation="start" operation_key="storage:1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="6">
++ <action_set>
++ <rsc_op id="34" operation="start" operation_key="storage:1_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <rsc_op id="12" operation="monitor" operation_key="storage:1_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="8" priority="1000000">
++ <action_set>
++ <rsc_op id="147" operation="notify" operation_key="storage:2_post_notify_start_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="start" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_notify_type="post" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <rsc_op id="37" operation="monitor" operation_key="storage:2_monitor_30000" on_node="metal-3" on_node_uuid="3">
++ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_notify="true" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="36" operation="start" operation_key="storage:2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="10">
++ <action_set>
++ <rsc_op id="36" operation="start" operation_key="storage:2_start_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource=" " CRM_meta_notify_active_uname=" " CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:0 storage:1 storage:2 storage:3" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource="storage:0 storage:1 storage:2" CRM_meta_notify_start_uname="metal-1 metal-2 metal-3" CRM_meta_notify_stop_resource=" " CRM_meta_notify_stop_uname=" " CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <rsc_op id="22" operation="monitor" operation_key="storage:2_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="12" priority="1000000">
++ <action_set>
++ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="145" operation="notify" operation_key="storage:0_post_notify_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="146" operation="notify" operation_key="storage:1_post_notify_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="147" operation="notify" operation_key="storage:2_post_notify_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13" priority="1000000">
++ <action_set>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_post_notify_running_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="running" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="start" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="39" operation="running" operation_key="storage-clone_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="41" operation="notified" operation_key="storage-clone_confirmed-pre_notify_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14">
++ <action_set>
++ <pseudo_event id="41" operation="notified" operation_key="storage-clone_confirmed-pre_notify_start_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="40" operation="notify" operation_key="storage-clone_pre_notify_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15">
++ <action_set>
++ <pseudo_event id="40" operation="notify" operation_key="storage-clone_pre_notify_start_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="start" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="start" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="16" priority="1000000">
++ <action_set>
++ <pseudo_event id="39" operation="running" operation_key="storage-clone_running_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="32" operation="start" operation_key="storage:0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="34" operation="start" operation_key="storage:1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="36" operation="start" operation_key="storage:2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="17">
++ <action_set>
++ <pseudo_event id="38" operation="start" operation_key="storage-clone_start_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="2" operation="monitor" operation_key="storage:0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="12" operation="monitor" operation_key="storage:1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="22" operation="monitor" operation_key="storage:2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="41" operation="notified" operation_key="storage-clone_confirmed-pre_notify_start_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="94" operation="running" operation_key="haproxy-bundle_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="131" operation="promoted" operation_key="redis-bundle_promoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="18">
++ <action_set>
++ <rsc_op id="68" operation="monitor" operation_key="galera:0_monitor_30000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_role="Slave" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="19">
++ <action_set>
++ <rsc_op id="67" operation="monitor" operation_key="galera:0_monitor_20000" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20">
++ <action_set>
++ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="21">
++ <action_set>
++ <rsc_op id="51" operation="monitor" operation_key="galera-bundle-docker-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="22">
++ <action_set>
++ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="4" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="5" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="14" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="15" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="23" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="24" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="25" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="23">
++ <action_set>
++ <rsc_op id="23" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="24">
++ <action_set>
++ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="25">
++ <action_set>
++ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="26">
++ <action_set>
++ <rsc_op id="53" operation="monitor" operation_key="galera-bundle-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="metal-1" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="27">
++ <action_set>
++ <rsc_op id="52" operation="start" operation_key="galera-bundle-0_start_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="metal-1" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="28">
++ <action_set>
++ <rsc_op id="71" operation="monitor" operation_key="galera:1_monitor_30000" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2">
++ <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-1" CRM_meta_on_node_uuid="galera-bundle-1" CRM_meta_role="Slave" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="29">
++ <action_set>
++ <rsc_op id="70" operation="monitor" operation_key="galera:1_monitor_20000" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2">
++ <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-1" CRM_meta_on_node_uuid="galera-bundle-1" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="30">
++ <action_set>
++ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2">
++ <primitive id="galera" long-id="galera:1" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-1" CRM_meta_on_node_uuid="galera-bundle-1" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="31">
++ <action_set>
++ <rsc_op id="55" operation="monitor" operation_key="galera-bundle-docker-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
++ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="32">
++ <action_set>
++ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="4" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="5" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="14" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="15" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="23" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="24" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="25" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="33">
++ <action_set>
++ <rsc_op id="24" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="34">
++ <action_set>
++ <rsc_op id="14" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="35">
++ <action_set>
++ <rsc_op id="4" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-1:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="36">
++ <action_set>
++ <rsc_op id="57" operation="monitor" operation_key="galera-bundle-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
++ <primitive id="galera-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-1" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="37">
++ <action_set>
++ <rsc_op id="56" operation="start" operation_key="galera-bundle-1_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="galera-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-1" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="38">
++ <action_set>
++ <rsc_op id="74" operation="monitor" operation_key="galera:2_monitor_30000" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3">
++ <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="30000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-2" CRM_meta_on_node_uuid="galera-bundle-2" CRM_meta_role="Slave" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="72" operation="start" operation_key="galera:2_start_0" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="39">
++ <action_set>
++ <rsc_op id="73" operation="monitor" operation_key="galera:2_monitor_20000" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3">
++ <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-2" CRM_meta_on_node_uuid="galera-bundle-2" CRM_meta_timeout="30000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="72" operation="start" operation_key="galera:2_start_0" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="40">
++ <action_set>
++ <rsc_op id="72" operation="start" operation_key="galera:2_start_0" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3">
++ <primitive id="galera" long-id="galera:2" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-2" CRM_meta_on_node_uuid="galera-bundle-2" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="41">
++ <action_set>
++ <rsc_op id="59" operation="monitor" operation_key="galera-bundle-docker-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
++ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="42">
++ <action_set>
++ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="4" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="5" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="13" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="14" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="15" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="23" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="24" operation="monitor" operation_key="galera-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="25" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="43">
++ <action_set>
++ <rsc_op id="25" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="44">
++ <action_set>
++ <rsc_op id="15" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="45">
++ <action_set>
++ <rsc_op id="5" operation="monitor" operation_key="galera-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-2:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="46">
++ <action_set>
++ <rsc_op id="61" operation="monitor" operation_key="galera-bundle-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
++ <primitive id="galera-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-2" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" addr="metal-3" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="47">
++ <action_set>
++ <rsc_op id="60" operation="start" operation_key="galera-bundle-2_start_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="galera-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-2" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" addr="metal-3" port="3123"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="48">
++ <action_set>
++ <rsc_op id="88" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="87" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="49">
++ <action_set>
++ <rsc_op id="87" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="6" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="7" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="16" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="17" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="18" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="26" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="27" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="28" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="93" operation="start" operation_key="haproxy-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="50">
++ <action_set>
++ <rsc_op id="26" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="51">
++ <action_set>
++ <rsc_op id="16" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="52">
++ <action_set>
++ <rsc_op id="6" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="53">
++ <action_set>
++ <rsc_op id="90" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
++ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="54">
++ <action_set>
++ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="6" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="7" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="16" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="17" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="18" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="26" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="27" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="28" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="93" operation="start" operation_key="haproxy-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="55">
++ <action_set>
++ <rsc_op id="27" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="56">
++ <action_set>
++ <rsc_op id="17" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="57">
++ <action_set>
++ <rsc_op id="7" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="58">
++ <action_set>
++ <rsc_op id="92" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
++ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="91" operation="start" operation_key="haproxy-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="59">
++ <action_set>
++ <rsc_op id="91" operation="start" operation_key="haproxy-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="6" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="7" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="16" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="17" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="18" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="26" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="27" operation="monitor" operation_key="haproxy-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="28" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="93" operation="start" operation_key="haproxy-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="60">
++ <action_set>
++ <rsc_op id="28" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="61">
++ <action_set>
++ <rsc_op id="18" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="62">
++ <action_set>
++ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="63">
++ <action_set>
++ <rsc_op id="115" operation="monitor" operation_key="redis:0_monitor_20000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="114" operation="promote" operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="64">
++ <action_set>
++ <rsc_op id="114" operation="promote" operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="120000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="126" operation="promote" operation_key="redis-bundle-master_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="65">
++ <action_set>
++ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="200000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="66">
++ <action_set>
++ <rsc_op id="98" operation="monitor" operation_key="redis-bundle-docker-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="67">
++ <action_set>
++ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="10" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="11" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="19" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="20" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="21" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="29" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="30" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="31" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="68">
++ <action_set>
++ <rsc_op id="29" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="69">
++ <action_set>
++ <rsc_op id="19" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="70">
++ <action_set>
++ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="71">
++ <action_set>
++ <rsc_op id="100" operation="monitor" operation_key="redis-bundle-0_monitor_60000" on_node="metal-1" on_node_uuid="1">
++ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="metal-1" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="72">
++ <action_set>
++ <rsc_op id="99" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="metal-1" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="73">
++ <action_set>
++ <rsc_op id="118" operation="monitor" operation_key="redis:1_monitor_20000" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2">
++ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="117" operation="promote" operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="74">
++ <action_set>
++ <rsc_op id="117" operation="promote" operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2">
++ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_timeout="120000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="114" operation="promote" operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="126" operation="promote" operation_key="redis-bundle-master_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="75">
++ <action_set>
++ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2">
++ <primitive id="redis" long-id="redis:1" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-1" CRM_meta_on_node_uuid="redis-bundle-1" CRM_meta_timeout="200000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="76">
++ <action_set>
++ <rsc_op id="102" operation="monitor" operation_key="redis-bundle-docker-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="77">
++ <action_set>
++ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="10" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="11" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="19" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="20" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="21" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="29" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="30" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="31" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="78">
++ <action_set>
++ <rsc_op id="30" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="79">
++ <action_set>
++ <rsc_op id="20" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="80">
++ <action_set>
++ <rsc_op id="10" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-1" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-1:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="81">
++ <action_set>
++ <rsc_op id="104" operation="monitor" operation_key="redis-bundle-1_monitor_60000" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-1" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="82">
++ <action_set>
++ <rsc_op id="103" operation="start" operation_key="redis-bundle-1_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-1" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="83">
++ <action_set>
++ <rsc_op id="121" operation="monitor" operation_key="redis:2_monitor_20000" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3">
++ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="20000" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_op_target_rc="8" CRM_meta_role="Master" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="119" operation="start" operation_key="redis:2_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="120" operation="promote" operation_key="redis:2_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="84">
++ <action_set>
++ <rsc_op id="120" operation="promote" operation_key="redis:2_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3">
++ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="promote" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_timeout="120000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="117" operation="promote" operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="119" operation="start" operation_key="redis:2_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="126" operation="promote" operation_key="redis-bundle-master_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="85">
++ <action_set>
++ <rsc_op id="119" operation="start" operation_key="redis:2_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3">
++ <primitive id="redis" long-id="redis:2" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-2" CRM_meta_on_node_uuid="redis-bundle-2" CRM_meta_timeout="200000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="86">
++ <action_set>
++ <rsc_op id="106" operation="monitor" operation_key="redis-bundle-docker-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
++ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="87">
++ <action_set>
++ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="10" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="11" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="19" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="20" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="21" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="29" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="30" operation="monitor" operation_key="redis-bundle-docker-1_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="31" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="88">
++ <action_set>
++ <rsc_op id="31" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="89">
++ <action_set>
++ <rsc_op id="21" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="90">
++ <action_set>
++ <rsc_op id="11" operation="monitor" operation_key="redis-bundle-docker-2_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-2" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-2:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="91">
++ <action_set>
++ <rsc_op id="108" operation="monitor" operation_key="redis-bundle-2_monitor_60000" on_node="metal-3" on_node_uuid="3">
++ <primitive id="redis-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-2" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" addr="metal-3" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="92">
++ <action_set>
++ <rsc_op id="107" operation="start" operation_key="redis-bundle-2_start_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="redis-bundle-2" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-2" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" addr="metal-3" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="93" priority="1000000">
++ <action_set>
++ <pseudo_event id="131" operation="promoted" operation_key="redis-bundle_promoted_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="127" operation="promoted" operation_key="redis-bundle-master_promoted_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="94">
++ <action_set>
++ <pseudo_event id="130" operation="promote" operation_key="redis-bundle_promote_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="110" operation="running" operation_key="redis-bundle_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="95" priority="1000000">
++ <action_set>
++ <pseudo_event id="127" operation="promoted" operation_key="redis-bundle-master_promoted_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="114" operation="promote" operation_key="redis:0_promote_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="117" operation="promote" operation_key="redis:1_promote_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="120" operation="promote" operation_key="redis:2_promote_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="96">
++ <action_set>
++ <pseudo_event id="126" operation="promote" operation_key="redis-bundle-master_promote_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="123" operation="running" operation_key="redis-bundle-master_running_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="130" operation="promote" operation_key="redis-bundle_promote_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="97" priority="1000000">
++ <action_set>
++ <pseudo_event id="123" operation="running" operation_key="redis-bundle-master_running_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="113" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="116" operation="start" operation_key="redis:1_start_0" on_node="redis-bundle-1" on_node_uuid="redis-bundle-1" router_node="metal-2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="119" operation="start" operation_key="redis:2_start_0" on_node="redis-bundle-2" on_node_uuid="redis-bundle-2" router_node="metal-3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="98">
++ <action_set>
++ <pseudo_event id="122" operation="start" operation_key="redis-bundle-master_start_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="99" priority="1000000">
++ <action_set>
++ <pseudo_event id="110" operation="running" operation_key="redis-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="97" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="101" operation="start" operation_key="redis-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="105" operation="start" operation_key="redis-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="123" operation="running" operation_key="redis-bundle-master_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="100">
++ <action_set>
++ <pseudo_event id="109" operation="start" operation_key="redis-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="101" priority="1000000">
++ <action_set>
++ <pseudo_event id="94" operation="running" operation_key="haproxy-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="87" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="89" operation="start" operation_key="haproxy-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="91" operation="start" operation_key="haproxy-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="102">
++ <action_set>
++ <pseudo_event id="93" operation="start" operation_key="haproxy-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="103" priority="1000000">
++ <action_set>
++ <pseudo_event id="76" operation="running" operation_key="galera-bundle-master_running_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="66" operation="start" operation_key="galera:0_start_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="69" operation="start" operation_key="galera:1_start_0" on_node="galera-bundle-1" on_node_uuid="galera-bundle-1" router_node="metal-2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="72" operation="start" operation_key="galera:2_start_0" on_node="galera-bundle-2" on_node_uuid="galera-bundle-2" router_node="metal-3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="104">
++ <action_set>
++ <pseudo_event id="75" operation="start" operation_key="galera-bundle-master_start_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="105" priority="1000000">
++ <action_set>
++ <pseudo_event id="63" operation="running" operation_key="galera-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="50" operation="start" operation_key="galera-bundle-docker-0_start_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="54" operation="start" operation_key="galera-bundle-docker-1_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="58" operation="start" operation_key="galera-bundle-docker-2_start_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="76" operation="running" operation_key="galera-bundle-master_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="106">
++ <action_set>
++ <pseudo_event id="62" operation="start" operation_key="galera-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-post_notify_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/bundle-order-startup-clone-2.scores b/pengine/test10/bundle-order-startup-clone-2.scores
+new file mode 100644
+index 0000000..493cd87
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone-2.scores
+@@ -0,0 +1,584 @@
++Allocation scores:
++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
++clone_color: galera-bundle-master allocation score on galera-bundle-1: 0
++clone_color: galera-bundle-master allocation score on galera-bundle-2: 0
++clone_color: galera-bundle-master allocation score on metal-1: -INFINITY
++clone_color: galera-bundle-master allocation score on metal-2: -INFINITY
++clone_color: galera-bundle-master allocation score on metal-3: -INFINITY
++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
++clone_color: galera:0 allocation score on galera-bundle-1: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-2: -INFINITY
++clone_color: galera:0 allocation score on metal-1: -INFINITY
++clone_color: galera:0 allocation score on metal-2: -INFINITY
++clone_color: galera:0 allocation score on metal-3: -INFINITY
++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:1 allocation score on galera-bundle-0: -INFINITY
++clone_color: galera:1 allocation score on galera-bundle-1: INFINITY
++clone_color: galera:1 allocation score on galera-bundle-2: -INFINITY
++clone_color: galera:1 allocation score on metal-1: -INFINITY
++clone_color: galera:1 allocation score on metal-2: -INFINITY
++clone_color: galera:1 allocation score on metal-3: -INFINITY
++clone_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:2 allocation score on galera-bundle-0: -INFINITY
++clone_color: galera:2 allocation score on galera-bundle-1: -INFINITY
++clone_color: galera:2 allocation score on galera-bundle-2: INFINITY
++clone_color: galera:2 allocation score on metal-1: -INFINITY
++clone_color: galera:2 allocation score on metal-2: -INFINITY
++clone_color: galera:2 allocation score on metal-3: -INFINITY
++clone_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY
++clone_color: redis-bundle-master allocation score on metal-1: -INFINITY
++clone_color: redis-bundle-master allocation score on metal-2: -INFINITY
++clone_color: redis-bundle-master allocation score on metal-3: -INFINITY
++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis-bundle-master allocation score on redis-bundle-1: 0
++clone_color: redis-bundle-master allocation score on redis-bundle-2: 0
++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-1: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-2: -INFINITY
++clone_color: redis:0 allocation score on metal-1: -INFINITY
++clone_color: redis:0 allocation score on metal-2: -INFINITY
++clone_color: redis:0 allocation score on metal-3: -INFINITY
++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: redis:0 allocation score on redis-bundle-1: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-2: -INFINITY
++clone_color: redis:1 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:1 allocation score on galera-bundle-1: -INFINITY
++clone_color: redis:1 allocation score on galera-bundle-2: -INFINITY
++clone_color: redis:1 allocation score on metal-1: -INFINITY
++clone_color: redis:1 allocation score on metal-2: -INFINITY
++clone_color: redis:1 allocation score on metal-3: -INFINITY
++clone_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:1 allocation score on redis-bundle-0: -INFINITY
++clone_color: redis:1 allocation score on redis-bundle-1: INFINITY
++clone_color: redis:1 allocation score on redis-bundle-2: -INFINITY
++clone_color: redis:2 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:2 allocation score on galera-bundle-1: -INFINITY
++clone_color: redis:2 allocation score on galera-bundle-2: -INFINITY
++clone_color: redis:2 allocation score on metal-1: -INFINITY
++clone_color: redis:2 allocation score on metal-2: -INFINITY
++clone_color: redis:2 allocation score on metal-3: -INFINITY
++clone_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:2 allocation score on redis-bundle-0: -INFINITY
++clone_color: redis:2 allocation score on redis-bundle-1: -INFINITY
++clone_color: redis:2 allocation score on redis-bundle-2: INFINITY
++clone_color: storage-clone allocation score on metal-1: 0
++clone_color: storage-clone allocation score on metal-2: 0
++clone_color: storage-clone allocation score on metal-3: 0
++clone_color: storage-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:0 allocation score on metal-1: 0
++clone_color: storage:0 allocation score on metal-2: 0
++clone_color: storage:0 allocation score on metal-3: 0
++clone_color: storage:0 allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:1 allocation score on metal-1: 0
++clone_color: storage:1 allocation score on metal-2: 0
++clone_color: storage:1 allocation score on metal-3: 0
++clone_color: storage:1 allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:2 allocation score on metal-1: 0
++clone_color: storage:2 allocation score on metal-2: 0
++clone_color: storage:2 allocation score on metal-3: 0
++clone_color: storage:2 allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:3 allocation score on metal-1: 0
++clone_color: storage:3 allocation score on metal-2: 0
++clone_color: storage:3 allocation score on metal-3: 0
++clone_color: storage:3 allocation score on rabbitmq-bundle-0: 0
++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle allocation score on metal-1: 0
++container_color: galera-bundle allocation score on metal-2: 0
++container_color: galera-bundle allocation score on metal-3: 0
++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-0 allocation score on metal-1: 0
++container_color: galera-bundle-0 allocation score on metal-2: 0
++container_color: galera-bundle-0 allocation score on metal-3: 0
++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-1 allocation score on metal-1: 0
++container_color: galera-bundle-1 allocation score on metal-2: 0
++container_color: galera-bundle-1 allocation score on metal-3: 0
++container_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-2 allocation score on metal-1: 0
++container_color: galera-bundle-2 allocation score on metal-2: 0
++container_color: galera-bundle-2 allocation score on metal-3: 0
++container_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on metal-1: 0
++container_color: galera-bundle-docker-0 allocation score on metal-2: 0
++container_color: galera-bundle-docker-0 allocation score on metal-3: 0
++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-docker-1 allocation score on metal-1: 0
++container_color: galera-bundle-docker-1 allocation score on metal-2: 0
++container_color: galera-bundle-docker-1 allocation score on metal-3: 0
++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-docker-2 allocation score on metal-1: 0
++container_color: galera-bundle-docker-2 allocation score on metal-2: 0
++container_color: galera-bundle-docker-2 allocation score on metal-3: 0
++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-master allocation score on metal-1: 0
++container_color: galera-bundle-master allocation score on metal-2: 0
++container_color: galera-bundle-master allocation score on metal-3: 0
++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: 0
++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:0 allocation score on galera-bundle-1: -INFINITY
++container_color: galera:0 allocation score on galera-bundle-2: -INFINITY
++container_color: galera:0 allocation score on metal-1: 0
++container_color: galera:0 allocation score on metal-2: 0
++container_color: galera:0 allocation score on metal-3: 0
++container_color: galera:0 allocation score on rabbitmq-bundle-0: 0
++container_color: galera:1 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:1 allocation score on galera-bundle-1: -INFINITY
++container_color: galera:1 allocation score on galera-bundle-2: -INFINITY
++container_color: galera:1 allocation score on metal-1: 0
++container_color: galera:1 allocation score on metal-2: 0
++container_color: galera:1 allocation score on metal-3: 0
++container_color: galera:1 allocation score on rabbitmq-bundle-0: 0
++container_color: galera:2 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:2 allocation score on galera-bundle-1: -INFINITY
++container_color: galera:2 allocation score on galera-bundle-2: -INFINITY
++container_color: galera:2 allocation score on metal-1: 0
++container_color: galera:2 allocation score on metal-2: 0
++container_color: galera:2 allocation score on metal-3: 0
++container_color: galera:2 allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: 0
++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle allocation score on metal-1: 0
++container_color: redis-bundle allocation score on metal-2: 0
++container_color: redis-bundle allocation score on metal-3: 0
++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-0 allocation score on metal-1: 0
++container_color: redis-bundle-0 allocation score on metal-2: 0
++container_color: redis-bundle-0 allocation score on metal-3: 0
++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-1 allocation score on metal-1: 0
++container_color: redis-bundle-1 allocation score on metal-2: 0
++container_color: redis-bundle-1 allocation score on metal-3: 0
++container_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-2 allocation score on metal-1: 0
++container_color: redis-bundle-2 allocation score on metal-2: 0
++container_color: redis-bundle-2 allocation score on metal-3: 0
++container_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on metal-1: 0
++container_color: redis-bundle-docker-0 allocation score on metal-2: 0
++container_color: redis-bundle-docker-0 allocation score on metal-3: 0
++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on metal-1: 0
++container_color: redis-bundle-docker-1 allocation score on metal-2: 0
++container_color: redis-bundle-docker-1 allocation score on metal-3: 0
++container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on metal-1: 0
++container_color: redis-bundle-docker-2 allocation score on metal-2: 0
++container_color: redis-bundle-docker-2 allocation score on metal-3: 0
++container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-master allocation score on metal-1: 0
++container_color: redis-bundle-master allocation score on metal-2: 0
++container_color: redis-bundle-master allocation score on metal-3: 0
++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: 0
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-2: -INFINITY
++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:0 allocation score on galera-bundle-1: -INFINITY
++container_color: redis:0 allocation score on galera-bundle-2: -INFINITY
++container_color: redis:0 allocation score on metal-1: 0
++container_color: redis:0 allocation score on metal-2: 0
++container_color: redis:0 allocation score on metal-3: 0
++container_color: redis:0 allocation score on rabbitmq-bundle-0: 0
++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-1: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-2: -INFINITY
++container_color: redis:1 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:1 allocation score on galera-bundle-1: -INFINITY
++container_color: redis:1 allocation score on galera-bundle-2: -INFINITY
++container_color: redis:1 allocation score on metal-1: 0
++container_color: redis:1 allocation score on metal-2: 0
++container_color: redis:1 allocation score on metal-3: 0
++container_color: redis:1 allocation score on rabbitmq-bundle-0: 0
++container_color: redis:1 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:1 allocation score on redis-bundle-1: -INFINITY
++container_color: redis:1 allocation score on redis-bundle-2: -INFINITY
++container_color: redis:2 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:2 allocation score on galera-bundle-1: -INFINITY
++container_color: redis:2 allocation score on galera-bundle-2: -INFINITY
++container_color: redis:2 allocation score on metal-1: 0
++container_color: redis:2 allocation score on metal-2: 0
++container_color: redis:2 allocation score on metal-3: 0
++container_color: redis:2 allocation score on rabbitmq-bundle-0: 0
++container_color: redis:2 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:2 allocation score on redis-bundle-1: -INFINITY
++container_color: redis:2 allocation score on redis-bundle-2: -INFINITY
++galera:0 promotion score on galera-bundle-0: -1
++galera:1 promotion score on galera-bundle-1: -1
++galera:2 promotion score on galera-bundle-2: -1
++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-0 allocation score on metal-1: 10000
++native_color: galera-bundle-0 allocation score on metal-2: 0
++native_color: galera-bundle-0 allocation score on metal-3: 0
++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-1 allocation score on metal-1: 0
++native_color: galera-bundle-1 allocation score on metal-2: 10000
++native_color: galera-bundle-1 allocation score on metal-3: 0
++native_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-2 allocation score on metal-1: 0
++native_color: galera-bundle-2 allocation score on metal-2: 0
++native_color: galera-bundle-2 allocation score on metal-3: 10000
++native_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on metal-1: 0
++native_color: galera-bundle-docker-0 allocation score on metal-2: 0
++native_color: galera-bundle-docker-0 allocation score on metal-3: 0
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on metal-1: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on metal-2: 0
++native_color: galera-bundle-docker-1 allocation score on metal-3: 0
++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on metal-1: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on metal-2: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on metal-3: 0
++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:0 allocation score on galera-bundle-0: INFINITY
++native_color: galera:0 allocation score on galera-bundle-1: -INFINITY
++native_color: galera:0 allocation score on galera-bundle-2: -INFINITY
++native_color: galera:0 allocation score on metal-1: -INFINITY
++native_color: galera:0 allocation score on metal-2: -INFINITY
++native_color: galera:0 allocation score on metal-3: -INFINITY
++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:1 allocation score on galera-bundle-0: -INFINITY
++native_color: galera:1 allocation score on galera-bundle-1: INFINITY
++native_color: galera:1 allocation score on galera-bundle-2: -INFINITY
++native_color: galera:1 allocation score on metal-1: -INFINITY
++native_color: galera:1 allocation score on metal-2: -INFINITY
++native_color: galera:1 allocation score on metal-3: -INFINITY
++native_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:2 allocation score on galera-bundle-0: -INFINITY
++native_color: galera:2 allocation score on galera-bundle-1: -INFINITY
++native_color: galera:2 allocation score on galera-bundle-2: INFINITY
++native_color: galera:2 allocation score on metal-1: -INFINITY
++native_color: galera:2 allocation score on metal-2: -INFINITY
++native_color: galera:2 allocation score on metal-3: -INFINITY
++native_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
++native_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++native_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on metal-2: 0
++native_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on metal-3: 0
++native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-0 allocation score on metal-1: 10000
++native_color: redis-bundle-0 allocation score on metal-2: 0
++native_color: redis-bundle-0 allocation score on metal-3: 0
++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-1 allocation score on metal-1: 0
++native_color: redis-bundle-1 allocation score on metal-2: 10000
++native_color: redis-bundle-1 allocation score on metal-3: 0
++native_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-2 allocation score on metal-1: 0
++native_color: redis-bundle-2 allocation score on metal-2: 0
++native_color: redis-bundle-2 allocation score on metal-3: 10000
++native_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on metal-1: 0
++native_color: redis-bundle-docker-0 allocation score on metal-2: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on metal-3: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on metal-1: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on metal-2: 0
++native_color: redis-bundle-docker-1 allocation score on metal-3: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on metal-1: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on metal-2: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on metal-3: 0
++native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY
++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:0 allocation score on galera-bundle-1: -INFINITY
++native_color: redis:0 allocation score on galera-bundle-2: -INFINITY
++native_color: redis:0 allocation score on metal-1: -INFINITY
++native_color: redis:0 allocation score on metal-2: -INFINITY
++native_color: redis:0 allocation score on metal-3: -INFINITY
++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: redis:0 allocation score on redis-bundle-1: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-2: -INFINITY
++native_color: redis:1 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:1 allocation score on galera-bundle-1: -INFINITY
++native_color: redis:1 allocation score on galera-bundle-2: -INFINITY
++native_color: redis:1 allocation score on metal-1: -INFINITY
++native_color: redis:1 allocation score on metal-2: -INFINITY
++native_color: redis:1 allocation score on metal-3: -INFINITY
++native_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:1 allocation score on redis-bundle-0: -INFINITY
++native_color: redis:1 allocation score on redis-bundle-1: INFINITY
++native_color: redis:1 allocation score on redis-bundle-2: -INFINITY
++native_color: redis:2 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:2 allocation score on galera-bundle-1: -INFINITY
++native_color: redis:2 allocation score on galera-bundle-2: -INFINITY
++native_color: redis:2 allocation score on metal-1: -INFINITY
++native_color: redis:2 allocation score on metal-2: -INFINITY
++native_color: redis:2 allocation score on metal-3: -INFINITY
++native_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:2 allocation score on redis-bundle-0: -INFINITY
++native_color: redis:2 allocation score on redis-bundle-1: -INFINITY
++native_color: redis:2 allocation score on redis-bundle-2: INFINITY
++native_color: storage:0 allocation score on metal-1: 0
++native_color: storage:0 allocation score on metal-2: 0
++native_color: storage:0 allocation score on metal-3: 0
++native_color: storage:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: storage:1 allocation score on metal-1: -INFINITY
++native_color: storage:1 allocation score on metal-2: 0
++native_color: storage:1 allocation score on metal-3: 0
++native_color: storage:1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: storage:2 allocation score on metal-1: -INFINITY
++native_color: storage:2 allocation score on metal-2: -INFINITY
++native_color: storage:2 allocation score on metal-3: 0
++native_color: storage:2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: storage:3 allocation score on metal-1: -INFINITY
++native_color: storage:3 allocation score on metal-2: -INFINITY
++native_color: storage:3 allocation score on metal-3: -INFINITY
++native_color: storage:3 allocation score on rabbitmq-bundle-0: -INFINITY
++redis:0 promotion score on redis-bundle-0: 99
++redis:1 promotion score on redis-bundle-1: 99
++redis:2 promotion score on redis-bundle-2: 99
+diff --git a/pengine/test10/bundle-order-startup-clone-2.summary b/pengine/test10/bundle-order-startup-clone-2.summary
+new file mode 100644
+index 0000000..e23d933
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone-2.summary
+@@ -0,0 +1,179 @@
++
++Current cluster status:
++Online: [ metal-1 metal-2 metal-3 ]
++RemoteOFFLINE: [ rabbitmq-bundle-0 ]
++
++ Clone Set: storage-clone [storage]
++ Stopped: [ metal-1 metal-2 metal-3 rabbitmq-bundle-0 ]
++ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Stopped
++ galera-bundle-1 (ocf::heartbeat:galera): Stopped
++ galera-bundle-2 (ocf::heartbeat:galera): Stopped
++ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Stopped
++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Stopped
++ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Stopped
++ redis-bundle-1 (ocf::heartbeat:redis): Stopped
++ redis-bundle-2 (ocf::heartbeat:redis): Stopped
++
++Transition Summary:
++ * Start storage:0 (metal-1)
++ * Start storage:1 (metal-2)
++ * Start storage:2 (metal-3)
++ * Start galera-bundle-docker-0 (metal-1)
++ * Start galera-bundle-0 (metal-1)
++ * Start galera:0 (galera-bundle-0)
++ * Start galera-bundle-docker-1 (metal-2)
++ * Start galera-bundle-1 (metal-2)
++ * Start galera:1 (galera-bundle-1)
++ * Start galera-bundle-docker-2 (metal-3)
++ * Start galera-bundle-2 (metal-3)
++ * Start galera:2 (galera-bundle-2)
++ * Start haproxy-bundle-docker-0 (metal-1)
++ * Start haproxy-bundle-docker-1 (metal-2)
++ * Start haproxy-bundle-docker-2 (metal-3)
++ * Start redis-bundle-docker-0 (metal-1)
++ * Start redis-bundle-0 (metal-1)
++ * Start redis:0 (redis-bundle-0)
++ * Promote redis:0 (Stopped -> Master redis-bundle-0)
++ * Start redis-bundle-docker-1 (metal-2)
++ * Start redis-bundle-1 (metal-2)
++ * Start redis:1 (redis-bundle-1)
++ * Promote redis:1 (Stopped -> Master redis-bundle-1)
++ * Start redis-bundle-docker-2 (metal-3)
++ * Start redis-bundle-2 (metal-3)
++ * Start redis:2 (redis-bundle-2)
++ * Promote redis:2 (Stopped -> Master redis-bundle-2)
++
++Executing cluster transition:
++ * Resource action: storage:0 monitor on metal-1
++ * Resource action: storage:1 monitor on metal-2
++ * Resource action: storage:2 monitor on metal-3
++ * Pseudo action: storage-clone_pre_notify_start_0
++ * Resource action: galera-bundle-docker-0 monitor on metal-3
++ * Resource action: galera-bundle-docker-0 monitor on metal-2
++ * Resource action: galera-bundle-docker-0 monitor on metal-1
++ * Resource action: galera-bundle-docker-1 monitor on metal-3
++ * Resource action: galera-bundle-docker-1 monitor on metal-2
++ * Resource action: galera-bundle-docker-1 monitor on metal-1
++ * Resource action: galera-bundle-docker-2 monitor on metal-3
++ * Resource action: galera-bundle-docker-2 monitor on metal-2
++ * Resource action: galera-bundle-docker-2 monitor on metal-1
++ * Resource action: haproxy-bundle-docker-0 monitor on metal-3
++ * Resource action: haproxy-bundle-docker-0 monitor on metal-2
++ * Resource action: haproxy-bundle-docker-0 monitor on metal-1
++ * Resource action: haproxy-bundle-docker-1 monitor on metal-3
++ * Resource action: haproxy-bundle-docker-1 monitor on metal-2
++ * Resource action: haproxy-bundle-docker-1 monitor on metal-1
++ * Resource action: haproxy-bundle-docker-2 monitor on metal-3
++ * Resource action: haproxy-bundle-docker-2 monitor on metal-2
++ * Resource action: haproxy-bundle-docker-2 monitor on metal-1
++ * Resource action: redis-bundle-docker-0 monitor on metal-3
++ * Resource action: redis-bundle-docker-0 monitor on metal-2
++ * Resource action: redis-bundle-docker-0 monitor on metal-1
++ * Resource action: redis-bundle-docker-1 monitor on metal-3
++ * Resource action: redis-bundle-docker-1 monitor on metal-2
++ * Resource action: redis-bundle-docker-1 monitor on metal-1
++ * Resource action: redis-bundle-docker-2 monitor on metal-3
++ * Resource action: redis-bundle-docker-2 monitor on metal-2
++ * Resource action: redis-bundle-docker-2 monitor on metal-1
++ * Pseudo action: redis-bundle_start_0
++ * Pseudo action: haproxy-bundle_start_0
++ * Pseudo action: storage-clone_confirmed-pre_notify_start_0
++ * Resource action: haproxy-bundle-docker-0 start on metal-1
++ * Resource action: haproxy-bundle-docker-1 start on metal-2
++ * Resource action: haproxy-bundle-docker-2 start on metal-3
++ * Resource action: redis-bundle-docker-0 start on metal-1
++ * Resource action: redis-bundle-0 start on metal-1
++ * Resource action: redis-bundle-docker-1 start on metal-2
++ * Resource action: redis-bundle-1 start on metal-2
++ * Resource action: redis-bundle-docker-2 start on metal-3
++ * Resource action: redis-bundle-2 start on metal-3
++ * Pseudo action: redis-bundle-master_start_0
++ * Pseudo action: haproxy-bundle_running_0
++ * Resource action: haproxy-bundle-docker-0 monitor=60000 on metal-1
++ * Resource action: haproxy-bundle-docker-1 monitor=60000 on metal-2
++ * Resource action: haproxy-bundle-docker-2 monitor=60000 on metal-3
++ * Resource action: redis:0 start on redis-bundle-0
++ * Resource action: redis-bundle-docker-0 monitor=60000 on metal-1
++ * Resource action: redis-bundle-0 monitor=60000 on metal-1
++ * Resource action: redis:1 start on redis-bundle-1
++ * Resource action: redis-bundle-docker-1 monitor=60000 on metal-2
++ * Resource action: redis-bundle-1 monitor=60000 on metal-2
++ * Resource action: redis:2 start on redis-bundle-2
++ * Resource action: redis-bundle-docker-2 monitor=60000 on metal-3
++ * Resource action: redis-bundle-2 monitor=60000 on metal-3
++ * Pseudo action: redis-bundle-master_running_0
++ * Pseudo action: redis-bundle_running_0
++ * Pseudo action: redis-bundle_promote_0
++ * Pseudo action: redis-bundle-master_promote_0
++ * Resource action: redis:0 promote on redis-bundle-0
++ * Resource action: redis:1 promote on redis-bundle-1
++ * Resource action: redis:2 promote on redis-bundle-2
++ * Pseudo action: redis-bundle-master_promoted_0
++ * Resource action: redis:0 monitor=20000 on redis-bundle-0
++ * Resource action: redis:1 monitor=20000 on redis-bundle-1
++ * Resource action: redis:2 monitor=20000 on redis-bundle-2
++ * Pseudo action: redis-bundle_promoted_0
++ * Pseudo action: storage-clone_start_0
++ * Resource action: storage:0 start on metal-1
++ * Resource action: storage:1 start on metal-2
++ * Resource action: storage:2 start on metal-3
++ * Pseudo action: storage-clone_running_0
++ * Pseudo action: storage-clone_post_notify_running_0
++ * Resource action: storage:0 notify on metal-1
++ * Resource action: storage:1 notify on metal-2
++ * Resource action: storage:2 notify on metal-3
++ * Pseudo action: storage-clone_confirmed-post_notify_running_0
++ * Pseudo action: galera-bundle_start_0
++ * Resource action: storage:0 monitor=30000 on metal-1
++ * Resource action: storage:1 monitor=30000 on metal-2
++ * Resource action: storage:2 monitor=30000 on metal-3
++ * Resource action: galera-bundle-docker-0 start on metal-1
++ * Resource action: galera-bundle-0 start on metal-1
++ * Resource action: galera-bundle-docker-1 start on metal-2
++ * Resource action: galera-bundle-1 start on metal-2
++ * Resource action: galera-bundle-docker-2 start on metal-3
++ * Resource action: galera-bundle-2 start on metal-3
++ * Pseudo action: galera-bundle-master_start_0
++ * Resource action: galera:0 start on galera-bundle-0
++ * Resource action: galera-bundle-docker-0 monitor=60000 on metal-1
++ * Resource action: galera-bundle-0 monitor=60000 on metal-1
++ * Resource action: galera:1 start on galera-bundle-1
++ * Resource action: galera-bundle-docker-1 monitor=60000 on metal-2
++ * Resource action: galera-bundle-1 monitor=60000 on metal-2
++ * Resource action: galera:2 start on galera-bundle-2
++ * Resource action: galera-bundle-docker-2 monitor=60000 on metal-3
++ * Resource action: galera-bundle-2 monitor=60000 on metal-3
++ * Pseudo action: galera-bundle-master_running_0
++ * Pseudo action: galera-bundle_running_0
++ * Resource action: galera:0 monitor=30000 on galera-bundle-0
++ * Resource action: galera:0 monitor=20000 on galera-bundle-0
++ * Resource action: galera:1 monitor=30000 on galera-bundle-1
++ * Resource action: galera:1 monitor=20000 on galera-bundle-1
++ * Resource action: galera:2 monitor=30000 on galera-bundle-2
++ * Resource action: galera:2 monitor=20000 on galera-bundle-2
++
++Revised cluster status:
++Online: [ metal-1 metal-2 metal-3 ]
++RemoteOFFLINE: [ rabbitmq-bundle-0 ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
++
++ Clone Set: storage-clone [storage]
++ Started: [ metal-1 metal-2 metal-3 ]
++ Stopped: [ rabbitmq-bundle-0 ]
++ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Slave metal-1
++ galera-bundle-1 (ocf::heartbeat:galera): Slave metal-2
++ galera-bundle-2 (ocf::heartbeat:galera): Slave metal-3
++ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started metal-1
++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started metal-2
++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started metal-3
++ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Master metal-1
++ redis-bundle-1 (ocf::heartbeat:redis): Master metal-2
++ redis-bundle-2 (ocf::heartbeat:redis): Master metal-3
++
+diff --git a/pengine/test10/bundle-order-startup-clone-2.xml b/pengine/test10/bundle-order-startup-clone-2.xml
+new file mode 100644
+index 0000000..e2c248e
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone-2.xml
+@@ -0,0 +1,190 @@
++<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="metal-1">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="2" uname="metal-2">
++ <instance_attributes id="nodes-2">
++ <nvpair id="nodes-2-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-2-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-2-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-2-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-2-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="3" uname="metal-3">
++ <instance_attributes id="nodes-3">
++ <nvpair id="nodes-3-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-3-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-3-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-3-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-3-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
++ <instance_attributes id="nodes-rabbitmq-bundle-0">
++ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@metal"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <clone id="storage-clone">
++ <meta_attributes id="storage_ma">
++ <nvpair id="storage_globally-unique" name="globally-unique" value="false"/>
++ <nvpair id="storage_notify" name="notify" value="true"/>
++ </meta_attributes>
++ <primitive class="ocf" id="storage" provider="heartbeat" type="Filesystem">
++ <operations>
++ <op id="storage_mon" interval="30s" name="monitor" timeout="30s"/>
++ </operations>
++ <instance_attributes id="storage_ia">
++ <nvpair id="storage_attr_0" name="device" value="nfs:/share/drbd_www/data/"/>
++ <nvpair id="storage_attr_1" name="directory" value="/data/www"/>
++ <nvpair id="storage_attr_2" name="fstype" value="nfs"/>
++ </instance_attributes>
++ </primitive>
++ </clone>
++ <bundle id="galera-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
++ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
++ <instance_attributes id="galera-instance_attributes">
++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:metal"/>
++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://metal"/>
++ </instance_attributes>
++ <meta_attributes id="galera-meta_attributes">
++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
++ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
++ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ </bundle>
++ <bundle id="redis-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3124"/>
++ <storage>
++ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
++ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
++ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
++ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
++ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
++ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
++ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
++ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
++ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
++ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
++ <rule id="location-redis-bundle-rule" score="0">
++ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
++ <rule id="location-galera-bundle-rule" score="0">
++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-redis-promote-1" rsc="redis-bundle" role="Master" node="redis-bundle-0" score="100"/>
++ <rsc_location id="location-redis-promote-2" rsc="redis-bundle" role="Master" node="redis-bundle-1" score="100"/>
++ <rsc_location id="location-redis-promote-3" rsc="redis-bundle" role="Master" node="redis-bundle-2" score="100"/>
++ <rsc_colocation id="colocation-1" rsc="redis-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_colocation id="colocation-2" rsc="galera-bundle" score="INFINITY" with-rsc="storage-clone"/>
++ <rsc_order first="redis-bundle" first-action="promote" id="order-1" kind="Mandatory" then="storage-clone" then-action="start"/>
++ <rsc_order first="storage-clone" first-action="start" id="order-2" kind="Mandatory" then="galera-bundle" then-action="start"/>
++ <rsc_order first="haproxy-bundle" first-action="start" id="order-3" kind="Mandatory" then="storage-clone" then-action="start"/>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="metal-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
++ <node_state id="2" uname="metal-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
++ <node_state id="3" uname="metal-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
++ </status>
++</cib>
+diff --git a/pengine/test10/bundle-order-startup-clone.dot b/pengine/test10/bundle-order-startup-clone.dot
+new file mode 100644
+index 0000000..92f019f
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone.dot
+@@ -0,0 +1,119 @@
++digraph "g" {
++"galera-bundle-0_monitor_60000 metal-1" [ style=dashed color="red" fontcolor="black"]
++"galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = dashed]
++"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed]
++"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_30000 galera-bundle-0" [ style = dashed]
++"galera-bundle-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
++"galera-bundle-0_start_0 metal-1" [ style=dashed color="red" fontcolor="black"]
++"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed]
++"galera-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed]
++"galera-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed]
++"galera-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_monitor_60000 metal-1" [ style=dashed color="red" fontcolor="black"]
++"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = dashed]
++"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-docker-0_monitor_60000 metal-1" [ style = dashed]
++"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle_running_0" [ style = dashed]
++"galera-bundle-docker-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
++"galera-bundle-docker-0_start_0 metal-1" [ style=dashed color="red" fontcolor="black"]
++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = dashed]
++"galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed]
++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
++"galera-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"]
++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed]
++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed]
++"galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"]
++"galera:0_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera:0_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = dashed]
++"galera:0_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-2" [ style = bold]
++"haproxy-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle-docker-0_start_0 metal-2" -> "haproxy-bundle-docker-0_monitor_60000 metal-2" [ style = bold]
++"haproxy-bundle-docker-0_start_0 metal-2" -> "haproxy-bundle_running_0" [ style = bold]
++"haproxy-bundle-docker-0_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"haproxy-bundle_running_0" -> "storage-clone_start_0" [ style = dashed]
++"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 metal-2" [ style = bold]
++"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-0_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-0_start_0 metal-2" -> "redis-bundle-0_monitor_60000 metal-2" [ style = bold]
++"redis-bundle-0_start_0 metal-2" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold]
++"redis-bundle-0_start_0 metal-2" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold]
++"redis-bundle-0_start_0 metal-2" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
++"redis-bundle-0_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-docker-0_start_0 metal-2" -> "redis-bundle-0_start_0 metal-2" [ style = bold]
++"redis-bundle-docker-0_start_0 metal-2" -> "redis-bundle-docker-0_monitor_60000 metal-2" [ style = bold]
++"redis-bundle-docker-0_start_0 metal-2" -> "redis-bundle_running_0" [ style = bold]
++"redis-bundle-docker-0_start_0 metal-2" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
++"redis-bundle-docker-0_start_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold]
++"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold]
++"redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold]
++"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"redis-bundle_start_0" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold]
++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold]
++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"redis:0_monitor_45000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis:0_monitor_60000 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"redis:0_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = bold]
++"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold]
++"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold]
++"redis:0_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"]
++"storage-clone_confirmed-post_notify_running_0" -> "galera-bundle_start_0" [ style = dashed]
++"storage-clone_confirmed-post_notify_running_0" -> "storage:0_monitor_30000 metal-1" [ style = dashed]
++"storage-clone_confirmed-post_notify_running_0" -> "storage:1_monitor_30000 metal-2" [ style = dashed]
++"storage-clone_confirmed-post_notify_running_0" -> "storage:2_monitor_30000 metal-3" [ style = dashed]
++"storage-clone_confirmed-post_notify_running_0" [ style=dashed color="red" fontcolor="orange"]
++"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_post_notify_running_0" [ style = dashed]
++"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_start_0" [ style = dashed]
++"storage-clone_confirmed-pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"]
++"storage-clone_post_notify_running_0" -> "storage-clone_confirmed-post_notify_running_0" [ style = dashed]
++"storage-clone_post_notify_running_0" [ style=dashed color="red" fontcolor="orange"]
++"storage-clone_pre_notify_start_0" -> "storage-clone_confirmed-pre_notify_start_0" [ style = dashed]
++"storage-clone_pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"]
++"storage-clone_running_0" -> "storage-clone_post_notify_running_0" [ style = dashed]
++"storage-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
++"storage-clone_start_0" -> "storage-clone_running_0" [ style = dashed]
++"storage-clone_start_0" -> "storage:0_start_0 metal-1" [ style = dashed]
++"storage-clone_start_0" -> "storage:1_start_0 metal-2" [ style = dashed]
++"storage-clone_start_0" -> "storage:2_start_0 metal-3" [ style = dashed]
++"storage-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
++"storage:0_monitor_0 metal-1" -> "storage-clone_start_0" [ style = dashed]
++"storage:0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"storage:0_monitor_30000 metal-1" [ style=dashed color="red" fontcolor="black"]
++"storage:0_start_0 metal-1" -> "storage-clone_running_0" [ style = dashed]
++"storage:0_start_0 metal-1" -> "storage:0_monitor_30000 metal-1" [ style = dashed]
++"storage:0_start_0 metal-1" [ style=dashed color="red" fontcolor="black"]
++"storage:1_monitor_0 metal-2" -> "storage-clone_start_0" [ style = dashed]
++"storage:1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"storage:1_monitor_30000 metal-2" [ style=dashed color="red" fontcolor="black"]
++"storage:1_start_0 metal-2" -> "storage-clone_running_0" [ style = dashed]
++"storage:1_start_0 metal-2" -> "storage:1_monitor_30000 metal-2" [ style = dashed]
++"storage:1_start_0 metal-2" [ style=dashed color="red" fontcolor="black"]
++"storage:2_monitor_0 metal-3" -> "storage-clone_start_0" [ style = dashed]
++"storage:2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"storage:2_monitor_30000 metal-3" [ style=dashed color="red" fontcolor="black"]
++"storage:2_start_0 metal-3" -> "storage-clone_running_0" [ style = dashed]
++"storage:2_start_0 metal-3" -> "storage:2_monitor_30000 metal-3" [ style = dashed]
++"storage:2_start_0 metal-3" [ style=dashed color="red" fontcolor="black"]
++}
+diff --git a/pengine/test10/bundle-order-startup-clone.exp b/pengine/test10/bundle-order-startup-clone.exp
+new file mode 100644
+index 0000000..c736cb9
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone.exp
+@@ -0,0 +1,327 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0">
++ <action_set>
++ <rsc_op id="2" operation="monitor" operation_key="storage:0_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="6" operation="monitor" operation_key="storage:1_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="2">
++ <action_set>
++ <rsc_op id="10" operation="monitor" operation_key="storage:2_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="30000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="11" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="4">
++ <action_set>
++ <rsc_op id="7" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="3" operation="monitor" operation_key="galera-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="6">
++ <action_set>
++ <rsc_op id="56" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_60000" on_node="metal-2" on_node_uuid="2">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="55" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7">
++ <action_set>
++ <rsc_op id="55" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="4" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="12" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="57" operation="start" operation_key="haproxy-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="8">
++ <action_set>
++ <rsc_op id="12" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <rsc_op id="8" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="10">
++ <action_set>
++ <rsc_op id="4" operation="monitor" operation_key="haproxy-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" monitor_cmd="/bin/true" mount_points="" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3121 -v /var/lib/kolla/config_files/haproxy.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/haproxy/etc:/etc:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <rsc_op id="71" operation="monitor" operation_key="redis:0_monitor_60000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="60000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_role="Slave" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="69" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="12">
++ <action_set>
++ <rsc_op id="70" operation="monitor" operation_key="redis:0_monitor_45000" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="45000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="60000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="69" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13">
++ <action_set>
++ <rsc_op id="69" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2">
++ <primitive id="redis" long-id="redis:0" class="ocf" provider="heartbeat" type="redis"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_on_node="redis-bundle-0" CRM_meta_on_node_uuid="redis-bundle-0" CRM_meta_timeout="200000" wait_last_known_master="true"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="72" operation="start" operation_key="redis-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14">
++ <action_set>
++ <rsc_op id="62" operation="monitor" operation_key="redis-bundle-docker-0_monitor_60000" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15">
++ <action_set>
++ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="5" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="13" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="65" operation="start" operation_key="redis-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16">
++ <action_set>
++ <rsc_op id="13" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="17">
++ <action_set>
++ <rsc_op id="9" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="18">
++ <action_set>
++ <rsc_op id="5" operation="monitor" operation_key="redis-bundle-docker-0_monitor_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/redis-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3124 -v /var/lib/kolla/config_files/redis.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/redis/etc/redis:/etc/redis:ro -v /var/lib/config-data/redis/etc/redis.conf:/etc/redis.conf:ro -v /var/lib/config-data/redis/etc/redis.conf.puppet:/etc/redis.conf.puppet:ro -v /var/lib/config-data/redis/etc/redis-sentinel.conf:/etc/redis-sentinel.conf:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/redis:/var/lib/redis:rw -v /var/log/redis:/var/log/redis:rw -v /var/run/redis:/var/run/redis:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/redis-bundle-0:/var/log -p 3124:3124 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="19">
++ <action_set>
++ <rsc_op id="64" operation="monitor" operation_key="redis-bundle-0_monitor_60000" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_interval="60000" CRM_meta_name="monitor" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20">
++ <action_set>
++ <rsc_op id="63" operation="start" operation_key="redis-bundle-0_start_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="redis-bundle-docker-0" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" addr="metal-2" port="3124"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="21" priority="1000000">
++ <action_set>
++ <pseudo_event id="73" operation="running" operation_key="redis-bundle-master_running_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="69" operation="start" operation_key="redis:0_start_0" on_node="redis-bundle-0" on_node_uuid="redis-bundle-0" router_node="metal-2"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="72" operation="start" operation_key="redis-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="22">
++ <action_set>
++ <pseudo_event id="72" operation="start" operation_key="redis-bundle-master_start_0">
++ <attributes CRM_meta_clone_max="1" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="65" operation="start" operation_key="redis-bundle_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="23" priority="1000000">
++ <action_set>
++ <pseudo_event id="66" operation="running" operation_key="redis-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="61" operation="start" operation_key="redis-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="73" operation="running" operation_key="redis-bundle-master_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="24">
++ <action_set>
++ <pseudo_event id="65" operation="start" operation_key="redis-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="25" priority="1000000">
++ <action_set>
++ <pseudo_event id="58" operation="running" operation_key="haproxy-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="55" operation="start" operation_key="haproxy-bundle-docker-0_start_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="26">
++ <action_set>
++ <pseudo_event id="57" operation="start" operation_key="haproxy-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/bundle-order-startup-clone.scores b/pengine/test10/bundle-order-startup-clone.scores
+new file mode 100644
+index 0000000..d907861
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone.scores
+@@ -0,0 +1,174 @@
++Allocation scores:
++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
++clone_color: galera-bundle-master allocation score on metal-1: -INFINITY
++clone_color: galera-bundle-master allocation score on metal-2: -INFINITY
++clone_color: galera-bundle-master allocation score on metal-3: -INFINITY
++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
++clone_color: galera:0 allocation score on metal-1: -INFINITY
++clone_color: galera:0 allocation score on metal-2: -INFINITY
++clone_color: galera:0 allocation score on metal-3: -INFINITY
++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on metal-1: -INFINITY
++clone_color: redis-bundle-master allocation score on metal-2: -INFINITY
++clone_color: redis-bundle-master allocation score on metal-3: -INFINITY
++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on metal-1: -INFINITY
++clone_color: redis:0 allocation score on metal-2: -INFINITY
++clone_color: redis:0 allocation score on metal-3: -INFINITY
++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: storage-clone allocation score on metal-1: 0
++clone_color: storage-clone allocation score on metal-2: 0
++clone_color: storage-clone allocation score on metal-3: 0
++clone_color: storage-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:0 allocation score on metal-1: 0
++clone_color: storage:0 allocation score on metal-2: 0
++clone_color: storage:0 allocation score on metal-3: 0
++clone_color: storage:0 allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:1 allocation score on metal-1: 0
++clone_color: storage:1 allocation score on metal-2: 0
++clone_color: storage:1 allocation score on metal-3: 0
++clone_color: storage:1 allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:2 allocation score on metal-1: 0
++clone_color: storage:2 allocation score on metal-2: 0
++clone_color: storage:2 allocation score on metal-3: 0
++clone_color: storage:2 allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:3 allocation score on metal-1: 0
++clone_color: storage:3 allocation score on metal-2: 0
++clone_color: storage:3 allocation score on metal-3: 0
++clone_color: storage:3 allocation score on rabbitmq-bundle-0: 0
++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on metal-1: 0
++container_color: galera-bundle allocation score on metal-2: 0
++container_color: galera-bundle allocation score on metal-3: 0
++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on metal-1: 0
++container_color: galera-bundle-0 allocation score on metal-2: 0
++container_color: galera-bundle-0 allocation score on metal-3: 0
++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on metal-1: 0
++container_color: galera-bundle-docker-0 allocation score on metal-2: 0
++container_color: galera-bundle-docker-0 allocation score on metal-3: 0
++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on metal-1: 0
++container_color: galera-bundle-master allocation score on metal-2: 0
++container_color: galera-bundle-master allocation score on metal-3: 0
++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: 0
++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:0 allocation score on metal-1: 0
++container_color: galera:0 allocation score on metal-2: 0
++container_color: galera:0 allocation score on metal-3: 0
++container_color: galera:0 allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: 0
++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on metal-1: 0
++container_color: redis-bundle allocation score on metal-2: 0
++container_color: redis-bundle allocation score on metal-3: 0
++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on metal-1: 0
++container_color: redis-bundle-0 allocation score on metal-2: 0
++container_color: redis-bundle-0 allocation score on metal-3: 0
++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on metal-1: 0
++container_color: redis-bundle-docker-0 allocation score on metal-2: 0
++container_color: redis-bundle-docker-0 allocation score on metal-3: 0
++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on metal-1: 0
++container_color: redis-bundle-master allocation score on metal-2: 0
++container_color: redis-bundle-master allocation score on metal-3: 0
++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: 0
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:0 allocation score on metal-1: 0
++container_color: redis:0 allocation score on metal-2: 0
++container_color: redis:0 allocation score on metal-3: 0
++container_color: redis:0 allocation score on rabbitmq-bundle-0: 0
++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
++galera:0 promotion score on galera-bundle-0: -1
++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on metal-1: 10000
++native_color: galera-bundle-0 allocation score on metal-2: 0
++native_color: galera-bundle-0 allocation score on metal-3: 0
++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on metal-1: 0
++native_color: galera-bundle-docker-0 allocation score on metal-2: 0
++native_color: galera-bundle-docker-0 allocation score on metal-3: 0
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:0 allocation score on galera-bundle-0: INFINITY
++native_color: galera:0 allocation score on metal-1: -INFINITY
++native_color: galera:0 allocation score on metal-2: -INFINITY
++native_color: galera:0 allocation score on metal-3: -INFINITY
++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on metal-1: 0
++native_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++native_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on metal-1: 0
++native_color: redis-bundle-0 allocation score on metal-2: 10000
++native_color: redis-bundle-0 allocation score on metal-3: 0
++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on metal-1: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on metal-2: 0
++native_color: redis-bundle-docker-0 allocation score on metal-3: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:0 allocation score on metal-1: -INFINITY
++native_color: redis:0 allocation score on metal-2: -INFINITY
++native_color: redis:0 allocation score on metal-3: -INFINITY
++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: storage:0 allocation score on metal-1: 0
++native_color: storage:0 allocation score on metal-2: 0
++native_color: storage:0 allocation score on metal-3: 0
++native_color: storage:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: storage:1 allocation score on metal-1: -INFINITY
++native_color: storage:1 allocation score on metal-2: 0
++native_color: storage:1 allocation score on metal-3: 0
++native_color: storage:1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: storage:2 allocation score on metal-1: -INFINITY
++native_color: storage:2 allocation score on metal-2: -INFINITY
++native_color: storage:2 allocation score on metal-3: 0
++native_color: storage:2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: storage:3 allocation score on metal-1: -INFINITY
++native_color: storage:3 allocation score on metal-2: -INFINITY
++native_color: storage:3 allocation score on metal-3: -INFINITY
++native_color: storage:3 allocation score on rabbitmq-bundle-0: -INFINITY
++redis:0 promotion score on redis-bundle-0: -1
+diff --git a/pengine/test10/bundle-order-startup-clone.summary b/pengine/test10/bundle-order-startup-clone.summary
+new file mode 100644
+index 0000000..f3f8be0
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone.summary
+@@ -0,0 +1,69 @@
++
++Current cluster status:
++Online: [ metal-1 metal-2 metal-3 ]
++RemoteOFFLINE: [ rabbitmq-bundle-0 ]
++
++ Clone Set: storage-clone [storage]
++ Stopped: [ metal-1 metal-2 metal-3 rabbitmq-bundle-0 ]
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Stopped
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Stopped
++
++Transition Summary:
++ * Start storage:0 (metal-1 - blocked)
++ * Start storage:1 (metal-2 - blocked)
++ * Start storage:2 (metal-3 - blocked)
++ * Start galera-bundle-docker-0 (metal-1 - blocked)
++ * Start galera-bundle-0 (metal-1 - blocked)
++ * Start galera:0 (galera-bundle-0 - blocked)
++ * Start haproxy-bundle-docker-0 (metal-2)
++ * Start redis-bundle-docker-0 (metal-2)
++ * Start redis-bundle-0 (metal-2)
++ * Start redis:0 (redis-bundle-0)
++
++Executing cluster transition:
++ * Resource action: storage:0 monitor on metal-1
++ * Resource action: storage:1 monitor on metal-2
++ * Resource action: storage:2 monitor on metal-3
++ * Resource action: galera-bundle-docker-0 monitor on metal-3
++ * Resource action: galera-bundle-docker-0 monitor on metal-2
++ * Resource action: galera-bundle-docker-0 monitor on metal-1
++ * Resource action: haproxy-bundle-docker-0 monitor on metal-3
++ * Resource action: haproxy-bundle-docker-0 monitor on metal-2
++ * Resource action: haproxy-bundle-docker-0 monitor on metal-1
++ * Resource action: redis-bundle-docker-0 monitor on metal-3
++ * Resource action: redis-bundle-docker-0 monitor on metal-2
++ * Resource action: redis-bundle-docker-0 monitor on metal-1
++ * Pseudo action: redis-bundle_start_0
++ * Pseudo action: haproxy-bundle_start_0
++ * Resource action: haproxy-bundle-docker-0 start on metal-2
++ * Resource action: redis-bundle-docker-0 start on metal-2
++ * Resource action: redis-bundle-0 start on metal-2
++ * Pseudo action: redis-bundle-master_start_0
++ * Pseudo action: haproxy-bundle_running_0
++ * Resource action: haproxy-bundle-docker-0 monitor=60000 on metal-2
++ * Resource action: redis:0 start on redis-bundle-0
++ * Resource action: redis-bundle-docker-0 monitor=60000 on metal-2
++ * Resource action: redis-bundle-0 monitor=60000 on metal-2
++ * Pseudo action: redis-bundle-master_running_0
++ * Pseudo action: redis-bundle_running_0
++ * Resource action: redis:0 monitor=60000 on redis-bundle-0
++ * Resource action: redis:0 monitor=45000 on redis-bundle-0
++
++Revised cluster status:
++Online: [ metal-1 metal-2 metal-3 ]
++RemoteOFFLINE: [ rabbitmq-bundle-0 ]
++Containers: [ redis-bundle-0:redis-bundle-docker-0 ]
++
++ Clone Set: storage-clone [storage]
++ Stopped: [ metal-1 metal-2 metal-3 rabbitmq-bundle-0 ]
++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Stopped
++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started metal-2
++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Slave metal-2
++
+diff --git a/pengine/test10/bundle-order-startup-clone.xml b/pengine/test10/bundle-order-startup-clone.xml
+new file mode 100644
+index 0000000..d24c707
+--- /dev/null
++++ b/pengine/test10/bundle-order-startup-clone.xml
+@@ -0,0 +1,187 @@
++<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="129" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="metal-1">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="2" uname="metal-2">
++ <instance_attributes id="nodes-2">
++ <nvpair id="nodes-2-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-2-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-2-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-2-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-2-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="3" uname="metal-3">
++ <instance_attributes id="nodes-3">
++ <nvpair id="nodes-3-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-3-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-3-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-3-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-3-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
++ <instance_attributes id="nodes-rabbitmq-bundle-0">
++ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@metal"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <clone id="storage-clone">
++ <meta_attributes id="storage_ma">
++ <nvpair id="storage_globally-unique" name="globally-unique" value="false"/>
++ <nvpair id="storage_notify" name="notify" value="true"/>
++ </meta_attributes>
++ <primitive class="ocf" id="storage" provider="heartbeat" type="Filesystem">
++ <operations>
++ <op id="storage_mon" interval="30s" name="monitor" timeout="30s"/>
++ </operations>
++ <instance_attributes id="storage_ia">
++ <nvpair id="storage_attr_0" name="device" value="nfs:/share/drbd_www/data/"/>
++ <nvpair id="storage_attr_1" name="directory" value="/data/www"/>
++ <nvpair id="storage_attr_2" name="fstype" value="nfs"/>
++ </instance_attributes>
++ </primitive>
++ </clone>
++ <bundle id="galera-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
++ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
++ <instance_attributes id="galera-instance_attributes">
++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:metal"/>
++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://metal"/>
++ </instance_attributes>
++ <meta_attributes id="galera-meta_attributes">
++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
++ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
++ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ </bundle>
++ <bundle id="redis-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="1" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="1" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3124"/>
++ <storage>
++ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
++ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
++ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
++ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
++ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
++ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
++ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
++ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
++ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
++ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
++ <rule id="location-redis-bundle-rule" score="0">
++ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
++ <rule id="location-galera-bundle-rule" score="0">
++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_colocation id="colocation-1" rsc="redis-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_colocation id="colocation-2" rsc="galera-bundle" score="INFINITY" with-rsc="storage-clone"/>
++ <rsc_order first="redis-bundle" first-action="promote" id="order-1" kind="Mandatory" then="storage-clone" then-action="start"/>
++ <rsc_order first="storage-clone" first-action="start" id="order-2" kind="Mandatory" then="galera-bundle" then-action="start"/>
++ <rsc_order first="haproxy-bundle" first-action="start" id="order-3" kind="Mandatory" then="storage-clone" then-action="start"/>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="metal-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
++ <node_state id="2" uname="metal-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
++ <node_state id="3" uname="metal-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member"/>
++ </status>
++</cib>
+diff --git a/pengine/test10/bundle-order-stop-clone.dot b/pengine/test10/bundle-order-stop-clone.dot
+new file mode 100644
+index 0000000..6545fb8
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop-clone.dot
+@@ -0,0 +1,80 @@
++digraph "g" {
++"all_stopped" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-0_monitor_60000 metal-1" [ style=dashed color="red" fontcolor="black"]
++"galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = dashed]
++"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed]
++"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_30000 galera-bundle-0" [ style = dashed]
++"galera-bundle-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
++"galera-bundle-0_start_0 metal-1" [ style=dashed color="red" fontcolor="black"]
++"galera-bundle-0_stop_0 metal-1" -> "all_stopped" [ style = bold]
++"galera-bundle-0_stop_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = dashed]
++"galera-bundle-0_stop_0 metal-1" -> "galera-bundle-docker-0_stop_0 metal-1" [ style = bold]
++"galera-bundle-0_stop_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-docker-0_stop_0 metal-1" -> "all_stopped" [ style = bold]
++"galera-bundle-docker-0_stop_0 metal-1" -> "galera-bundle_stopped_0" [ style = bold]
++"galera-bundle-docker-0_stop_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold]
++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold]
++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold]
++"galera-bundle-master_stop_0" -> "galera:0_stop_0 galera-bundle-0" [ style = bold]
++"galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = bold]
++"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold]
++"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold]
++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 metal-1" [ style = bold]
++"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold]
++"galera-bundle_stop_0" -> "galera:0_stop_0 galera-bundle-0" [ style = bold]
++"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"]
++"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = bold]
++"galera-bundle_stopped_0" -> "storage-clone_stop_0" [ style = bold]
++"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"galera:0_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera:0_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed]
++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = dashed]
++"galera:0_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"]
++"galera:0_stop_0 galera-bundle-0" -> "all_stopped" [ style = bold]
++"galera:0_stop_0 galera-bundle-0" -> "galera-bundle-0_stop_0 metal-1" [ style = bold]
++"galera:0_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold]
++"galera:0_stop_0 galera-bundle-0" -> "galera:0_start_0 galera-bundle-0" [ style = dashed]
++"galera:0_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"]
++"storage-clone_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold]
++"storage-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_confirmed-pre_notify_stop_0" -> "storage-clone_post_notify_stopped_0" [ style = bold]
++"storage-clone_confirmed-pre_notify_stop_0" -> "storage-clone_stop_0" [ style = bold]
++"storage-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_post_notify_stopped_0" -> "storage-clone_confirmed-post_notify_stopped_0" [ style = bold]
++"storage-clone_post_notify_stopped_0" -> "storage:1_post_notify_stop_0 metal-2" [ style = bold]
++"storage-clone_post_notify_stopped_0" -> "storage:2_post_notify_stop_0 metal-3" [ style = bold]
++"storage-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_pre_notify_stop_0" -> "storage-clone_confirmed-pre_notify_stop_0" [ style = bold]
++"storage-clone_pre_notify_stop_0" -> "storage:0_pre_notify_stop_0 metal-1" [ style = bold]
++"storage-clone_pre_notify_stop_0" -> "storage:1_pre_notify_stop_0 metal-2" [ style = bold]
++"storage-clone_pre_notify_stop_0" -> "storage:2_pre_notify_stop_0 metal-3" [ style = bold]
++"storage-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_stop_0" -> "storage-clone_stopped_0" [ style = bold]
++"storage-clone_stop_0" -> "storage:0_stop_0 metal-1" [ style = bold]
++"storage-clone_stop_0" [ style=bold color="green" fontcolor="orange"]
++"storage-clone_stopped_0" -> "storage-clone_post_notify_stopped_0" [ style = bold]
++"storage-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]
++"storage:0_pre_notify_stop_0 metal-1" -> "storage-clone_confirmed-pre_notify_stop_0" [ style = bold]
++"storage:0_pre_notify_stop_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"storage:0_stop_0 metal-1" -> "all_stopped" [ style = bold]
++"storage:0_stop_0 metal-1" -> "storage-clone_stopped_0" [ style = bold]
++"storage:0_stop_0 metal-1" [ style=bold color="green" fontcolor="black"]
++"storage:1_post_notify_stop_0 metal-2" -> "storage-clone_confirmed-post_notify_stopped_0" [ style = bold]
++"storage:1_post_notify_stop_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"storage:1_pre_notify_stop_0 metal-2" -> "storage-clone_confirmed-pre_notify_stop_0" [ style = bold]
++"storage:1_pre_notify_stop_0 metal-2" [ style=bold color="green" fontcolor="black"]
++"storage:2_post_notify_stop_0 metal-3" -> "storage-clone_confirmed-post_notify_stopped_0" [ style = bold]
++"storage:2_post_notify_stop_0 metal-3" [ style=bold color="green" fontcolor="black"]
++"storage:2_pre_notify_stop_0 metal-3" -> "storage-clone_confirmed-pre_notify_stop_0" [ style = bold]
++"storage:2_pre_notify_stop_0 metal-3" [ style=bold color="green" fontcolor="black"]
++}
+diff --git a/pengine/test10/bundle-order-stop-clone.exp b/pengine/test10/bundle-order-stop-clone.exp
+new file mode 100644
+index 0000000..ac0ae05
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop-clone.exp
+@@ -0,0 +1,345 @@
++<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY" transition_id="0">
++ <synapse id="0">
++ <action_set>
++ <rsc_op id="140" operation="notify" operation_key="storage:0_pre_notify_stop_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="pre" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="1">
++ <action_set>
++ <rsc_op id="29" operation="stop" operation_key="storage:0_stop_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="storage" long-id="storage:0" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="40" operation="stop" operation_key="storage-clone_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="2" priority="1000000">
++ <action_set>
++ <rsc_op id="142" operation="notify" operation_key="storage:1_post_notify_stop_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="post" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="44" operation="notify" operation_key="storage-clone_post_notify_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="3">
++ <action_set>
++ <rsc_op id="141" operation="notify" operation_key="storage:1_pre_notify_stop_0" on_node="metal-2" on_node_uuid="2">
++ <primitive id="storage" long-id="storage:1" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="1" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="pre" CRM_meta_on_node="metal-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="4" priority="1000000">
++ <action_set>
++ <rsc_op id="144" operation="notify" operation_key="storage:2_post_notify_stop_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="post" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="44" operation="notify" operation_key="storage-clone_post_notify_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="5">
++ <action_set>
++ <rsc_op id="143" operation="notify" operation_key="storage:2_pre_notify_stop_0" on_node="metal-3" on_node_uuid="3">
++ <primitive id="storage" long-id="storage:2" class="ocf" provider="heartbeat" type="Filesystem"/>
++ <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_active_resource="storage:0 storage:1 storage:2" CRM_meta_notify_active_uname="metal-1 metal-2 metal-3" CRM_meta_notify_all_uname="galera-bundle-0 galera-bundle-1 galera-bundle-2 metal-1 metal-2 metal-3 rabbitmq-bundle-0 redis-bundle-0 redis-bundle-1 redis-bundle-2" CRM_meta_notify_available_uname="metal-2 metal-3 rabbitmq-bundle-0 metal-1" CRM_meta_notify_demote_resource=" " CRM_meta_notify_demote_uname=" " CRM_meta_notify_inactive_resource="storage:3" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_master_resource=" " CRM_meta_notify_master_uname=" " CRM_meta_notify_operation="stop" CRM_meta_notify_promote_resource=" " CRM_meta_notify_promote_uname=" " CRM_meta_notify_slave_resource=" " CRM_meta_notify_slave_uname=" " CRM_meta_notify_start_resource=" " CRM_meta_notify_start_uname=" " CRM_meta_notify_stop_resource="storage:0" CRM_meta_notify_stop_uname="metal-1" CRM_meta_notify_type="pre" CRM_meta_on_node="metal-3" CRM_meta_on_node_uuid="3" CRM_meta_timeout="20000" device="nfs:/share/drbd_www/data/" directory="/data/www" fstype="nfs"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="6" priority="1000000">
++ <action_set>
++ <pseudo_event id="45" operation="notified" operation_key="storage-clone_confirmed-post_notify_stopped_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="confirmed-post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="44" operation="notify" operation_key="storage-clone_post_notify_stopped_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="142" operation="notify" operation_key="storage:1_post_notify_stop_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="144" operation="notify" operation_key="storage:2_post_notify_stop_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="7" priority="1000000">
++ <action_set>
++ <pseudo_event id="44" operation="notify" operation_key="storage-clone_post_notify_stopped_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stopped" CRM_meta_notify_key_type="post" CRM_meta_notify_operation="stop" CRM_meta_notify_type="post" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="41" operation="stopped" operation_key="storage-clone_stopped_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-pre_notify_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="8">
++ <action_set>
++ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-pre_notify_stop_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="confirmed-pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="140" operation="notify" operation_key="storage:0_pre_notify_stop_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="141" operation="notify" operation_key="storage:1_pre_notify_stop_0" on_node="metal-2" on_node_uuid="2"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="143" operation="notify" operation_key="storage:2_pre_notify_stop_0" on_node="metal-3" on_node_uuid="3"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="9">
++ <action_set>
++ <pseudo_event id="42" operation="notify" operation_key="storage-clone_pre_notify_stop_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_notify_key_operation="stop" CRM_meta_notify_key_type="pre" CRM_meta_notify_operation="stop" CRM_meta_notify_type="pre" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="10" priority="1000000">
++ <action_set>
++ <pseudo_event id="41" operation="stopped" operation_key="storage-clone_stopped_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="stop" operation_key="storage:0_stop_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="40" operation="stop" operation_key="storage-clone_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="11">
++ <action_set>
++ <pseudo_event id="40" operation="stop" operation_key="storage-clone_stop_0">
++ <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="true" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="43" operation="notified" operation_key="storage-clone_confirmed-pre_notify_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="60" operation="stopped" operation_key="galera-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="12">
++ <action_set>
++ <rsc_op id="62" operation="stop" operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1">
++ <primitive id="galera" long-id="galera:0" class="ocf" provider="heartbeat" type="galera"/>
++ <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_name="stop" CRM_meta_notify="false" CRM_meta_on_node="galera-bundle-0" CRM_meta_on_node_uuid="galera-bundle-0" CRM_meta_timeout="120000" additional_parameters="--open-files-limit=16384" cluster_host_map="galera-bundle-0:metal" enable_creation="true" wsrep_cluster_address="gcomm://metal"/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="59" operation="stop" operation_key="galera-bundle_stop_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="69" operation="stop" operation_key="galera-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="13">
++ <action_set>
++ <rsc_op id="46" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker"/>
++ <attributes CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" allow_pull="true" force_kill="false" image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" monitor_cmd="/bin/true" mount_points="/var/log/pacemaker/bundles/galera-bundle-0" reuse="false" run_cmd="/bin/bash /usr/local/bin/kolla_start" run_opts=" --restart=no --net=host -e PCMK_remote_port=3123 -v /var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:ro -v /var/lib/config-data/mysql:/var/lib/kolla/config_files/src:ro -v /etc/hosts:/etc/hosts:ro -v /etc/localtime:/etc/localtime:ro -v /var/lib/mysql:/var/lib/mysql:rw -v /var/log/mariadb:/var/log/mariadb:rw -v /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro -v /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro -v /etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro -v /etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro -v /dev/log:/dev/log:rw -v /etc/pacemaker/authkey:/etc/pacemaker/authkey -v /var/log/pacemaker/bundles/galera-bundle-0:/var/log -p 3123:3123 --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS "/>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="47" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="59" operation="stop" operation_key="galera-bundle_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="14">
++ <action_set>
++ <rsc_op id="47" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="metal-1" on_node_uuid="1">
++ <primitive id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote"/>
++ <attributes CRM_meta_container="galera-bundle-docker-0" CRM_meta_on_node="metal-1" CRM_meta_on_node_uuid="1" CRM_meta_timeout="20000" addr="#uname" port="3123"/>
++ <downed>
++ <node id="galera-bundle-0"/>
++ </downed>
++ </rsc_op>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="62" operation="stop" operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="15" priority="1000000">
++ <action_set>
++ <pseudo_event id="70" operation="stopped" operation_key="galera-bundle-master_stopped_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="62" operation="stop" operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="69" operation="stop" operation_key="galera-bundle-master_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="16">
++ <action_set>
++ <pseudo_event id="69" operation="stop" operation_key="galera-bundle-master_stop_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="59" operation="stop" operation_key="galera-bundle_stop_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="17" priority="1000000">
++ <action_set>
++ <pseudo_event id="68" operation="running" operation_key="galera-bundle-master_running_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="67" operation="start" operation_key="galera-bundle-master_start_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="18">
++ <action_set>
++ <pseudo_event id="67" operation="start" operation_key="galera-bundle-master_start_0">
++ <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="3" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="57" operation="start" operation_key="galera-bundle_start_0"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="70" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="19" priority="1000000">
++ <action_set>
++ <pseudo_event id="60" operation="stopped" operation_key="galera-bundle_stopped_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="46" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="70" operation="stopped" operation_key="galera-bundle-master_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="20">
++ <action_set>
++ <pseudo_event id="59" operation="stop" operation_key="galera-bundle_stop_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs/>
++ </synapse>
++ <synapse id="21" priority="1000000">
++ <action_set>
++ <pseudo_event id="58" operation="running" operation_key="galera-bundle_running_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="68" operation="running" operation_key="galera-bundle-master_running_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="22">
++ <action_set>
++ <pseudo_event id="57" operation="start" operation_key="galera-bundle_start_0">
++ <attributes CRM_meta_timeout="20000" />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <pseudo_event id="60" operation="stopped" operation_key="galera-bundle_stopped_0"/>
++ </trigger>
++ </inputs>
++ </synapse>
++ <synapse id="23">
++ <action_set>
++ <pseudo_event id="28" operation="all_stopped" operation_key="all_stopped">
++ <attributes />
++ </pseudo_event>
++ </action_set>
++ <inputs>
++ <trigger>
++ <rsc_op id="29" operation="stop" operation_key="storage:0_stop_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <pseudo_event id="45" operation="notified" operation_key="storage-clone_confirmed-post_notify_stopped_0"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="46" operation="stop" operation_key="galera-bundle-docker-0_stop_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="47" operation="stop" operation_key="galera-bundle-0_stop_0" on_node="metal-1" on_node_uuid="1"/>
++ </trigger>
++ <trigger>
++ <rsc_op id="62" operation="stop" operation_key="galera:0_stop_0" on_node="galera-bundle-0" on_node_uuid="galera-bundle-0" router_node="metal-1"/>
++ </trigger>
++ </inputs>
++ </synapse>
++</transition_graph>
+diff --git a/pengine/test10/bundle-order-stop-clone.scores b/pengine/test10/bundle-order-stop-clone.scores
+new file mode 100644
+index 0000000..df53f32
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop-clone.scores
+@@ -0,0 +1,591 @@
++Allocation scores:
++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0
++clone_color: galera-bundle-master allocation score on galera-bundle-1: 0
++clone_color: galera-bundle-master allocation score on galera-bundle-2: 0
++clone_color: galera-bundle-master allocation score on metal-1: -INFINITY
++clone_color: galera-bundle-master allocation score on metal-2: -INFINITY
++clone_color: galera-bundle-master allocation score on metal-3: -INFINITY
++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY
++clone_color: galera:0 allocation score on galera-bundle-1: -INFINITY
++clone_color: galera:0 allocation score on galera-bundle-2: -INFINITY
++clone_color: galera:0 allocation score on metal-1: -INFINITY
++clone_color: galera:0 allocation score on metal-2: -INFINITY
++clone_color: galera:0 allocation score on metal-3: -INFINITY
++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:1 allocation score on galera-bundle-0: -INFINITY
++clone_color: galera:1 allocation score on galera-bundle-1: INFINITY
++clone_color: galera:1 allocation score on galera-bundle-2: -INFINITY
++clone_color: galera:1 allocation score on metal-1: -INFINITY
++clone_color: galera:1 allocation score on metal-2: -INFINITY
++clone_color: galera:1 allocation score on metal-3: -INFINITY
++clone_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: galera:2 allocation score on galera-bundle-0: -INFINITY
++clone_color: galera:2 allocation score on galera-bundle-1: -INFINITY
++clone_color: galera:2 allocation score on galera-bundle-2: INFINITY
++clone_color: galera:2 allocation score on metal-1: -INFINITY
++clone_color: galera:2 allocation score on metal-2: -INFINITY
++clone_color: galera:2 allocation score on metal-3: -INFINITY
++clone_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY
++clone_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY
++clone_color: redis-bundle-master allocation score on metal-1: -INFINITY
++clone_color: redis-bundle-master allocation score on metal-2: -INFINITY
++clone_color: redis-bundle-master allocation score on metal-3: -INFINITY
++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0
++clone_color: redis-bundle-master allocation score on redis-bundle-1: 0
++clone_color: redis-bundle-master allocation score on redis-bundle-2: 0
++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-1: -INFINITY
++clone_color: redis:0 allocation score on galera-bundle-2: -INFINITY
++clone_color: redis:0 allocation score on metal-1: -INFINITY
++clone_color: redis:0 allocation score on metal-2: -INFINITY
++clone_color: redis:0 allocation score on metal-3: -INFINITY
++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY
++clone_color: redis:0 allocation score on redis-bundle-1: -INFINITY
++clone_color: redis:0 allocation score on redis-bundle-2: -INFINITY
++clone_color: redis:1 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:1 allocation score on galera-bundle-1: -INFINITY
++clone_color: redis:1 allocation score on galera-bundle-2: -INFINITY
++clone_color: redis:1 allocation score on metal-1: -INFINITY
++clone_color: redis:1 allocation score on metal-2: -INFINITY
++clone_color: redis:1 allocation score on metal-3: -INFINITY
++clone_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:1 allocation score on redis-bundle-0: -INFINITY
++clone_color: redis:1 allocation score on redis-bundle-1: INFINITY
++clone_color: redis:1 allocation score on redis-bundle-2: -INFINITY
++clone_color: redis:2 allocation score on galera-bundle-0: -INFINITY
++clone_color: redis:2 allocation score on galera-bundle-1: -INFINITY
++clone_color: redis:2 allocation score on galera-bundle-2: -INFINITY
++clone_color: redis:2 allocation score on metal-1: -INFINITY
++clone_color: redis:2 allocation score on metal-2: -INFINITY
++clone_color: redis:2 allocation score on metal-3: -INFINITY
++clone_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY
++clone_color: redis:2 allocation score on redis-bundle-0: -INFINITY
++clone_color: redis:2 allocation score on redis-bundle-1: -INFINITY
++clone_color: redis:2 allocation score on redis-bundle-2: INFINITY
++clone_color: storage-clone allocation score on metal-1: -INFINITY
++clone_color: storage-clone allocation score on metal-2: 0
++clone_color: storage-clone allocation score on metal-3: 0
++clone_color: storage-clone allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:0 allocation score on metal-1: -INFINITY
++clone_color: storage:0 allocation score on metal-2: 0
++clone_color: storage:0 allocation score on metal-3: 0
++clone_color: storage:0 allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:1 allocation score on metal-1: -INFINITY
++clone_color: storage:1 allocation score on metal-2: INFINITY
++clone_color: storage:1 allocation score on metal-3: 0
++clone_color: storage:1 allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:2 allocation score on metal-1: -INFINITY
++clone_color: storage:2 allocation score on metal-2: 0
++clone_color: storage:2 allocation score on metal-3: INFINITY
++clone_color: storage:2 allocation score on rabbitmq-bundle-0: 0
++clone_color: storage:3 allocation score on metal-1: -INFINITY
++clone_color: storage:3 allocation score on metal-2: 0
++clone_color: storage:3 allocation score on metal-3: 0
++clone_color: storage:3 allocation score on rabbitmq-bundle-0: 0
++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle allocation score on metal-1: 0
++container_color: galera-bundle allocation score on metal-2: 0
++container_color: galera-bundle allocation score on metal-3: 0
++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-0 allocation score on metal-1: INFINITY
++container_color: galera-bundle-0 allocation score on metal-2: 0
++container_color: galera-bundle-0 allocation score on metal-3: 0
++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-1 allocation score on metal-1: 0
++container_color: galera-bundle-1 allocation score on metal-2: INFINITY
++container_color: galera-bundle-1 allocation score on metal-3: 0
++container_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-2 allocation score on metal-1: 0
++container_color: galera-bundle-2 allocation score on metal-2: 0
++container_color: galera-bundle-2 allocation score on metal-3: INFINITY
++container_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-docker-0 allocation score on metal-1: INFINITY
++container_color: galera-bundle-docker-0 allocation score on metal-2: 0
++container_color: galera-bundle-docker-0 allocation score on metal-3: 0
++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-docker-1 allocation score on metal-1: 0
++container_color: galera-bundle-docker-1 allocation score on metal-2: INFINITY
++container_color: galera-bundle-docker-1 allocation score on metal-3: 0
++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-docker-2 allocation score on metal-1: 0
++container_color: galera-bundle-docker-2 allocation score on metal-2: 0
++container_color: galera-bundle-docker-2 allocation score on metal-3: INFINITY
++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-1: -INFINITY
++container_color: galera-bundle-master allocation score on galera-bundle-2: -INFINITY
++container_color: galera-bundle-master allocation score on metal-1: 0
++container_color: galera-bundle-master allocation score on metal-2: 0
++container_color: galera-bundle-master allocation score on metal-3: 0
++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: 0
++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:0 allocation score on galera-bundle-1: -INFINITY
++container_color: galera:0 allocation score on galera-bundle-2: -INFINITY
++container_color: galera:0 allocation score on metal-1: 0
++container_color: galera:0 allocation score on metal-2: 0
++container_color: galera:0 allocation score on metal-3: 0
++container_color: galera:0 allocation score on rabbitmq-bundle-0: 0
++container_color: galera:1 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:1 allocation score on galera-bundle-1: -INFINITY
++container_color: galera:1 allocation score on galera-bundle-2: -INFINITY
++container_color: galera:1 allocation score on metal-1: 0
++container_color: galera:1 allocation score on metal-2: 0
++container_color: galera:1 allocation score on metal-3: 0
++container_color: galera:1 allocation score on rabbitmq-bundle-0: 0
++container_color: galera:2 allocation score on galera-bundle-0: -INFINITY
++container_color: galera:2 allocation score on galera-bundle-1: -INFINITY
++container_color: galera:2 allocation score on galera-bundle-2: -INFINITY
++container_color: galera:2 allocation score on metal-1: 0
++container_color: galera:2 allocation score on metal-2: 0
++container_color: galera:2 allocation score on metal-3: 0
++container_color: galera:2 allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-1: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-2: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on metal-3: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: 0
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-1: 0
++container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-2: 0
++container_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: 0
++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle allocation score on metal-1: 0
++container_color: redis-bundle allocation score on metal-2: 0
++container_color: redis-bundle allocation score on metal-3: 0
++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-0 allocation score on metal-1: INFINITY
++container_color: redis-bundle-0 allocation score on metal-2: 0
++container_color: redis-bundle-0 allocation score on metal-3: 0
++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-1 allocation score on metal-1: 0
++container_color: redis-bundle-1 allocation score on metal-2: INFINITY
++container_color: redis-bundle-1 allocation score on metal-3: 0
++container_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-2 allocation score on metal-1: 0
++container_color: redis-bundle-2 allocation score on metal-2: 0
++container_color: redis-bundle-2 allocation score on metal-3: INFINITY
++container_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on metal-1: INFINITY
++container_color: redis-bundle-docker-0 allocation score on metal-2: 0
++container_color: redis-bundle-docker-0 allocation score on metal-3: 0
++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on metal-1: 0
++container_color: redis-bundle-docker-1 allocation score on metal-2: INFINITY
++container_color: redis-bundle-docker-1 allocation score on metal-3: 0
++container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on metal-1: 0
++container_color: redis-bundle-docker-2 allocation score on metal-2: 0
++container_color: redis-bundle-docker-2 allocation score on metal-3: INFINITY
++container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY
++container_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY
++container_color: redis-bundle-master allocation score on metal-1: 0
++container_color: redis-bundle-master allocation score on metal-2: 0
++container_color: redis-bundle-master allocation score on metal-3: 0
++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: 0
++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-1: -INFINITY
++container_color: redis-bundle-master allocation score on redis-bundle-2: -INFINITY
++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:0 allocation score on galera-bundle-1: -INFINITY
++container_color: redis:0 allocation score on galera-bundle-2: -INFINITY
++container_color: redis:0 allocation score on metal-1: 0
++container_color: redis:0 allocation score on metal-2: 0
++container_color: redis:0 allocation score on metal-3: 0
++container_color: redis:0 allocation score on rabbitmq-bundle-0: 0
++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-1: -INFINITY
++container_color: redis:0 allocation score on redis-bundle-2: -INFINITY
++container_color: redis:1 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:1 allocation score on galera-bundle-1: -INFINITY
++container_color: redis:1 allocation score on galera-bundle-2: -INFINITY
++container_color: redis:1 allocation score on metal-1: 0
++container_color: redis:1 allocation score on metal-2: 0
++container_color: redis:1 allocation score on metal-3: 0
++container_color: redis:1 allocation score on rabbitmq-bundle-0: 0
++container_color: redis:1 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:1 allocation score on redis-bundle-1: -INFINITY
++container_color: redis:1 allocation score on redis-bundle-2: -INFINITY
++container_color: redis:2 allocation score on galera-bundle-0: -INFINITY
++container_color: redis:2 allocation score on galera-bundle-1: -INFINITY
++container_color: redis:2 allocation score on galera-bundle-2: -INFINITY
++container_color: redis:2 allocation score on metal-1: 0
++container_color: redis:2 allocation score on metal-2: 0
++container_color: redis:2 allocation score on metal-3: 0
++container_color: redis:2 allocation score on rabbitmq-bundle-0: 0
++container_color: redis:2 allocation score on redis-bundle-0: -INFINITY
++container_color: redis:2 allocation score on redis-bundle-1: -INFINITY
++container_color: redis:2 allocation score on redis-bundle-2: -INFINITY
++galera:0 promotion score on galera-bundle-0: -1
++galera:1 promotion score on galera-bundle-1: -1
++galera:2 promotion score on galera-bundle-2: -1
++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-0 allocation score on metal-1: INFINITY
++native_color: galera-bundle-0 allocation score on metal-2: -10000
++native_color: galera-bundle-0 allocation score on metal-3: -10000
++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-1 allocation score on metal-1: 0
++native_color: galera-bundle-1 allocation score on metal-2: INFINITY
++native_color: galera-bundle-1 allocation score on metal-3: 0
++native_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-2 allocation score on metal-1: 0
++native_color: galera-bundle-2 allocation score on metal-2: 0
++native_color: galera-bundle-2 allocation score on metal-3: INFINITY
++native_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on metal-1: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on metal-1: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on metal-2: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on metal-2: 0
++native_color: galera-bundle-docker-0 allocation score on metal-3: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on metal-3: 0
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on metal-1: -INFINITY
++native_color: galera-bundle-docker-1 allocation score on metal-2: INFINITY
++native_color: galera-bundle-docker-1 allocation score on metal-3: 0
++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on metal-1: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on metal-2: -INFINITY
++native_color: galera-bundle-docker-2 allocation score on metal-3: INFINITY
++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:0 allocation score on galera-bundle-0: INFINITY
++native_color: galera:0 allocation score on galera-bundle-1: -INFINITY
++native_color: galera:0 allocation score on galera-bundle-2: -INFINITY
++native_color: galera:0 allocation score on metal-1: -INFINITY
++native_color: galera:0 allocation score on metal-2: -INFINITY
++native_color: galera:0 allocation score on metal-3: -INFINITY
++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:1 allocation score on galera-bundle-0: -INFINITY
++native_color: galera:1 allocation score on galera-bundle-1: INFINITY
++native_color: galera:1 allocation score on galera-bundle-2: -INFINITY
++native_color: galera:1 allocation score on metal-1: -INFINITY
++native_color: galera:1 allocation score on metal-2: -INFINITY
++native_color: galera:1 allocation score on metal-3: -INFINITY
++native_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: galera:2 allocation score on galera-bundle-0: -INFINITY
++native_color: galera:2 allocation score on galera-bundle-1: -INFINITY
++native_color: galera:2 allocation score on galera-bundle-2: INFINITY
++native_color: galera:2 allocation score on metal-1: -INFINITY
++native_color: galera:2 allocation score on metal-2: -INFINITY
++native_color: galera:2 allocation score on metal-3: -INFINITY
++native_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on metal-1: INFINITY
++native_color: haproxy-bundle-docker-0 allocation score on metal-2: 0
++native_color: haproxy-bundle-docker-0 allocation score on metal-3: 0
++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on metal-1: -INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on metal-2: INFINITY
++native_color: haproxy-bundle-docker-1 allocation score on metal-3: 0
++native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on metal-1: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on metal-2: -INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on metal-3: INFINITY
++native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-0 allocation score on metal-1: INFINITY
++native_color: redis-bundle-0 allocation score on metal-2: 0
++native_color: redis-bundle-0 allocation score on metal-3: 0
++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-1 allocation score on metal-1: 0
++native_color: redis-bundle-1 allocation score on metal-2: INFINITY
++native_color: redis-bundle-1 allocation score on metal-3: 0
++native_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-2 allocation score on metal-1: 0
++native_color: redis-bundle-2 allocation score on metal-2: 0
++native_color: redis-bundle-2 allocation score on metal-3: INFINITY
++native_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on metal-1: INFINITY
++native_color: redis-bundle-docker-0 allocation score on metal-2: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on metal-3: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on metal-1: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on metal-2: INFINITY
++native_color: redis-bundle-docker-1 allocation score on metal-3: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on metal-1: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on metal-2: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on metal-3: INFINITY
++native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY
++native_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY
++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:0 allocation score on galera-bundle-1: -INFINITY
++native_color: redis:0 allocation score on galera-bundle-2: -INFINITY
++native_color: redis:0 allocation score on metal-1: -INFINITY
++native_color: redis:0 allocation score on metal-2: -INFINITY
++native_color: redis:0 allocation score on metal-3: -INFINITY
++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-0: INFINITY
++native_color: redis:0 allocation score on redis-bundle-1: -INFINITY
++native_color: redis:0 allocation score on redis-bundle-2: -INFINITY
++native_color: redis:1 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:1 allocation score on galera-bundle-1: -INFINITY
++native_color: redis:1 allocation score on galera-bundle-2: -INFINITY
++native_color: redis:1 allocation score on metal-1: -INFINITY
++native_color: redis:1 allocation score on metal-2: -INFINITY
++native_color: redis:1 allocation score on metal-3: -INFINITY
++native_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:1 allocation score on redis-bundle-0: -INFINITY
++native_color: redis:1 allocation score on redis-bundle-1: INFINITY
++native_color: redis:1 allocation score on redis-bundle-2: -INFINITY
++native_color: redis:2 allocation score on galera-bundle-0: -INFINITY
++native_color: redis:2 allocation score on galera-bundle-1: -INFINITY
++native_color: redis:2 allocation score on galera-bundle-2: -INFINITY
++native_color: redis:2 allocation score on metal-1: -INFINITY
++native_color: redis:2 allocation score on metal-2: -INFINITY
++native_color: redis:2 allocation score on metal-3: -INFINITY
++native_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: redis:2 allocation score on redis-bundle-0: -INFINITY
++native_color: redis:2 allocation score on redis-bundle-1: -INFINITY
++native_color: redis:2 allocation score on redis-bundle-2: INFINITY
++native_color: storage:0 allocation score on metal-1: -INFINITY
++native_color: storage:0 allocation score on metal-2: -INFINITY
++native_color: storage:0 allocation score on metal-3: -INFINITY
++native_color: storage:0 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: storage:1 allocation score on metal-1: -INFINITY
++native_color: storage:1 allocation score on metal-2: INFINITY
++native_color: storage:1 allocation score on metal-3: 0
++native_color: storage:1 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: storage:2 allocation score on metal-1: -INFINITY
++native_color: storage:2 allocation score on metal-2: -INFINITY
++native_color: storage:2 allocation score on metal-3: INFINITY
++native_color: storage:2 allocation score on rabbitmq-bundle-0: -INFINITY
++native_color: storage:3 allocation score on metal-1: -INFINITY
++native_color: storage:3 allocation score on metal-2: -INFINITY
++native_color: storage:3 allocation score on metal-3: -INFINITY
++native_color: storage:3 allocation score on rabbitmq-bundle-0: -INFINITY
++redis:0 promotion score on redis-bundle-0: 99
++redis:1 promotion score on redis-bundle-1: 99
++redis:2 promotion score on redis-bundle-2: 99
+diff --git a/pengine/test10/bundle-order-stop-clone.summary b/pengine/test10/bundle-order-stop-clone.summary
+new file mode 100644
+index 0000000..404eecd
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop-clone.summary
+@@ -0,0 +1,75 @@
++
++Current cluster status:
++Online: [ metal-1 metal-2 metal-3 ]
++RemoteOFFLINE: [ rabbitmq-bundle-0 ]
++Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
++
++ Clone Set: storage-clone [storage]
++ Started: [ metal-1 metal-2 metal-3 ]
++ Stopped: [ rabbitmq-bundle-0 ]
++ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Slave metal-1
++ galera-bundle-1 (ocf::heartbeat:galera): Slave metal-2
++ galera-bundle-2 (ocf::heartbeat:galera): Slave metal-3
++ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started metal-1
++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started metal-2
++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started metal-3
++ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Master metal-1
++ redis-bundle-1 (ocf::heartbeat:redis): Master metal-2
++ redis-bundle-2 (ocf::heartbeat:redis): Master metal-3
++
++Transition Summary:
++ * Stop storage:0 (metal-1)
++ * Stop galera-bundle-docker-0 (metal-1)
++ * Stop galera-bundle-0 (Started metal-1)
++ * Stop galera:0 (Slave galera-bundle-0)
++
++Executing cluster transition:
++ * Pseudo action: storage-clone_pre_notify_stop_0
++ * Pseudo action: galera-bundle_stop_0
++ * Resource action: storage:0 notify on metal-1
++ * Resource action: storage:1 notify on metal-2
++ * Resource action: storage:2 notify on metal-3
++ * Pseudo action: storage-clone_confirmed-pre_notify_stop_0
++ * Pseudo action: galera-bundle-master_stop_0
++ * Resource action: galera:0 stop on galera-bundle-0
++ * Resource action: galera-bundle-0 stop on metal-1
++ * Pseudo action: galera-bundle-master_stopped_0
++ * Resource action: galera-bundle-docker-0 stop on metal-1
++ * Pseudo action: galera-bundle_stopped_0
++ * Pseudo action: galera-bundle_start_0
++ * Pseudo action: storage-clone_stop_0
++ * Pseudo action: galera-bundle-master_start_0
++ * Resource action: storage:0 stop on metal-1
++ * Pseudo action: storage-clone_stopped_0
++ * Pseudo action: galera-bundle-master_running_0
++ * Pseudo action: galera-bundle_running_0
++ * Pseudo action: storage-clone_post_notify_stopped_0
++ * Resource action: storage:1 notify on metal-2
++ * Resource action: storage:2 notify on metal-3
++ * Pseudo action: storage-clone_confirmed-post_notify_stopped_0
++ * Pseudo action: all_stopped
++
++Revised cluster status:
++Online: [ metal-1 metal-2 metal-3 ]
++RemoteOFFLINE: [ rabbitmq-bundle-0 ]
++Containers: [ galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ]
++
++ Clone Set: storage-clone [storage]
++ Started: [ metal-2 metal-3 ]
++ Stopped: [ metal-1 rabbitmq-bundle-0 ]
++ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest]
++ galera-bundle-0 (ocf::heartbeat:galera): Stopped
++ galera-bundle-1 (ocf::heartbeat:galera): Slave metal-2
++ galera-bundle-2 (ocf::heartbeat:galera): Slave metal-3
++ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest]
++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started metal-1
++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started metal-2
++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started metal-3
++ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest]
++ redis-bundle-0 (ocf::heartbeat:redis): Master metal-1
++ redis-bundle-1 (ocf::heartbeat:redis): Master metal-2
++ redis-bundle-2 (ocf::heartbeat:redis): Master metal-3
++
+diff --git a/pengine/test10/bundle-order-stop-clone.xml b/pengine/test10/bundle-order-stop-clone.xml
+new file mode 100644
+index 0000000..60db64d
+--- /dev/null
++++ b/pengine/test10/bundle-order-stop-clone.xml
+@@ -0,0 +1,398 @@
++<cib crm_feature_set="3.0.12" validate-with="pacemaker-2.8" epoch="58" num_updates="222" admin_epoch="0" cib-last-written="Thu Jun 1 03:59:09 2017" update-origin="undercloud" update-client="cibadmin" update-user="root" have-quorum="1" dc-uuid="1">
++ <configuration>
++ <crm_config>
++ <cluster_property_set id="cib-bootstrap-options">
++ <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
++ <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.16-10.el7-94ff4df"/>
++ <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
++ <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="tripleo_cluster"/>
++ <nvpair id="cib-bootstrap-options-stonith-enabled" name="stonith-enabled" value="false"/>
++ </cluster_property_set>
++ <cluster_property_set id="redis_replication">
++ <nvpair id="redis_replication-redis_REPL_INFO" name="redis_REPL_INFO" value="redis-bundle-0"/>
++ </cluster_property_set>
++ </crm_config>
++ <nodes>
++ <node id="1" uname="metal-1">
++ <instance_attributes id="nodes-1">
++ <nvpair id="nodes-1-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-1-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-1-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-1-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-1-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="2" uname="metal-2">
++ <instance_attributes id="nodes-2">
++ <nvpair id="nodes-2-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-2-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-2-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-2-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-2-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="3" uname="metal-3">
++ <instance_attributes id="nodes-3">
++ <nvpair id="nodes-3-rabbitmq-role" name="rabbitmq-role" value="true"/>
++ <nvpair id="nodes-3-galera-role" name="galera-role" value="true"/>
++ <nvpair id="nodes-3-redis-role" name="redis-role" value="true"/>
++ <nvpair id="nodes-3-haproxy-role" name="haproxy-role" value="true"/>
++ <nvpair id="nodes-3-cinder-volume-role" name="cinder-volume-role" value="true"/>
++ </instance_attributes>
++ </node>
++ <node id="rabbitmq-bundle-0" type="remote" uname="rabbitmq-bundle-0">
++ <instance_attributes id="nodes-rabbitmq-bundle-0">
++ <nvpair id="nodes-rabbitmq-bundle-0-rmq-node-attr-last-known-rabbitmq" name="rmq-node-attr-last-known-rabbitmq" value="rabbit@metal"/>
++ </instance_attributes>
++ </node>
++ </nodes>
++ <resources>
++ <clone id="storage-clone">
++ <meta_attributes id="storage_ma">
++ <nvpair id="storage_globally-unique" name="globally-unique" value="false"/>
++ <nvpair id="storage_notify" name="notify" value="true"/>
++ </meta_attributes>
++ <primitive class="ocf" id="storage" provider="heartbeat" type="Filesystem">
++ <operations>
++ <op id="storage_mon" interval="30s" name="monitor" timeout="30s"/>
++ </operations>
++ <instance_attributes id="storage_ia">
++ <nvpair id="storage_attr_0" name="device" value="nfs:/share/drbd_www/data/"/>
++ <nvpair id="storage_attr_1" name="directory" value="/data/www"/>
++ <nvpair id="storage_attr_2" name="fstype" value="nfs"/>
++ </instance_attributes>
++ </primitive>
++ </clone>
++ <bundle id="galera-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3123"/>
++ <storage>
++ <storage-mapping id="mysql-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/mysql.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="mysql-cfg-data" options="ro" source-dir="/var/lib/config-data/mysql" target-dir="/var/lib/kolla/config_files/src"/>
++ <storage-mapping id="mysql-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="mysql-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="mysql-lib" options="rw" source-dir="/var/lib/mysql" target-dir="/var/lib/mysql"/>
++ <storage-mapping id="mysql-log-mariadb" options="rw" source-dir="/var/log/mariadb" target-dir="/var/log/mariadb"/>
++ <storage-mapping id="mysql-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="mysql-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="mysql-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="mysql-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="mysql-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="galera" provider="heartbeat" type="galera">
++ <instance_attributes id="galera-instance_attributes">
++ <nvpair id="galera-instance_attributes-additional_parameters" name="additional_parameters" value="--open-files-limit=16384"/>
++ <nvpair id="galera-instance_attributes-cluster_host_map" name="cluster_host_map" value="galera-bundle-0:metal"/>
++ <nvpair id="galera-instance_attributes-enable_creation" name="enable_creation" value="true"/>
++ <nvpair id="galera-instance_attributes-wsrep_cluster_address" name="wsrep_cluster_address" value="gcomm://metal"/>
++ </instance_attributes>
++ <meta_attributes id="galera-meta_attributes">
++ <nvpair id="galera-meta_attributes-master-max" name="master-max" value="1"/>
++ <nvpair id="galera-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="galera-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="galera-monitor-interval-20" interval="20" name="monitor" timeout="30"/>
++ <op id="galera-monitor-interval-10" interval="10" name="monitor" role="Master" timeout="30"/>
++ <op id="galera-monitor-interval-30" interval="30" name="monitor" role="Slave" timeout="30"/>
++ <op id="galera-promote-interval-0s" interval="0s" name="promote" on-fail="block" timeout="300s"/>
++ <op id="galera-start-interval-0s" interval="0s" name="start" timeout="120"/>
++ <op id="galera-stop-interval-0s" interval="0s" name="stop" timeout="120"/>
++ </operations>
++ </primitive>
++ </bundle>
++ <bundle id="haproxy-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" replicas="3" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <storage>
++ <storage-mapping id="haproxy-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/haproxy.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="haproxy-cfg-data" options="ro" source-dir="/var/lib/config-data/haproxy/etc" target-dir="/etc"/>
++ <storage-mapping id="haproxy-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="haproxy-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="haproxy-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="haproxy-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="haproxy-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="haproxy-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ </bundle>
++ <bundle id="redis-bundle">
++ <docker image="192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest" masters="3" network="host" options="--user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS" run-command="/bin/bash /usr/local/bin/kolla_start"/>
++ <network control-port="3124"/>
++ <storage>
++ <storage-mapping id="redis-cfg-files" options="ro" source-dir="/var/lib/kolla/config_files/redis.json" target-dir="/var/lib/kolla/config_files/config.json"/>
++ <storage-mapping id="redis-cfg-data-redis" options="ro" source-dir="/var/lib/config-data/redis/etc/redis" target-dir="/etc/redis"/>
++ <storage-mapping id="redis-cfg-data-redis-conf" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf" target-dir="/etc/redis.conf"/>
++ <storage-mapping id="redis-cfg-data-redis-conf-puppet" options="ro" source-dir="/var/lib/config-data/redis/etc/redis.conf.puppet" target-dir="/etc/redis.conf.puppet"/>
++ <storage-mapping id="redis-cfg-data-redis-sentinel" options="ro" source-dir="/var/lib/config-data/redis/etc/redis-sentinel.conf" target-dir="/etc/redis-sentinel.conf"/>
++ <storage-mapping id="redis-hosts" options="ro" source-dir="/etc/hosts" target-dir="/etc/hosts"/>
++ <storage-mapping id="redis-localtime" options="ro" source-dir="/etc/localtime" target-dir="/etc/localtime"/>
++ <storage-mapping id="redis-lib" options="rw" source-dir="/var/lib/redis" target-dir="/var/lib/redis"/>
++ <storage-mapping id="redis-log" options="rw" source-dir="/var/log/redis" target-dir="/var/log/redis"/>
++ <storage-mapping id="redis-run" options="rw" source-dir="/var/run/redis" target-dir="/var/run/redis"/>
++ <storage-mapping id="redis-pki-extracted" options="ro" source-dir="/etc/pki/ca-trust/extracted" target-dir="/etc/pki/ca-trust/extracted"/>
++ <storage-mapping id="redis-pki-ca-bundle-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.crt" target-dir="/etc/pki/tls/certs/ca-bundle.crt"/>
++ <storage-mapping id="redis-pki-ca-bundle-trust-crt" options="ro" source-dir="/etc/pki/tls/certs/ca-bundle.trust.crt" target-dir="/etc/pki/tls/certs/ca-bundle.trust.crt"/>
++ <storage-mapping id="redis-pki-cert" options="ro" source-dir="/etc/pki/tls/cert.pem" target-dir="/etc/pki/tls/cert.pem"/>
++ <storage-mapping id="redis-dev-log" options="rw" source-dir="/dev/log" target-dir="/dev/log"/>
++ </storage>
++ <primitive class="ocf" id="redis" provider="heartbeat" type="redis">
++ <instance_attributes id="redis-instance_attributes">
++ <nvpair id="redis-instance_attributes-wait_last_known_master" name="wait_last_known_master" value="true"/>
++ </instance_attributes>
++ <meta_attributes id="redis-meta_attributes">
++ <nvpair id="redis-meta_attributes-interleave" name="interleave" value="true"/>
++ <nvpair id="redis-meta_attributes-notify" name="notify" value="true"/>
++ <nvpair id="redis-meta_attributes-ordered" name="ordered" value="true"/>
++ </meta_attributes>
++ <operations>
++ <op id="redis-demote-interval-0s" interval="0s" name="demote" timeout="120"/>
++ <op id="redis-monitor-interval-45" interval="45" name="monitor" timeout="60"/>
++ <op id="redis-monitor-interval-20" interval="20" name="monitor" role="Master" timeout="60"/>
++ <op id="redis-monitor-interval-60" interval="60" name="monitor" role="Slave" timeout="60"/>
++ <op id="redis-promote-interval-0s" interval="0s" name="promote" timeout="120"/>
++ <op id="redis-start-interval-0s" interval="0s" name="start" timeout="200s"/>
++ <op id="redis-stop-interval-0s" interval="0s" name="stop" timeout="200s"/>
++ </operations>
++ </primitive>
++ </bundle>
++ </resources>
++ <constraints>
++ <rsc_location id="location-redis-bundle" resource-discovery="exclusive" rsc="redis-bundle">
++ <rule id="location-redis-bundle-rule" score="0">
++ <expression attribute="redis-role" id="location-redis-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="location-galera-bundle" resource-discovery="exclusive" rsc="galera-bundle">
++ <rule id="location-galera-bundle-rule" score="0">
++ <expression attribute="galera-role" id="location-galera-bundle-rule-expr" operation="eq" value="true"/>
++ </rule>
++ </rsc_location>
++ <rsc_location id="storage-ban-1" rsc="storage-clone" node="metal-1" score="-INFINITY"/>
++ <rsc_location id="make-promote-now-1" rsc="redis-bundle" role="Master" node="redis-bundle-0" score="100"/>
++ <rsc_location id="make-promote-now-2" rsc="redis-bundle" role="Master" node="redis-bundle-1" score="100"/>
++ <rsc_location id="make-promote-now-3" rsc="redis-bundle" role="Master" node="redis-bundle-2" score="100"/>
++ <rsc_colocation id="colocation-1" rsc="redis-bundle" score="INFINITY" with-rsc="haproxy-bundle"/>
++ <rsc_colocation id="colocation-2" rsc="galera-bundle" score="INFINITY" with-rsc="storage-clone"/>
++ <rsc_order first="redis-bundle" first-action="promote" id="order-1" kind="Mandatory" then="storage-clone" then-action="start"/>
++ <rsc_order first="storage-clone" first-action="start" id="order-2" kind="Mandatory" then="galera-bundle" then-action="start"/>
++ <rsc_order first="haproxy-bundle" first-action="start" id="order-3" kind="Mandatory" then="storage-clone" then-action="start"/>
++ </constraints>
++ <rsc_defaults>
++ <meta_attributes id="rsc_defaults-options">
++ <nvpair id="rsc_defaults-options-resource-stickiness" name="resource-stickiness" value="INFINITY"/>
++ </meta_attributes>
++ </rsc_defaults>
++ </configuration>
++ <status>
++ <node_state id="1" uname="metal-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="1">
++ <lrm_resources>
++ <lrm_resource id="storage:0" class="ocf" provider="heartbeat" type="Filesystem">
++ <lrm_rsc_op id="storage_last_0" operation_key="storage_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
++ <lrm_rsc_op id="storage_post_notify_start_0" operation_key="storage_notify_0" operation="notify" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
++ <lrm_rsc_op id="storage_monitor_30000" operation_key="storage_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="4" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="5fdd98b808a9cd8496a49fc679446175"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
++ <lrm_rsc_op id="galera-bundle-docker-0_monitor_60000" operation_key="galera-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="f7e456f82e276e07f726e6e86e93bbb3"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="810c6ca70f61efcf84f03ce4ebf1583e"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="d382f6658eb81d66313704072c2d9704"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ <lrm_rsc_op id="haproxy-bundle-docker-0_monitor_60000" operation_key="haproxy-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="8e98096b4e2923c7be3ce5fa2646c524"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="haproxy-bundle-docker-1_last_0" operation_key="haproxy-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="haproxy-bundle-docker-2_last_0" operation_key="haproxy-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
++ <lrm_rsc_op id="redis-bundle-docker-0_monitor_60000" operation_key="redis-bundle-docker-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="81e9d490c493f8d22fab2822ec0bed8a"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="029dcdb8e6d039453207529ddf76e626"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="78dfacba4be0b97e9a6426b342bb2fdd"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-0" class="ocf" provider="pacemaker" type="remote">
++ <lrm_rsc_op id="redis-bundle-0_last_0" operation_key="redis-bundle-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="cdc2746943a66a3b93738e3548dfc96d"/>
++ <lrm_rsc_op id="redis-bundle-0_monitor_60000" operation_key="redis-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="e1e2f8a6aaa00cc4f0e71adecd9e2301"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-0" class="ocf" provider="pacemaker" type="remote">
++ <lrm_rsc_op id="galera-bundle-0_last_0" operation_key="galera-bundle-0_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="504a42401176c2e94483b1e091f79a5a"/>
++ <lrm_rsc_op id="galera-bundle-0_monitor_60000" operation_key="galera-bundle-0_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="9901da5f8c52b9011a1b27b74e2c4eee"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state id="2" uname="metal-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="2">
++ <lrm_resources>
++ <lrm_resource id="storage:1" class="ocf" provider="heartbeat" type="Filesystem">
++ <lrm_rsc_op id="storage_last_0" operation_key="storage_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
++ <lrm_rsc_op id="storage_post_notify_start_0" operation_key="storage_notify_0" operation="notify" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
++ <lrm_rsc_op id="storage_monitor_30000" operation_key="storage_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="4" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="5fdd98b808a9cd8496a49fc679446175"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="810c6ca70f61efcf84f03ce4ebf1583e"/>
++ <lrm_rsc_op id="galera-bundle-docker-1_monitor_60000" operation_key="galera-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="9d23f939271b60f65c5d55115613887d"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="d382f6658eb81d66313704072c2d9704"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="haproxy-bundle-docker-1_last_0" operation_key="haproxy-bundle-docker-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ <lrm_rsc_op id="haproxy-bundle-docker-1_monitor_60000" operation_key="haproxy-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="8e98096b4e2923c7be3ce5fa2646c524"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="haproxy-bundle-docker-2_last_0" operation_key="haproxy-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="029dcdb8e6d039453207529ddf76e626"/>
++ <lrm_rsc_op id="redis-bundle-docker-1_monitor_60000" operation_key="redis-bundle-docker-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="d37abb44bbe352fe0b7d7664151cc6ab"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="78dfacba4be0b97e9a6426b342bb2fdd"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-1" class="ocf" provider="pacemaker" type="remote">
++ <lrm_rsc_op id="redis-bundle-1_last_0" operation_key="redis-bundle-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="40c70f9d61dc1fe2dba5a6fd7d5bd567"/>
++ <lrm_rsc_op id="redis-bundle-1_monitor_60000" operation_key="redis-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="869db8f221b6f9b135143fdc50dad3f4"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-1" class="ocf" provider="pacemaker" type="remote">
++ <lrm_rsc_op id="galera-bundle-1_last_0" operation_key="galera-bundle-1_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="912f68441e818e51b92b60f340f6faa5"/>
++ <lrm_rsc_op id="galera-bundle-1_monitor_60000" operation_key="galera-bundle-1_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="5f3d272e783249eaddff8c313cb262f9"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state id="3" uname="metal-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
++ <lrm id="3">
++ <lrm_resources>
++ <lrm_resource id="storage:2" class="ocf" provider="heartbeat" type="Filesystem">
++ <lrm_rsc_op id="storage_last_0" operation_key="storage_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
++ <lrm_rsc_op id="storage_post_notify_start_0" operation_key="storage_notify_0" operation="notify" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011d6ab00aa8677f8496f1a3e768a370"/>
++ <lrm_rsc_op id="storage_monitor_30000" operation_key="storage_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;4:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="4" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="5fdd98b808a9cd8496a49fc679446175"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="galera-bundle-docker-0_last_0" operation_key="galera-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="ed3aad657e7994e60ac4dbd3a200b0b7"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="galera-bundle-docker-1_last_0" operation_key="galera-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="810c6ca70f61efcf84f03ce4ebf1583e"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="galera-bundle-docker-2_last_0" operation_key="galera-bundle-docker-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="d382f6658eb81d66313704072c2d9704"/>
++ <lrm_rsc_op id="galera-bundle-docker-2_monitor_60000" operation_key="galera-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="7c2c0a8373a43288eb9e4c59a8a11546"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="haproxy-bundle-docker-0_last_0" operation_key="haproxy-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="haproxy-bundle-docker-1_last_0" operation_key="haproxy-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ </lrm_resource>
++ <lrm_resource id="haproxy-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="haproxy-bundle-docker-2_last_0" operation_key="haproxy-bundle-docker-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="50c5ea783cadafec1f3787e5cf2dcd62"/>
++ <lrm_rsc_op id="haproxy-bundle-docker-2_monitor_60000" operation_key="haproxy-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="8e98096b4e2923c7be3ce5fa2646c524"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-0" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="redis-bundle-docker-0_last_0" operation_key="redis-bundle-docker-0_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="90ccdc75136c9ebfb1985f14df781477"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-1" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="redis-bundle-docker-1_last_0" operation_key="redis-bundle-docker-1_monitor_0" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:7;1:-1:7:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="029dcdb8e6d039453207529ddf76e626"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-docker-2" class="ocf" provider="heartbeat" type="docker">
++ <lrm_rsc_op id="redis-bundle-docker-2_last_0" operation_key="redis-bundle-docker-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="78dfacba4be0b97e9a6426b342bb2fdd"/>
++ <lrm_rsc_op id="redis-bundle-docker-2_monitor_60000" operation_key="redis-bundle-docker-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="24676507e61be2e5cc014443ba54c560"/>
++ </lrm_resource>
++ <lrm_resource id="redis-bundle-2" class="ocf" provider="pacemaker" type="remote">
++ <lrm_rsc_op id="redis-bundle-2_last_0" operation_key="redis-bundle-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="3c205c3ee79f4db4f37351207ebff95e"/>
++ <lrm_rsc_op id="redis-bundle-2_monitor_60000" operation_key="redis-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="3786b0a658dcf4c2d3df6b2cea5cc84b"/>
++ </lrm_resource>
++ <lrm_resource id="galera-bundle-2" class="ocf" provider="pacemaker" type="remote">
++ <lrm_rsc_op id="galera-bundle-2_last_0" operation_key="galera-bundle-2_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="b64c504325efb1370a8509833d3fe8ee"/>
++ <lrm_rsc_op id="galera-bundle-2_monitor_60000" operation_key="galera-bundle-2_monitor_60000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="60000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="011536401935dcba9470adbf1db4a633"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state id="redis-bundle-0" uname="redis-bundle-0">
++ <lrm id="redis-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="redis:0" class="ocf" provider="heartbeat" type="redis">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:8;3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="8" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state id="redis-bundle-1" uname="redis-bundle-1">
++ <lrm id="redis-bundle-1">
++ <lrm_resources>
++ <lrm_resource id="redis:1" class="ocf" provider="heartbeat" type="redis">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:8;3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="8" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state id="redis-bundle-2" uname="redis-bundle-2">
++ <lrm id="redis-bundle-2">
++ <lrm_resources>
++ <lrm_resource id="redis:2" class="ocf" provider="heartbeat" type="redis">
++ <lrm_rsc_op id="redis_last_0" operation_key="redis_promote_0" operation="promote" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a5b2a4f5c557278af14d6cbffc5a229d"/>
++ <lrm_rsc_op id="redis_monitor_20000" operation_key="redis_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:8;3:-1:8:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="8" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="992feffd37882eb5ce9bfc847b2fa75e"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state id="galera-bundle-0" uname="galera-bundle-0">
++ <lrm id="galera-bundle-0">
++ <lrm_resources>
++ <lrm_resource id="galera:0" class="ocf" provider="heartbeat" type="galera">
++ <lrm_rsc_op id="galera_last_0" operation_key="galera_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a9aaa400c381f8dd43ac3a0cbee94c90"/>
++ <lrm_rsc_op id="galera_monitor_30000" operation_key="galera_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
++ <lrm_rsc_op id="galera_monitor_20000" operation_key="galera_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state id="galera-bundle-1" uname="galera-bundle-1">
++ <lrm id="galera-bundle-1">
++ <lrm_resources>
++ <lrm_resource id="galera:1" class="ocf" provider="heartbeat" type="galera">
++ <lrm_rsc_op id="galera_last_0" operation_key="galera_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a9aaa400c381f8dd43ac3a0cbee94c90"/>
++ <lrm_rsc_op id="galera_monitor_30000" operation_key="galera_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
++ <lrm_rsc_op id="galera_monitor_20000" operation_key="galera_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ <node_state id="galera-bundle-2" uname="galera-bundle-2">
++ <lrm id="galera-bundle-2">
++ <lrm_resources>
++ <lrm_resource id="galera:2" class="ocf" provider="heartbeat" type="galera">
++ <lrm_rsc_op id="galera_last_0" operation_key="galera_start_0" operation="start" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;1:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1498103792" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="a9aaa400c381f8dd43ac3a0cbee94c90"/>
++ <lrm_rsc_op id="galera_monitor_30000" operation_key="galera_monitor_30000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;2:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="2" rc-code="0" op-status="0" interval="30000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
++ <lrm_rsc_op id="galera_monitor_20000" operation_key="galera_monitor_20000" operation="monitor" crm-debug-origin="crm_simulate" crm_feature_set="3.0.14" transition-key="3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" transition-magic="0:0;3:-1:0:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" call-id="3" rc-code="0" op-status="0" interval="20000" last-rc-change="1498103792" exec-time="0" queue-time="0" op-digest="372bdbc835a0728d2a83ece1d27f89b8"/>
++ </lrm_resource>
++ </lrm_resources>
++ </lrm>
++ </node_state>
++ </status>
++</cib>
+--
+1.8.3.1
+
+
+From 79fea20c78979e1d849b4ec36e3ebde185811aab Mon Sep 17 00:00:00 2001
+From: Andrew Beekhof <andrew@beekhof.net>
+Date: Mon, 26 Jun 2017 11:02:40 +1000
+Subject: [PATCH 8/8] Fix: PE: Ensure bundle nodes and child resources are
+ correctly cleaned up
+
+---
+ lib/pengine/container.c | 20 ++++++++++----------
+ 1 file changed, 10 insertions(+), 10 deletions(-)
+
+diff --git a/lib/pengine/container.c b/lib/pengine/container.c
+index 6e98e6f..5d4e697 100644
+--- a/lib/pengine/container.c
++++ b/lib/pengine/container.c
+@@ -1029,7 +1029,10 @@ tuple_free(container_grouping_t *tuple)
+ return;
+ }
+
+- // TODO: Free tuple->node ?
++ if(tuple->node) {
++ free(tuple->node);
++ tuple->node = NULL;
++ }
+
+ if(tuple->ip) {
+ tuple->ip->fns->free(tuple->ip);
+@@ -1037,12 +1040,6 @@ tuple_free(container_grouping_t *tuple)
+ free_xml(tuple->ip->xml);
+ tuple->ip = NULL;
+ }
+- if(tuple->child) {
+- free_xml(tuple->child->xml);
+- tuple->child->xml = NULL;
+- tuple->child->fns->free(tuple->child);
+- tuple->child = NULL;
+- }
+ if(tuple->docker) {
+ free_xml(tuple->docker->xml);
+ tuple->docker->xml = NULL;
+@@ -1079,13 +1076,16 @@ container_free(resource_t * rsc)
+ free(container_data->docker_run_command);
+ free(container_data->docker_host_options);
+
+- if(container_data->child) {
+- free_xml(container_data->child->xml);
+- }
+ g_list_free_full(container_data->tuples, (GDestroyNotify)tuple_free);
+ g_list_free_full(container_data->mounts, (GDestroyNotify)mount_free);
+ g_list_free_full(container_data->ports, (GDestroyNotify)port_free);
+ g_list_free(rsc->children);
++
++ if(container_data->child) {
++ free_xml(container_data->child->xml);
++ container_data->child->xml = NULL;
++ container_data->child->fns->free(container_data->child);
++ }
+ common_free(rsc);
+ }
+
+--
+1.8.3.1
+
diff --git a/SOURCES/075-bundle-memory.patch b/SOURCES/075-bundle-memory.patch
new file mode 100644
index 0000000..7e2acba
--- /dev/null
+++ b/SOURCES/075-bundle-memory.patch
@@ -0,0 +1,54 @@
+From 12a6af187dec7b0c229673958dd0916e2479584e Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Mon, 26 Jun 2017 18:39:30 -0500
+Subject: [PATCH 1/2] Low: pengine: avoid use of NULL
+
+---
+ pengine/clone.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/pengine/clone.c b/pengine/clone.c
+index 28f368e..1722166 100644
+--- a/pengine/clone.c
++++ b/pengine/clone.c
+@@ -868,7 +868,7 @@ clone_create_pseudo_actions(
+ if (stop_notify != NULL && *stop_notify == NULL) {
+ *stop_notify = create_notification_boundaries(rsc, RSC_STOP, stop, stopped, data_set);
+
+- if (*stop_notify && *start_notify) {
++ if (start_notify && *start_notify && *stop_notify) {
+ order_actions((*stop_notify)->post_done, (*start_notify)->pre, pe_order_optional);
+ }
+ }
+--
+1.8.3.1
+
+
+From 4882ac112253f1f3afece01eb2e32a87b6d3689e Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Tue, 27 Jun 2017 11:11:45 -0500
+Subject: [PATCH 2/2] Low: libpe_status: avoid use-after-free
+
+---
+ lib/pengine/container.c | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/lib/pengine/container.c b/lib/pengine/container.c
+index 5d4e697..a21f0a4 100644
+--- a/lib/pengine/container.c
++++ b/lib/pengine/container.c
+@@ -1035,9 +1035,9 @@ tuple_free(container_grouping_t *tuple)
+ }
+
+ if(tuple->ip) {
+- tuple->ip->fns->free(tuple->ip);
+- tuple->ip->xml = NULL;
+ free_xml(tuple->ip->xml);
++ tuple->ip->xml = NULL;
++ tuple->ip->fns->free(tuple->ip);
+ tuple->ip = NULL;
+ }
+ if(tuple->docker) {
+--
+1.8.3.1
+
diff --git a/SOURCES/076-quorum-loss.patch b/SOURCES/076-quorum-loss.patch
new file mode 100644
index 0000000..8bd1923
--- /dev/null
+++ b/SOURCES/076-quorum-loss.patch
@@ -0,0 +1,37 @@
+From 0b6890554a1928ec96cc22f71497d7874e72aa4f Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Tue, 27 Jun 2017 12:27:32 -0500
+Subject: [PATCH] Fix: crmd: abort transition whenever we lose quorum
+
+If a node drops us below quorum because it was unexpectedly lost, the peer
+update callback will abort the transition. If it was shut down cleanly while
+running resources, the CIB recovery results will abort the transition. However,
+if it was shut down cleanly while not running any resources (e.g. in standby
+mode), previously we would not trigger a new transition, and the remaining
+nodes would not stop their resources until the next recheck interval.
+---
+ crmd/membership.c | 8 ++++++++
+ 1 file changed, 8 insertions(+)
+
+diff --git a/crmd/membership.c b/crmd/membership.c
+index b292f23..c36dbed 100644
+--- a/crmd/membership.c
++++ b/crmd/membership.c
+@@ -437,6 +437,14 @@ crm_update_quorum(gboolean quorum, gboolean force_update)
+ crm_debug("Updating quorum status to %s (call=%d)", quorum ? "true" : "false", call_id);
+ fsa_register_cib_callback(call_id, FALSE, NULL, cib_quorum_update_complete);
+ free_xml(update);
++
++ /* If a node not running any resources is cleanly shut down and drops us
++ * below quorum, we won't necessarily abort the transition, so abort it
++ * here to be safe.
++ */
++ if (quorum == FALSE) {
++ abort_transition(INFINITY, tg_restart, "Quorum loss", NULL);
++ }
+ }
+ fsa_has_quorum = quorum;
+ }
+--
+1.8.3.1
+
diff --git a/SOURCES/077-reenable-fence-device.patch b/SOURCES/077-reenable-fence-device.patch
new file mode 100644
index 0000000..f380bd8
--- /dev/null
+++ b/SOURCES/077-reenable-fence-device.patch
@@ -0,0 +1,26 @@
+From 4ad87e494d56e18bcaafa058554573f890517eed Mon Sep 17 00:00:00 2001
+From: Klaus Wenninger <klaus.wenninger@aon.at>
+Date: Fri, 21 Jul 2017 17:57:48 +0200
+Subject: [PATCH] Fix: stonith-ng: make fencing-device reappear properly after
+ reenabling
+
+---
+ fencing/main.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/fencing/main.c b/fencing/main.c
+index 360bc25..f58ccc7 100644
+--- a/fencing/main.c
++++ b/fencing/main.c
+@@ -781,7 +781,7 @@ update_cib_stonith_devices_v2(const char *event, xmlNode * msg)
+ }
+ free(mutable);
+
+- } else if(strstr(xpath, "/"XML_CIB_TAG_RESOURCES)) {
++ } else if(strstr(xpath, XML_CIB_TAG_RESOURCES)) {
+ shortpath = strrchr(xpath, '/'); CRM_ASSERT(shortpath);
+ reason = crm_strdup_printf("%s %s", op, shortpath+1);
+ needs_update = TRUE;
+--
+1.8.3.1
+
diff --git a/SOURCES/078-fencing-memory.patch b/SOURCES/078-fencing-memory.patch
new file mode 100644
index 0000000..8237470
--- /dev/null
+++ b/SOURCES/078-fencing-memory.patch
@@ -0,0 +1,25 @@
+From e7027e9d303be5e3f9531c0cb0ef8af914f2adda Mon Sep 17 00:00:00 2001
+From: Klaus Wenninger <klaus.wenninger@aon.at>
+Date: Tue, 11 Jul 2017 16:35:33 +0200
+Subject: [PATCH] fix: stonith-ng: avoid double-free of pending-ops in
+ free_device
+
+---
+ fencing/commands.c | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/fencing/commands.c b/fencing/commands.c
+index b336d3b..57a1a4e 100644
+--- a/fencing/commands.c
++++ b/fencing/commands.c
+@@ -509,7 +509,6 @@ free_device(gpointer data)
+
+ crm_warn("Removal of device '%s' purged operation %s", device->id, cmd->action);
+ cmd->done_cb(0, -ENODEV, NULL, cmd);
+- free_async_command(cmd);
+ }
+ g_list_free(device->pending_ops);
+
+--
+1.8.3.1
+
diff --git a/SOURCES/079-crm_report.patch b/SOURCES/079-crm_report.patch
new file mode 100644
index 0000000..43a9874
--- /dev/null
+++ b/SOURCES/079-crm_report.patch
@@ -0,0 +1,194 @@
+From 6a387a3a9d150ff972e6842e07a96a288eda4471 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot@redhat.com>
+Date: Wed, 16 Aug 2017 20:37:36 -0500
+Subject: [PATCH] Fix: tools: allow crm_report to work with no log files
+ specified
+
+582e886d broke crm_report --sos-mode
+
+Specifying no log files is a valid use case: someone might want to collect only
+the other information (cluster configuration, policy engine inputs, etc.);
+or someone might use only the systemd journal and no log files.
+---
+ tools/report.collector | 143 ++++++++++++++++++++++++++++++++-----------------
+ 1 file changed, 93 insertions(+), 50 deletions(-)
+
+diff --git a/tools/report.collector b/tools/report.collector
+index 0130a2b..d7c7c51 100644
+--- a/tools/report.collector
++++ b/tools/report.collector
+@@ -693,6 +693,85 @@ EOF
+ esac
+ }
+
++# Trim leading and ending whitespace (using only POSIX expressions)
++trim() {
++ TRIM_S="$1"
++
++ TRIM_S="${TRIM_S#"${TRIM_S%%[![:space:]]*}"}"
++ TRIM_S="${TRIM_S%"${TRIM_S##*[![:space:]]}"}"
++ echo -n "$TRIM_S"
++}
++
++collect_logs() {
++ CL_START="$1"
++ shift
++ CL_END="$1"
++ shift
++ CL_LOGFILES="$@"
++
++ which journalctl > /dev/null 2>&1
++ if [ $? -eq 0 ]; then
++ cl_have_journald=1
++ else
++ cl_have_journald=0
++ fi
++
++ cl_lognames="$CL_LOGFILES"
++ if [ $cl_have_journald -eq 1 ]; then
++ cl_lognames="$cl_lognames journalctl"
++ fi
++ cl_lognames=$(trim "$cl_lognames")
++ if [ -z "$cl_lognames" ]; then
++ return
++ fi
++
++ # YYYY-MM-DD HH:MM:SS
++ cl_start_ymd=$(date -d @${CL_START} +"%F %T")
++ cl_end_ymd=$(date -d @${CL_END} +"%F %T")
++
++ debug "Gathering logs from $cl_start_ymd to $cl_end_ymd:"
++ debug " $cl_lognames"
++
++ # Remove our temporary file if we get interrupted here
++ trap '[ -z "$cl_pattfile" ] || rm -f "$cl_pattfile"' 0
++
++ # Create a temporary file with patterns to grep for
++ cl_pattfile=$(mktemp) || fatal "cannot create temporary files"
++ for cl_pattern in $LOG_PATTERNS; do
++ echo "$cl_pattern"
++ done > $cl_pattfile
++
++ echo "Log pattern matches from $REPORT_TARGET:" > $ANALYSIS_F
++ if [ -n "$CL_LOGFILES" ]; then
++ for cl_logfile in $CL_LOGFILES; do
++ cl_extract="$(basename $cl_logfile).extract.txt"
++
++ if [ ! -f "$cl_logfile" ]; then
++ # Not a file
++ continue
++
++ elif [ -f "$cl_extract" ]; then
++ # We already have it
++ continue
++ fi
++
++ dumplogset "$cl_logfile" $LOG_START $LOG_END > "$cl_extract"
++ sanitize "$cl_extract"
++
++ grep -f "$cl_pattfile" "$cl_extract" >> $ANALYSIS_F
++ done
++ fi
++
++ # Collect systemd logs if present
++ if [ $cl_have_journald -eq 1 ]; then
++ journalctl --since "$cl_start_ymd" --until "$cl_end_ymd" > journal.log
++ grep -f "$cl_pattfile" journal.log >> $ANALYSIS_F
++ fi
++
++ rm -f $cl_pattfile
++ trap "" 0
++}
++
+ debug "Initializing $REPORT_TARGET subdir"
+ if [ "$REPORT_MASTER" != "$REPORT_TARGET" ]; then
+ if [ -e $REPORT_HOME/$REPORT_TARGET ]; then
+@@ -718,17 +797,19 @@ if [ -z "$cluster_cf" ] && [ $cluster != "any" ]; then
+ warning "Could not determine the location of your cluster configuration"
+ fi
+
+-if [ $SEARCH_LOGS = 1 ]; then
+- logfiles=`get_logfiles $cluster "$cluster_cf" "$logd_cf" | sort -u`
+- if [ -z "$logfiles" ]; then
+- fatal "Logfile discovery disabled, try specifying --logfile /some/path"
+- fi
+-
+-elif [ -z "$EXTRA_LOGS" ]; then
+- fatal "Could not determine the location of your cluster logs, try specifying --logfile /some/path"
++if [ "$SEARCH_LOGS" = "1" ]; then
++ logfiles=$(get_logfiles "$cluster" "$cluster_cf" "$logd_cf" | sort -u)
++fi
++logfiles="$(trim "$logfiles $EXTRA_LOGS")"
+
+-else
+- logfiles="$EXTRA_LOGS"
++if [ -z "$logfiles" ]; then
++ which journalctl > /dev/null 2>&1
++ if [ $? -eq 0 ]; then
++ info "Systemd journal will be only log collected"
++ else
++ info "No logs will be collected"
++ fi
++ info "No log files found or specified with --logfile /some/path"
+ fi
+
+ debug "Config: $cluster ($cluster_cf $logd_cf) $logfiles"
+@@ -783,45 +864,7 @@ done
+ # in it (AFTER sanitizing, so we don't need to sanitize this output)
+ get_readable_cib "$REPORT_HOME/$REPORT_TARGET"
+
+-# Grab logs
+-start=`date -d @${LOG_START} +"%F %T"`
+-end=`date -d @${LOG_END} +"%F %T"`
+-
+-debug "Gathering logs from $start to $end: $logfiles $EXTRA_LOGS"
+-trap '[ -z "$pattfile" ] || rm -f "$pattfile"' 0
+-pattfile=`mktemp` || fatal "cannot create temporary files"
+-for p in $LOG_PATTERNS; do
+- echo "$p"
+-done > $pattfile
+-
+-for l in $logfiles $EXTRA_LOGS; do
+- b="$(basename $l).extract.txt"
+-
+- if [ ! -f "$l" ]; then
+- # Not a file
+- continue
+-
+- elif [ -f "$b" ]; then
+- # We already have it
+- continue
+- fi
+-
+- dumplogset "$l" $LOG_START $LOG_END > "$b"
+- sanitize "$b"
+-
+- echo "Log patterns $REPORT_TARGET:" > $ANALYSIS_F
+- grep -f "$pattfile" "$b" >> $ANALYSIS_F
+-done
+-
+-which journalctl > /dev/null 2>&1
+-if [ $? = 0 ]; then
+- log "Including segment [$LOG_START-$LOG_END] from journald"
+- journalctl --since "$start" --until "$end" > journal.log
+- cat journal.log | grep -f $pattfile >> $ANALYSIS_F
+-fi
+-
+-rm -f $pattfile
+-trap "" 0
++collect_logs "$LOG_START" "$LOG_END" $logfiles
+
+ # Purge files containing no information
+ for f in `ls -1`; do
+@@ -838,7 +881,7 @@ for f in `ls -1`; do
+ done
+
+ # Parse for events
+-for l in $logfiles $EXTRA_LOGS; do
++for l in $logfiles; do
+ b="$(basename $l).extract.txt"
+ node_events "$b" > $EVENTS_F
+
+--
+1.8.3.1
+
diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec
index 2fbddc2..3f421c8 100644
--- a/SPECS/pacemaker.spec
+++ b/SPECS/pacemaker.spec
@@ -143,7 +143,7 @@
Name: pacemaker
Summary: Scalable High-Availability cluster resource manager
Version: %{pcmkversion}
-Release: %{pcmk_release}%{?dist}
+Release: %{pcmk_release}%{?dist}.2
%if %{defined _unitdir}
License: GPLv2+ and LGPLv2+
%else
@@ -226,6 +226,17 @@ Patch65: 065-coverity-cleanup.patch
Patch66: 066-forward-compat.patch
Patch67: 067-bundle-constraints.patch
Patch68: 068-bundle-weight-fix.patch
+Patch69: 069-ipc-refactor.patch
+Patch70: 070-check_positive_number.patch
+Patch71: 071-cluster-ipc-limit.patch
+Patch72: 072-bundle-placement.patch
+Patch73: 073-shutdown-logging.patch
+Patch74: 074-bundle-ordering.patch
+Patch75: 075-bundle-memory.patch
+Patch76: 076-quorum-loss.patch
+Patch77: 077-reenable-fence-device.patch
+Patch78: 078-fencing-memory.patch
+Patch79: 079-crm_report.patch
# patches that aren't from upstream
Patch100: lrmd-protocol-version.patch
@@ -895,6 +906,23 @@ exit 0
%attr(0644,root,root) %{_datadir}/pacemaker/nagios/plugins-metadata/*
%changelog
+* Fri Aug 18 2017 Ken Gaillot <kgaillot@redhat.com> - 1.1.16-12.2
+- Allow crm_report to work when no log files are specified
+- Resolves: rhbz#1482852
+
+%changelog
+* Tue Aug 15 2017 Ken Gaillot <kgaillot@redhat.com> - 1.1.16-12.1
+- Add cluster-ipc-limit option to avoid CIB eviction in large clusters
+- Implement ordering constraints involving bundles
+- Always re-check resource placement after quorum loss
+- Avoid stonithd crash when disabling fence device with queued actions
+- Allow re-enabled fence devices to be used
+- Resolves: rhbz#1478298
+- Resolves: rhbz#1481139
+- Resolves: rhbz#1481140
+- Resolves: rhbz#1481141
+- Resolves: rhbz#1481142
+
* Tue Jun 20 2017 Ken Gaillot <kgaillot@redhat.com> - 1.1.16-12
- Avoid unnecessary restarts when recovering remote connections
- Resolves: rhbz#1448773