diff --git a/SOURCES/069-ipc-refactor.patch b/SOURCES/069-ipc-refactor.patch new file mode 100644 index 0000000..22e8fda --- /dev/null +++ b/SOURCES/069-ipc-refactor.patch @@ -0,0 +1,126 @@ +From 769c210a4e9494757456c2896fdf67e2eb4b76c1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 9 Jun 2017 10:49:07 -0500 +Subject: [PATCH] Refactor: libcrmcommon,cib,lrmd: functionize allocating IPC + client object + +reduces duplication +--- + cib/remote.c | 6 +----- + include/crm/common/ipcs.h | 1 + + lib/common/ipc.c | 25 +++++++++++++++++++------ + lrmd/tls_backend.c | 5 +---- + 4 files changed, 22 insertions(+), 15 deletions(-) + +diff --git a/cib/remote.c b/cib/remote.c +index 9011552..0160c7e 100644 +--- a/cib/remote.c ++++ b/cib/remote.c +@@ -325,13 +325,9 @@ cib_remote_listen(gpointer data) + num_clients++; + + crm_client_init(); +- new_client = calloc(1, sizeof(crm_client_t)); ++ new_client = crm_client_alloc(NULL); + new_client->remote = calloc(1, sizeof(crm_remote_t)); + +- new_client->id = crm_generate_uuid(); +- +- g_hash_table_insert(client_connections, new_client->id /* Should work */ , new_client); +- + if (ssock == remote_tls_fd) { + #ifdef HAVE_GNUTLS_GNUTLS_H + new_client->kind = CRM_CLIENT_TLS; +diff --git a/include/crm/common/ipcs.h b/include/crm/common/ipcs.h +index ba1ccef..06cade9 100644 +--- a/include/crm/common/ipcs.h ++++ b/include/crm/common/ipcs.h +@@ -105,6 +105,7 @@ crm_client_t *crm_client_get(qb_ipcs_connection_t * c); + crm_client_t *crm_client_get_by_id(const char *id); + const char *crm_client_name(crm_client_t * c); + ++crm_client_t *crm_client_alloc(void *key); + crm_client_t *crm_client_new(qb_ipcs_connection_t * c, uid_t uid, gid_t gid); + void crm_client_destroy(crm_client_t * c); + void crm_client_disconnect_all(qb_ipcs_service_t *s); +diff --git a/lib/common/ipc.c b/lib/common/ipc.c +index d32e373..efe6480 100644 +--- a/lib/common/ipc.c ++++ b/lib/common/ipc.c +@@ -293,6 +293,24 @@ crm_client_disconnect_all(qb_ipcs_service_t *service) + } + } + ++/*! ++ * \brief Allocate a new crm_client_t object and generate its ID ++ * ++ * \param[in] key What to use as connections hash table key (NULL to use ID) ++ * ++ * \return Pointer to new crm_client_t (asserts on failure) ++ */ ++crm_client_t * ++crm_client_alloc(void *key) ++{ ++ crm_client_t *client = calloc(1, sizeof(crm_client_t)); ++ ++ CRM_ASSERT(client != NULL); ++ client->id = crm_generate_uuid(); ++ g_hash_table_insert(client_connections, (key? key : client->id), client); ++ return client; ++} ++ + crm_client_t * + crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client) + { +@@ -324,21 +342,16 @@ crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client) + crm_client_init(); + + /* TODO: Do our own auth checking, return NULL if unauthorized */ +- client = calloc(1, sizeof(crm_client_t)); +- ++ client = crm_client_alloc(c); + client->ipcs = c; + client->kind = CRM_CLIENT_IPC; + client->pid = crm_ipcs_client_pid(c); + +- client->id = crm_generate_uuid(); +- + crm_debug("Connecting %p for uid=%d gid=%d pid=%u id=%s", c, uid_client, gid_client, client->pid, client->id); + + #if ENABLE_ACL + client->user = uid2username(uid_client); + #endif +- +- g_hash_table_insert(client_connections, c, client); + return client; + } + +diff --git a/lrmd/tls_backend.c b/lrmd/tls_backend.c +index 8c36434..7d790cf 100644 +--- a/lrmd/tls_backend.c ++++ b/lrmd/tls_backend.c +@@ -212,11 +212,10 @@ lrmd_remote_listen(gpointer data) + return TRUE; + } + +- new_client = calloc(1, sizeof(crm_client_t)); ++ new_client = crm_client_alloc(NULL); + new_client->remote = calloc(1, sizeof(crm_remote_t)); + new_client->kind = CRM_CLIENT_TLS; + new_client->remote->tls_session = session; +- new_client->id = crm_generate_uuid(); + new_client->remote->auth_timeout = + g_timeout_add(LRMD_REMOTE_AUTH_TIMEOUT, lrmd_auth_timeout_cb, new_client); + crm_notice("LRMD client connection established. %p id: %s", new_client, new_client->id); +@@ -224,8 +223,6 @@ lrmd_remote_listen(gpointer data) + new_client->remote->source = + mainloop_add_fd("lrmd-remote-client", G_PRIORITY_DEFAULT, csock, new_client, + &lrmd_remote_fd_cb); +- g_hash_table_insert(client_connections, new_client->id, new_client); +- + return TRUE; + } + +-- +1.8.3.1 + diff --git a/SOURCES/070-check_positive_number.patch b/SOURCES/070-check_positive_number.patch new file mode 100644 index 0000000..7022a7a --- /dev/null +++ b/SOURCES/070-check_positive_number.patch @@ -0,0 +1,45 @@ +From 52153c502ad12c723231f5c7cb80d50a5d0691cd Mon Sep 17 00:00:00 2001 +From: aravind-kumar +Date: Wed, 15 Mar 2017 22:46:42 +0530 +Subject: [PATCH] check_positive_number + +--- + include/crm_internal.h | 1 + + lib/common/utils.c | 9 +++++++++ + 2 files changed, 10 insertions(+), 0 deletions(-) + +diff --git a/include/crm_internal.h b/include/crm_internal.h +index 96a68bd..31d4efe 100644 +--- a/include/crm_internal.h ++++ b/include/crm_internal.h +@@ -126,6 +126,7 @@ gboolean check_time(const char *value); + gboolean check_timer(const char *value); + gboolean check_boolean(const char *value); + gboolean check_number(const char *value); ++gboolean check_positive_number(const char *value); + gboolean check_quorum(const char *value); + gboolean check_script(const char *value); + gboolean check_utilization(const char *value); +diff --git a/lib/common/utils.c b/lib/common/utils.c +index dbf84c0..32c004d 100644 +--- a/lib/common/utils.c ++++ b/lib/common/utils.c +@@ -162,6 +162,15 @@ check_number(const char *value) + } + + gboolean ++check_positive_number(const char* value) ++{ ++ if (check_number(value) && (value[0] != '-') && !(safe_str_eq(value,"0"))) { ++ return TRUE; ++ } ++ return FALSE; ++} ++ ++gboolean + check_quorum(const char *value) + { + if (safe_str_eq(value, "stop")) { +-- +1.8.3.1 + diff --git a/SOURCES/071-cluster-ipc-limit.patch b/SOURCES/071-cluster-ipc-limit.patch new file mode 100644 index 0000000..7a0e350 --- /dev/null +++ b/SOURCES/071-cluster-ipc-limit.patch @@ -0,0 +1,517 @@ +From ae780515cd4db1e6f23db9f75a628ce9c39bdd49 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 9 Jun 2017 12:35:36 -0500 +Subject: [PATCH 1/8] Refactor: libcrmcommon: remember when IPC client is root + or cluster user + +will allow using a different eviction threshold +--- + include/crm/common/ipcs.h | 3 ++- + lib/common/ipc.c | 13 ++++++++++--- + 2 files changed, 12 insertions(+), 4 deletions(-) + +diff --git a/include/crm/common/ipcs.h b/include/crm/common/ipcs.h +index 52338e3..43b7b60 100644 +--- a/include/crm/common/ipcs.h ++++ b/include/crm/common/ipcs.h +@@ -60,7 +60,8 @@ struct crm_remote_s { + + enum crm_client_flags + { +- crm_client_flag_ipc_proxied = 0x00001, /* ipc_proxy code only */ ++ crm_client_flag_ipc_proxied = 0x00001, /* ipc_proxy code only */ ++ crm_client_flag_ipc_privileged = 0x00002, /* root or cluster user */ + }; + + struct crm_client_s { +diff --git a/lib/common/ipc.c b/lib/common/ipc.c +index 50980ec..e0a7a5c 100644 +--- a/lib/common/ipc.c ++++ b/lib/common/ipc.c +@@ -314,6 +314,7 @@ crm_client_alloc(void *key) + crm_client_t * + crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client) + { ++ static gid_t uid_cluster = 0; + static gid_t gid_cluster = 0; + + crm_client_t *client = NULL; +@@ -323,11 +324,12 @@ crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client) + return NULL; + } + +- if (gid_cluster == 0) { +- if(crm_user_lookup(CRM_DAEMON_USER, NULL, &gid_cluster) < 0) { ++ if (uid_cluster == 0) { ++ if (crm_user_lookup(CRM_DAEMON_USER, &uid_cluster, &gid_cluster) < 0) { + static bool have_error = FALSE; + if(have_error == FALSE) { +- crm_warn("Could not find group for user %s", CRM_DAEMON_USER); ++ crm_warn("Could not find user and group IDs for user %s", ++ CRM_DAEMON_USER); + have_error = TRUE; + } + } +@@ -347,6 +349,11 @@ crm_client_new(qb_ipcs_connection_t * c, uid_t uid_client, gid_t gid_client) + client->kind = CRM_CLIENT_IPC; + client->pid = crm_ipcs_client_pid(c); + ++ if ((uid_client == 0) || (uid_client == uid_cluster)) { ++ /* Remember when a connection came from root or hacluster */ ++ set_bit(client->flags, crm_client_flag_ipc_privileged); ++ } ++ + crm_debug("Connecting %p for uid=%d gid=%d pid=%u id=%s", c, uid_client, gid_client, client->pid, client->id); + + #if ENABLE_ACL +-- +1.8.3.1 + + +From f93ce6bdd6be5d2670ab0cd8dd10d3b8b9972a65 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 9 Jun 2017 14:46:16 -0500 +Subject: [PATCH 2/8] Low: libcrmcommon: add function for testing daemon name + +--- + include/crm/common/util.h | 2 ++ + lib/common/utils.c | 21 +++++++++++++++++++++ + 2 files changed, 23 insertions(+) + +diff --git a/include/crm/common/util.h b/include/crm/common/util.h +index 682b346..aa192f9 100644 +--- a/include/crm/common/util.h ++++ b/include/crm/common/util.h +@@ -26,6 +26,7 @@ + + # include + # include ++# include + # include + # include + # include +@@ -127,6 +128,7 @@ gboolean did_rsc_op_fail(lrmd_event_data_t * event, int target_rc); + char *crm_md5sum(const char *buffer); + + char *crm_generate_uuid(void); ++bool crm_is_daemon_name(const char *name); + + int crm_user_lookup(const char *name, uid_t * uid, gid_t * gid); + +diff --git a/lib/common/utils.c b/lib/common/utils.c +index 27ed60d..a652197 100644 +--- a/lib/common/utils.c ++++ b/lib/common/utils.c +@@ -1959,6 +1959,27 @@ crm_generate_uuid(void) + return buffer; + } + ++/*! ++ * \brief Check whether a string represents a cluster daemon name ++ * ++ * \param[in] name String to check ++ * ++ * \return TRUE if name is standard client name used by daemons, FALSE otherwise ++ */ ++bool ++crm_is_daemon_name(const char *name) ++{ ++ return (name && ++ (!strcmp(name, CRM_SYSTEM_CRMD) ++ || !strcmp(name, CRM_SYSTEM_STONITHD) ++ || !strcmp(name, T_ATTRD) ++ || !strcmp(name, CRM_SYSTEM_CIB) ++ || !strcmp(name, CRM_SYSTEM_MCP) ++ || !strcmp(name, CRM_SYSTEM_DC) ++ || !strcmp(name, CRM_SYSTEM_TENGINE) ++ || !strcmp(name, CRM_SYSTEM_LRMD))); ++} ++ + #include + + char * +-- +1.8.3.1 + + +From 19497fa9785c854084253167b7dc54a8c026e1ad Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 9 Jun 2017 11:11:50 -0500 +Subject: [PATCH 3/8] Low: libcrmcommon: support setting max queue length per + IPC client + +--- + include/crm/common/ipcs.h | 3 +++ + lib/common/ipc.c | 39 ++++++++++++++++++++++++++------------- + 2 files changed, 29 insertions(+), 13 deletions(-) + +diff --git a/include/crm/common/ipcs.h b/include/crm/common/ipcs.h +index 43b7b60..d2db212 100644 +--- a/include/crm/common/ipcs.h ++++ b/include/crm/common/ipcs.h +@@ -19,6 +19,7 @@ + #ifndef CRM_COMMON_IPCS__H + # define CRM_COMMON_IPCS__H + ++# include + # include + # ifdef HAVE_GNUTLS_GNUTLS_H + # undef KEYFILE +@@ -95,6 +96,7 @@ struct crm_client_s { + struct crm_remote_s *remote; /* TCP/TLS */ + + unsigned int queue_backlog; /* IPC queue length after last flush */ ++ unsigned int queue_max; /* Evict client whose queue grows this big */ + }; + + extern GHashTable *client_connections; +@@ -110,6 +112,7 @@ crm_client_t *crm_client_alloc(void *key); + crm_client_t *crm_client_new(qb_ipcs_connection_t * c, uid_t uid, gid_t gid); + void crm_client_destroy(crm_client_t * c); + void crm_client_disconnect_all(qb_ipcs_service_t *s); ++bool crm_set_client_queue_max(crm_client_t *client, const char *qmax); + + void crm_ipcs_send_ack(crm_client_t * c, uint32_t request, uint32_t flags, + const char *tag, const char *function, int line); +diff --git a/lib/common/ipc.c b/lib/common/ipc.c +index e0a7a5c..3110334 100644 +--- a/lib/common/ipc.c ++++ b/lib/common/ipc.c +@@ -37,8 +37,9 @@ + + #define PCMK_IPC_VERSION 1 + +-/* Evict clients whose event queue grows this large */ +-#define PCMK_IPC_MAX_QUEUE 500 ++/* Evict clients whose event queue grows this large (by default) */ ++#define PCMK_IPC_DEFAULT_QUEUE_MAX 500 ++#define PCMK_IPC_DEFAULT_QUEUE_MAX_S "500" + + struct crm_ipc_response_header { + struct qb_ipc_response_header qb; +@@ -409,6 +410,24 @@ crm_client_destroy(crm_client_t * c) + free(c); + } + ++/*! ++ * \brief Raise IPC eviction threshold for a client, if allowed ++ * ++ * \param[in,out] client Client to modify ++ * \param[in] queue_max New threshold (as string) ++ * ++ * \return TRUE if change was allowed, FALSE otherwise ++ */ ++bool ++crm_set_client_queue_max(crm_client_t *client, const char *qmax) ++{ ++ if (is_set(client->flags, crm_client_flag_ipc_privileged)) { ++ client->queue_max = crm_parse_int(qmax, PCMK_IPC_DEFAULT_QUEUE_MAX_S); ++ return TRUE; ++ } ++ return FALSE; ++} ++ + int + crm_ipcs_client_pid(qb_ipcs_connection_t * c) + { +@@ -553,18 +572,12 @@ crm_ipcs_flush_events(crm_client_t * c) + } + + if (queue_len) { +- /* We want to allow clients to briefly fall behind on processing +- * incoming messages, but drop completely unresponsive clients so the +- * connection doesn't consume resources indefinitely. +- * +- * @TODO It is possible that the queue could reasonably grow large in a +- * short time. An example is a reprobe of hundreds of resources on many +- * nodes resulting in a surge of CIB replies to the crmd. We could +- * possibly give cluster daemons a higher threshold here, and/or prevent +- * such a surge by throttling LRM history writes in the crmd. +- */ + +- if (queue_len > PCMK_IPC_MAX_QUEUE) { ++ /* Allow clients to briefly fall behind on processing incoming messages, ++ * but drop completely unresponsive clients so the connection doesn't ++ * consume resources indefinitely. ++ */ ++ if (queue_len > QB_MAX(c->queue_max, PCMK_IPC_DEFAULT_QUEUE_MAX)) { + if ((c->queue_backlog <= 1) || (queue_len < c->queue_backlog)) { + /* Don't evict for a new or shrinking backlog */ + crm_warn("Client with process ID %u has a backlog of %u messages " +-- +1.8.3.1 + + +From 324e241d9d9666dfe6543ddf88d965b12d5a164f Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 12 Jun 2017 17:47:12 -0500 +Subject: [PATCH 4/8] Low: libcib: always use current values when unpacking + config + +Previously, cib_read_config() called unpack_instance_attributes() with +overwrite=FALSE. This meant that changes to an option would not take effect +unless the option was not set before. + +The only significant use of cib_read_config() was in cib_acl_enabled(), which +used a new, empty hash table for every call, so the issue didn't matter. + +The cib daemon also used cib_read_config() to maintain a global config_hash, +which was affected by the issue, but didn't matter because it was never used. +This change will allow config_hash to be used. +--- + lib/cib/cib_utils.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index ab48f16..8aeed67 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -743,7 +743,7 @@ cib_read_config(GHashTable * options, xmlNode * current_cib) + config = get_object_root(XML_CIB_TAG_CRMCONFIG, current_cib); + if (config) { + unpack_instance_attributes(current_cib, config, XML_CIB_TAG_PROPSET, NULL, options, +- CIB_OPTIONS_FIRST, FALSE, now); ++ CIB_OPTIONS_FIRST, TRUE, now); + } + + verify_cib_options(options); +-- +1.8.3.1 + + +From b986acbe131216aaa372681e97dfc0f2ef8f70ad Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 13 Jun 2017 16:03:36 -0500 +Subject: [PATCH 5/8] Low: libcib: correctly search for v2 patchset changes + +cib_internal_config_changed() was never updated for v2 patch format +--- + lib/cib/cib_utils.c | 23 ++++++++++++----------- + 1 file changed, 12 insertions(+), 11 deletions(-) + +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index 8aeed67..f639ada 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -794,23 +794,24 @@ cib_apply_patch_event(xmlNode * event, xmlNode * input, xmlNode ** output, int l + return rc; + } + ++/* v2 and v2 patch formats */ ++#define XPATH_CONFIG_CHANGE \ ++ "//" XML_CIB_TAG_CRMCONFIG " | " \ ++ "//" XML_DIFF_CHANGE "[contains(@" XML_DIFF_PATH ",'/" XML_CIB_TAG_CRMCONFIG "/')]" ++ + gboolean +-cib_internal_config_changed(xmlNode * diff) ++cib_internal_config_changed(xmlNode *diff) + { + gboolean changed = FALSE; +- xmlXPathObject *xpathObj = NULL; + +- if (diff == NULL) { +- return FALSE; +- } ++ if (diff) { ++ xmlXPathObject *xpathObj = xpath_search(diff, XPATH_CONFIG_CHANGE); + +- xpathObj = xpath_search(diff, "//" XML_CIB_TAG_CRMCONFIG); +- if (numXpathResults(xpathObj) > 0) { +- changed = TRUE; ++ if (numXpathResults(xpathObj) > 0) { ++ changed = TRUE; ++ } ++ freeXpathObject(xpathObj); + } +- +- freeXpathObject(xpathObj); +- + return changed; + } + +-- +1.8.3.1 + + +From 3e5cdd9b329b3328a88912db48c4e4a4d6f94563 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 9 Jun 2017 15:33:27 -0500 +Subject: [PATCH 6/8] Feature: cib,libcib: support option for IPC eviction + threshold + +Only the cib utilizes the option currently, but the other daemons +could easily do the same, if they are subject to large IPC bursts. +--- + cib/callbacks.c | 13 +++++++++++++ + cib/callbacks.h | 10 ++++++++++ + lib/cib/cib_utils.c | 24 ++++++++++++++++++++---- + 3 files changed, 43 insertions(+), 4 deletions(-) + +diff --git a/cib/callbacks.c b/cib/callbacks.c +index 4708f10..544a920 100644 +--- a/cib/callbacks.c ++++ b/cib/callbacks.c +@@ -281,6 +281,19 @@ cib_common_callback(qb_ipcs_connection_t * c, void *data, size_t size, gboolean + cib_client->name = crm_itoa(cib_client->pid); + } else { + cib_client->name = strdup(value); ++ if (crm_is_daemon_name(value)) { ++ set_bit(cib_client->options, cib_is_daemon); ++ } ++ } ++ } ++ ++ /* Allow cluster daemons more leeway before being evicted */ ++ if (is_set(cib_client->options, cib_is_daemon)) { ++ const char *qmax = cib_config_lookup("cluster-ipc-limit"); ++ ++ if (crm_set_client_queue_max(cib_client, qmax)) { ++ crm_trace("IPC threshold for %s[%u] is now %u", ++ cib_client->name, cib_client->pid, cib_client->queue_max); + } + } + +diff --git a/cib/callbacks.h b/cib/callbacks.h +index b4d48d6..bddff09 100644 +--- a/cib/callbacks.h ++++ b/cib/callbacks.h +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -43,6 +44,9 @@ enum cib_notifications + cib_notify_replace = 0x0004, + cib_notify_confirm = 0x0008, + cib_notify_diff = 0x0010, ++ ++ /* not a notification, but uses the same IPC bitmask */ ++ cib_is_daemon = 0x1000, /* whether client is another cluster daemon */ + }; + /* *INDENT-ON* */ + +@@ -80,3 +84,9 @@ extern void cib_ha_peer_callback(HA_Message * msg, void *private_data); + extern int cib_ccm_dispatch(gpointer user_data); + extern void cib_ccm_msg_callback(oc_ed_t event, void *cookie, size_t size, const void *data); + #endif ++ ++static inline const char * ++cib_config_lookup(const char *opt) ++{ ++ return g_hash_table_lookup(config_hash, opt); ++} +diff --git a/lib/cib/cib_utils.c b/lib/cib/cib_utils.c +index f639ada..adbf9c8 100644 +--- a/lib/cib/cib_utils.c ++++ b/lib/cib/cib_utils.c +@@ -699,10 +699,26 @@ cib_native_notify(gpointer data, gpointer user_data) + } + + pe_cluster_option cib_opts[] = { +- /* name, old-name, validate, default, description */ +- {"enable-acl", NULL, "boolean", NULL, "false", &check_boolean, +- "Enable CIB ACL", NULL} +- , ++ /* ++ * name, legacy name, ++ * type, allowed values, default, validator, ++ * short description, ++ * long description ++ */ ++ { ++ "enable-acl", NULL, ++ "boolean", NULL, "false", &check_boolean, ++ "Enable CIB ACL", ++ NULL ++ }, ++ { ++ "cluster-ipc-limit", NULL, ++ "integer", NULL, "500", &check_positive_number, ++ "Maximum IPC message backlog before disconnecting a cluster daemon", ++ "Raise this if log has \"Evicting client\" messages for cluster daemon" ++ " PIDs (a good value is the number of resources in the cluster" ++ " multiplied by the number of nodes)" ++ }, + }; + + void +-- +1.8.3.1 + + +From 3f855794adbe9ad46aaca3848d591f5dc26d3371 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 13 Jun 2017 18:07:45 -0500 +Subject: [PATCH 7/8] Doc: Pacemaker Explained: document cluster-ipc-limit + +--- + doc/Pacemaker_Explained/en-US/Ch-Options.txt | 9 +++++++++ + 1 file changed, 9 insertions(+) + +diff --git a/doc/Pacemaker_Explained/en-US/Ch-Options.txt b/doc/Pacemaker_Explained/en-US/Ch-Options.txt +index f4e1af7..ec0c6b9 100644 +--- a/doc/Pacemaker_Explained/en-US/Ch-Options.txt ++++ b/doc/Pacemaker_Explained/en-US/Ch-Options.txt +@@ -267,6 +267,15 @@ take effect, we can optionally poll the cluster's status for changes. A value + of 0 disables polling. Positive values are an interval (in seconds unless other + SI units are specified, e.g. 5min). + ++| cluster-ipc-limit | 500 | ++indexterm:[cluster-ipc-limit,Cluster Option] ++indexterm:[Cluster,Option,cluster-ipc-limit] ++The maximum IPC message backlog before one cluster daemon will disconnect ++another. This is of use in large clusters, for which a good value is the number ++of resources in the cluster multiplied by the number of nodes. The default of ++500 is also the minimum. Raise this if you see "Evicting client" messages for ++cluster daemon PIDs in the logs. ++ + | pe-error-series-max | -1 | + indexterm:[pe-error-series-max,Cluster Option] + indexterm:[Cluster,Option,pe-error-series-max] +-- +1.8.3.1 + + +From bc6d723a94221419b15650a52e8ed1d843a32ff1 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 15 Jun 2017 14:20:27 -0500 +Subject: [PATCH 8/8] Refactor: libcrmcommon: avoid redundant constant + definitions + +--- + lib/common/ipc.c | 9 ++++++--- + 1 file changed, 6 insertions(+), 3 deletions(-) + +diff --git a/lib/common/ipc.c b/lib/common/ipc.c +index 3110334..c238bca 100644 +--- a/lib/common/ipc.c ++++ b/lib/common/ipc.c +@@ -39,7 +39,6 @@ + + /* Evict clients whose event queue grows this large (by default) */ + #define PCMK_IPC_DEFAULT_QUEUE_MAX 500 +-#define PCMK_IPC_DEFAULT_QUEUE_MAX_S "500" + + struct crm_ipc_response_header { + struct qb_ipc_response_header qb; +@@ -422,8 +421,12 @@ bool + crm_set_client_queue_max(crm_client_t *client, const char *qmax) + { + if (is_set(client->flags, crm_client_flag_ipc_privileged)) { +- client->queue_max = crm_parse_int(qmax, PCMK_IPC_DEFAULT_QUEUE_MAX_S); +- return TRUE; ++ int qmax_int = crm_int_helper(qmax, NULL); ++ ++ if ((errno == 0) && (qmax_int > 0)) { ++ client->queue_max = qmax_int; ++ return TRUE; ++ } + } + return FALSE; + } +-- +1.8.3.1 + diff --git a/SOURCES/072-bundle-placement.patch b/SOURCES/072-bundle-placement.patch new file mode 100644 index 0000000..2f669f5 --- /dev/null +++ b/SOURCES/072-bundle-placement.patch @@ -0,0 +1,74 @@ +From 648f048aeaee089ec74953107e6252543b710966 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 21 Jun 2017 20:40:34 -0500 +Subject: [PATCH] Fix: libpe_status: properly disallow other resources from + bundle nodes + +--- + lib/pengine/container.c | 34 +++++++++++++++++++++++++++++----- + 1 file changed, 29 insertions(+), 5 deletions(-) + +diff --git a/lib/pengine/container.c b/lib/pengine/container.c +index e916446..cf81253 100644 +--- a/lib/pengine/container.c ++++ b/lib/pengine/container.c +@@ -335,6 +335,29 @@ create_docker_resource( + return TRUE; + } + ++/*! ++ * \brief Ban a node from a resource's (and its children's) allowed nodes list ++ * ++ * \param[in,out] rsc Resource to modify ++ * \param[in] uname Name of node to ban ++ */ ++static void ++disallow_node(resource_t *rsc, const char *uname) ++{ ++ gpointer match = g_hash_table_lookup(rsc->allowed_nodes, uname); ++ ++ if (match) { ++ ((pe_node_t *) match)->weight = -INFINITY; ++ } ++ if (rsc->children) { ++ GListPtr child; ++ ++ for (child = rsc->children; child != NULL; child = child->next) { ++ disallow_node((resource_t *) (child->data), uname); ++ } ++ } ++} ++ + static bool + create_remote_resource( + resource_t *parent, container_variant_data_t *data, container_grouping_t *tuple, +@@ -413,7 +436,7 @@ create_remote_resource( + } + + /* unpack_remote_nodes() ensures that each remote node and guest node +- * has a node_t entry. Ideally, it would do the same for bundle nodes. ++ * has a pe_node_t entry. Ideally, it would do the same for bundle nodes. + * Unfortunately, a bundle has to be mostly unpacked before it's obvious + * what nodes will be needed, so we do it just above. + * +@@ -424,12 +447,13 @@ create_remote_resource( + * This adds a node *copy* to each resource's allowed nodes, and these + * copies will have the wrong weight. + * +- * As a hacky workaround, clear those copies here. ++ * As a hacky workaround, fix those copies here. ++ * ++ * @TODO Possible alternative: ensure bundles are unpacked before other ++ * resources, so the weight is correct before any copies are made. + */ + for (rsc_iter = data_set->resources; rsc_iter; rsc_iter = rsc_iter->next) { +- resource_t *rsc = (resource_t *) rsc_iter->data; +- +- g_hash_table_remove(rsc->allowed_nodes, uname); ++ disallow_node((resource_t *) (rsc_iter->data), uname); + } + + tuple->node = node_copy(node); +-- +1.8.3.1 + diff --git a/SOURCES/073-shutdown-logging.patch b/SOURCES/073-shutdown-logging.patch new file mode 100644 index 0000000..553634e --- /dev/null +++ b/SOURCES/073-shutdown-logging.patch @@ -0,0 +1,1616 @@ +From db1de472c1d1a4bef98b67f181a6e4fc8b5cbcfb Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Mon, 29 May 2017 19:24:50 +1000 +Subject: [PATCH] Log: PE: Improve logging of node fencing and shutdown + +--- + lib/pengine/unpack.c | 58 ++++++++++++---------- + lib/pengine/utils.c | 2 +- + pengine/allocate.c | 47 ++++++++++++++++-- + pengine/allocate.h | 3 +- + pengine/native.c | 4 +- + pengine/test10/594.summary | 2 + + pengine/test10/662.summary | 1 + + pengine/test10/797.summary | 1 + + pengine/test10/829.summary | 1 + + pengine/test10/bug-1572-1.summary | 1 + + pengine/test10/bug-1572-2.summary | 1 + + pengine/test10/bug-1573.summary | 1 + + pengine/test10/bug-1820-1.summary | 1 + + pengine/test10/bug-1820.summary | 1 + + pengine/test10/bug-1822.summary | 1 + + pengine/test10/bug-5028-bottom.summary | 1 + + pengine/test10/bug-5028-detach.summary | 1 + + pengine/test10/bug-5028.summary | 1 + + pengine/test10/bug-5186-partial-migrate.summary | 1 + + pengine/test10/bug-cl-5247.summary | 1 + + pengine/test10/bug-lf-2508.summary | 1 + + pengine/test10/bug-lf-2551.summary | 1 + + pengine/test10/bug-lf-2606.summary | 1 + + pengine/test10/bug-rh-1097457.summary | 1 + + .../test10/colocate-primitive-with-clone.summary | 1 + + pengine/test10/concurrent-fencing.summary | 3 ++ + pengine/test10/guest-node-host-dies.summary | 3 ++ + pengine/test10/inc12.summary | 6 +++ + pengine/test10/interleave-pseudo-stop.summary | 1 + + pengine/test10/master-7.summary | 1 + + pengine/test10/master-8.summary | 1 + + pengine/test10/master-9.summary | 1 + + pengine/test10/migrate-fencing.summary | 1 + + pengine/test10/migrate-shutdown.summary | 4 ++ + pengine/test10/novell-239082.summary | 1 + + pengine/test10/novell-252693.summary | 1 + + pengine/test10/params-2.summary | 1 + + pengine/test10/rec-node-11.summary | 1 + + pengine/test10/rec-node-12.summary | 1 + + pengine/test10/rec-node-13.summary | 1 + + pengine/test10/rec-node-14.summary | 3 ++ + pengine/test10/rec-node-15.summary | 1 + + pengine/test10/rec-node-2.summary | 1 + + pengine/test10/rec-node-4.summary | 1 + + pengine/test10/rec-node-6.summary | 1 + + pengine/test10/rec-node-7.summary | 1 + + pengine/test10/rec-rsc-5.summary | 1 + + .../test10/remote-fence-before-reconnect.summary | 1 + + pengine/test10/remote-fence-unclean.summary | 1 + + pengine/test10/remote-fence-unclean2.summary | 1 + + pengine/test10/remote-partial-migrate2.summary | 1 + + pengine/test10/remote-recover-all.summary | 3 ++ + pengine/test10/remote-recover-connection.summary | 1 + + pengine/test10/remote-recover-fail.summary | 1 + + pengine/test10/remote-recover-no-resources.summary | 2 + + pengine/test10/remote-recover-unknown.summary | 3 ++ + pengine/test10/remote-recovery.summary | 1 + + pengine/test10/remote-unclean2.summary | 1 + + pengine/test10/simple7.summary | 1 + + .../test10/start-then-stop-with-unfence.summary | 2 + + pengine/test10/stonith-0.summary | 2 + + pengine/test10/stonith-1.summary | 1 + + pengine/test10/stonith-2.summary | 1 + + pengine/test10/stonith-3.summary | 1 + + pengine/test10/stonith-4.summary | 4 ++ + pengine/test10/stop-failure-no-quorum.summary | 1 + + pengine/test10/stop-failure-with-fencing.summary | 1 + + pengine/test10/systemhealth1.summary | 2 + + pengine/test10/systemhealth2.summary | 1 + + pengine/test10/systemhealth3.summary | 1 + + pengine/test10/systemhealthm1.summary | 2 + + pengine/test10/systemhealthm2.summary | 1 + + pengine/test10/systemhealthm3.summary | 1 + + pengine/test10/systemhealthn1.summary | 2 + + pengine/test10/systemhealthn2.summary | 1 + + pengine/test10/systemhealthn3.summary | 1 + + pengine/test10/systemhealtho1.summary | 2 + + pengine/test10/systemhealtho2.summary | 1 + + pengine/test10/systemhealtho3.summary | 1 + + pengine/test10/systemhealthp1.summary | 2 + + pengine/test10/systemhealthp2.summary | 1 + + pengine/test10/systemhealthp3.summary | 1 + + pengine/test10/ticket-clone-21.summary | 2 + + pengine/test10/ticket-clone-9.summary | 2 + + pengine/test10/ticket-group-21.summary | 1 + + pengine/test10/ticket-group-9.summary | 1 + + pengine/test10/ticket-master-21.summary | 1 + + pengine/test10/ticket-master-9.summary | 1 + + pengine/test10/ticket-primitive-21.summary | 1 + + pengine/test10/ticket-primitive-9.summary | 1 + + pengine/test10/unfence-definition.summary | 5 ++ + pengine/test10/unfence-parameters.summary | 5 ++ + pengine/test10/unfence-startup.summary | 5 ++ + pengine/test10/unmanaged-master.summary | 2 + + pengine/test10/whitebox-fail1.summary | 1 + + pengine/test10/whitebox-fail2.summary | 1 + + .../test10/whitebox-imply-stop-on-fence.summary | 3 ++ + pengine/test10/whitebox-ms-ordering.summary | 2 + + .../test10/whitebox-unexpectedly-running.summary | 1 + + tools/crm_simulate.c | 1 + + 100 files changed, 223 insertions(+), 34 deletions(-) + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 8caf522..377100c 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -62,6 +62,7 @@ is_dangling_container_remote_node(node_t *node) + return FALSE; + } + ++ + void + pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + { +@@ -73,12 +74,12 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + + if (is_set(rsc->flags, pe_rsc_failed) == FALSE) { + if (!is_set(rsc->flags, pe_rsc_managed)) { +- crm_notice("Not fencing guest node %s because the container is " +- "unmanaged, otherwise we would do so recovering %s " +- "%s", node->details->uname, rsc->id, reason); ++ crm_notice("Not fencing node %s due to '%s': container %s is" ++ " unmanaged" ++ "%s", node->details->uname, reason, rsc->id); + } else { +- crm_warn("Guest node %s will be fenced (by recovering %s) %s", +- node->details->uname, rsc->id, reason); ++ crm_warn("Remote node %s will be fenced due to '%s' by recovering %s", ++ node->details->uname, rsc->id, reason); + + /* We don't mark the node as unclean because that would prevent the + * node from running resources. We want to allow it to run resources +@@ -88,8 +89,9 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + set_bit(rsc->flags, pe_rsc_failed); + } + } ++ + } else if (is_dangling_container_remote_node(node)) { +- crm_info("Cleaning up dangling connection resource for guest node %s %s" ++ crm_info("Cleaning up dangling connection resource for guest node %s due to '%s'" + " (fencing is already done, guest resource no longer exists)", + node->details->uname, reason); + set_bit(node->details->remote_rsc->flags, pe_rsc_failed); +@@ -98,28 +100,32 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + resource_t *rsc = node->details->remote_rsc; + + if (rsc && (!is_set(rsc->flags, pe_rsc_managed))) { +- crm_notice("Not fencing node %s because connection is unmanaged, " +- "otherwise would %s", node->details->uname, reason); ++ crm_notice("Not fencing node %s due to '%s': connection is unmanaged", ++ node->details->uname, reason); + } else if(node->details->remote_requires_reset == FALSE) { + node->details->remote_requires_reset = TRUE; + if (pe_can_fence(data_set, node)) { +- crm_warn("Node %s will be fenced %s", node->details->uname, reason); ++ crm_warn("Remote node %s will be fenced due to %s", node->details->uname, reason); + } else { +- crm_warn("Node %s is unclean %s", node->details->uname, reason); ++ crm_warn("Remote node %s is unclean due to %s", node->details->uname, reason); + } + } + node->details->unclean = TRUE; + +- } else if (node->details->unclean == FALSE) { ++ } else if (node->details->unclean) { + if (pe_can_fence(data_set, node)) { +- crm_warn("Node %s will be fenced %s", node->details->uname, reason); ++ crm_trace("Node %s would also be fenced due to '%s'", node->details->uname, reason); + } else { +- crm_warn("Node %s is unclean %s", node->details->uname, reason); ++ crm_trace("Node %s is also unclean due to '%s'", node->details->uname, reason); + } ++ ++ } else if (pe_can_fence(data_set, node)) { ++ crm_warn("Node %s will be fenced due to %s", node->details->uname, reason); + node->details->unclean = TRUE; + + } else { +- crm_trace("Node %s would also be fenced '%s'", node->details->uname, reason); ++ crm_warn("Node %s is unclean due to %s", node->details->uname, reason); ++ node->details->unclean = TRUE; + } + } + +@@ -1260,7 +1266,7 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set) + /* Everything else should flow from this automatically + * At least until the PE becomes able to migrate off healthy resources + */ +- pe_fence_node(data_set, this_node, "because the cluster does not have quorum"); ++ pe_fence_node(data_set, this_node, "cluster does not have quorum"); + } + } + } +@@ -1316,7 +1322,7 @@ determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_s + + } else { + /* mark it unclean */ +- pe_fence_node(data_set, this_node, "because node is unexpectedly down"); ++ pe_fence_node(data_set, this_node, "peer is unexpectedly down"); + crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s", + crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state)); + } +@@ -1373,10 +1379,10 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat + online = crm_is_true(is_peer); + + } else if (in_cluster == NULL) { +- pe_fence_node(data_set, this_node, "because the peer has not been seen by the cluster"); ++ pe_fence_node(data_set, this_node, "peer has not been seen by the cluster"); + + } else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) { +- pe_fence_node(data_set, this_node, "because it failed the pacemaker membership criteria"); ++ pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria"); + + } else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) { + +@@ -1395,14 +1401,14 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat + online = FALSE; + + } else if (crm_is_true(in_cluster) == FALSE) { +- pe_fence_node(data_set, this_node, "because the node is no longer part of the cluster"); ++ pe_fence_node(data_set, this_node, "peer is no longer part of the cluster"); + + } else if (crm_is_true(is_peer) == FALSE) { +- pe_fence_node(data_set, this_node, "because our peer process is no longer available"); ++ pe_fence_node(data_set, this_node, "peer process is no longer available"); + + /* Everything is running at this point, now check join state */ + } else if (do_terminate) { +- pe_fence_node(data_set, this_node, "because termination was requested"); ++ pe_fence_node(data_set, this_node, "termination was requested"); + + } else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) { + crm_info("Node %s is active", this_node->details->uname); +@@ -1414,7 +1420,7 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat + this_node->details->pending = TRUE; + + } else { +- pe_fence_node(data_set, this_node, "because the peer was in an unknown state"); ++ pe_fence_node(data_set, this_node, "peer was in an unknown state"); + crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d", + this_node->details->uname, crm_str(in_cluster), crm_str(is_peer), + crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown); +@@ -1923,14 +1929,14 @@ process_rsc_state(resource_t * rsc, node_t * node, + * another node without requiring the baremetal remote nodes to be fenced + * as well. */ + node->details->unseen = TRUE; +- reason = crm_strdup_printf("because %s is active there. Fencing will be revoked if remote-node connection can be re-established on another cluster-node.", rsc->id); ++ reason = crm_strdup_printf("%s is active there. Fencing will be revoked if remote-node connection can be re-established on another cluster-node.", rsc->id); + } + should_fence = TRUE; + } + + if (should_fence) { + if (reason == NULL) { +- reason = crm_strdup_printf("because %s is thought to be active there", rsc->id); ++ reason = crm_strdup_printf("%s is thought to be active there", rsc->id); + } + pe_fence_node(data_set, node, reason); + } +@@ -1953,7 +1959,7 @@ process_rsc_state(resource_t * rsc, node_t * node, + /* treat it as if it is still running + * but also mark the node as unclean + */ +- pe_fence_node(data_set, node, "because of resource failure(s)"); ++ pe_fence_node(data_set, node, "resource failure(s)"); + break; + + case action_fail_standby: +@@ -2009,7 +2015,7 @@ process_rsc_state(resource_t * rsc, node_t * node, + + /* connection resource to baremetal resource failed in a way that + * should result in fencing the remote-node. */ +- pe_fence_node(data_set, tmpnode, "because of connection failure(s)"); ++ pe_fence_node(data_set, tmpnode, "of connection failure(s)"); + } + } + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 7acd90c..44fbb51 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -517,7 +517,7 @@ custom_action(resource_t * rsc, char *key, const char *task, + if (is_set(action->rsc->flags, pe_rsc_managed) + && save_action && a_task == stop_rsc + && action->node->details->unclean == FALSE) { +- pe_fence_node(data_set, action->node, "because of unrunnable resource actions"); ++ pe_fence_node(data_set, action->node, "resource actions are unrunnable"); + } + + } else if (action->node->details->pending) { +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 795ed56..0020af6 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -467,7 +467,7 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki + set_bit(action_clear->flags, pe_action_runnable); + + crm_notice("Clearing failure of %s on %s " +- "because action definition changed " CRM_XS " %s", ++ "action definition changed " CRM_XS " %s", + rsc->id, node->details->uname, action_clear->uuid); + } + } +@@ -1798,7 +1798,7 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set) + CRM_ASSERT(container); + + if(is_set(container->flags, pe_rsc_failed)) { +- pe_fence_node(data_set, action->node, " because the container failed"); ++ pe_fence_node(data_set, action->node, "container failed"); + } + + crm_trace("%s %s %s %s %d", action->uuid, action->task, remote_rsc->id, container->id, is_set(container->flags, pe_rsc_failed)); +@@ -1967,7 +1967,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + * way to stop it, it is necessary to fence the + * node. + */ +- pe_fence_node(data_set, action->node, "because resources are active and the connection is unrecoverable"); ++ pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable"); + } + + custom_action_order(action->rsc, NULL, action, +@@ -2009,7 +2009,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + * Since we have no way to find out, it is + * necessary to fence the node. + */ +- pe_fence_node(data_set, action->node, "because resources are in an unknown state and the connection is unrecoverable"); ++ pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable"); + } + + if(cluster_node && state == remote_state_stopped) { +@@ -2270,6 +2270,7 @@ stage7(pe_working_set_t * data_set) + + crm_trace("Processing reloads"); + ++ LogNodeActions(data_set, FALSE); + for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { + resource_t *rsc = (resource_t *) gIter->data; + +@@ -2379,6 +2380,44 @@ stage8(pe_working_set_t * data_set) + } + + void ++LogNodeActions(pe_working_set_t * data_set, gboolean terminal) ++{ ++ GListPtr gIter = NULL; ++ ++ for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { ++ char *node_name = NULL; ++ const char *task = NULL; ++ action_t *action = (action_t *) gIter->data; ++ ++ if (action->rsc != NULL) { ++ continue; ++ } ++ ++ if (is_container_remote_node(action->node)) { ++ node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id); ++ } else if(action->node) { ++ node_name = crm_strdup_printf("%s", action->node->details->uname); ++ } ++ ++ if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) { ++ task = "Shutdown"; ++ } else if (safe_str_eq(action->task, CRM_OP_FENCE)) { ++ task = "Fence"; ++ } ++ ++ if(task == NULL) { ++ /* Nothing to report */ ++ } else if(terminal) { ++ printf(" * %s %s\n", task, node_name); ++ } else { ++ crm_notice(" * %s %s\n", task, node_name); ++ } ++ ++ free(node_name); ++ } ++} ++ ++void + cleanup_alloc_calculations(pe_working_set_t * data_set) + { + if (data_set == NULL) { +diff --git a/pengine/allocate.h b/pengine/allocate.h +index 3d7b7ce..d89943d 100644 +--- a/pengine/allocate.h ++++ b/pengine/allocate.h +@@ -152,7 +152,8 @@ extern gboolean unpack_location(xmlNode * xml_obj, pe_working_set_t * data_set); + + extern gboolean unpack_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set); + +-extern void LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal); ++void LogNodeActions(pe_working_set_t * data_set, gboolean terminal); ++void LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal); + void container_LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal); + + extern void cleanup_alloc_calculations(pe_working_set_t * data_set); +diff --git a/pengine/native.c b/pengine/native.c +index 2c4e2a1..dd5ff18 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -1818,7 +1818,7 @@ rsc_ticket_constraint(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working + for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + +- pe_fence_node(data_set, node, "because deadman ticket was lost"); ++ pe_fence_node(data_set, node, "deadman ticket was lost"); + } + break; + +@@ -2782,7 +2782,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + + if (force == FALSE && running != NULL) { + /* we already know the status of the resource on this node */ +- pe_rsc_trace(rsc, "Skipping active: %s on %s", rsc->id, node->details->uname); ++ pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname); + return FALSE; + } + +diff --git a/pengine/test10/594.summary b/pengine/test10/594.summary +index 6c208ee..d9fe8c1 100644 +--- a/pengine/test10/594.summary ++++ b/pengine/test10/594.summary +@@ -13,6 +13,8 @@ Online: [ hadev1 hadev2 ] + child_DoFencing:2 (stonith:ssh): Started hadev1 + + Transition Summary: ++ * Fence hadev3 ++ * Shutdown hadev2 + * Move DcIPaddr (Started hadev2 -> hadev1) + * Move rsc_hadev2 (Started hadev2 -> hadev1) + * Stop child_DoFencing:0 (hadev2) +diff --git a/pengine/test10/662.summary b/pengine/test10/662.summary +index 8d73b1d..1726f35 100644 +--- a/pengine/test10/662.summary ++++ b/pengine/test10/662.summary +@@ -14,6 +14,7 @@ Online: [ c001n02 c001n03 c001n04 c001n09 ] + child_DoFencing:3 (stonith:ssh): Started c001n09 + + Transition Summary: ++ * Shutdown c001n02 + * Move rsc_c001n02 (Started c001n02 -> c001n03) + * Stop child_DoFencing:0 (c001n02) + +diff --git a/pengine/test10/797.summary b/pengine/test10/797.summary +index 6e78255..3184eae 100644 +--- a/pengine/test10/797.summary ++++ b/pengine/test10/797.summary +@@ -15,6 +15,7 @@ Online: [ c001n01 c001n02 c001n03 ] + child_DoFencing:3 (stonith:ssh): Stopped + + Transition Summary: ++ * Shutdown c001n02 + * Stop DcIPaddr (Started c001n03) + * Stop rsc_c001n08 (Started c001n02) + * Stop rsc_c001n02 (Started c001n02) +diff --git a/pengine/test10/829.summary b/pengine/test10/829.summary +index d95a2ff..a9d25e0 100644 +--- a/pengine/test10/829.summary ++++ b/pengine/test10/829.summary +@@ -15,6 +15,7 @@ Online: [ c001n01 c001n03 c001n08 ] + child_DoFencing:3 (stonith:ssh): Started c001n08 + + Transition Summary: ++ * Fence c001n02 + * Move rsc_c001n02 (Started c001n02 -> c001n01) + * Stop child_DoFencing:0 (c001n02) + +diff --git a/pengine/test10/bug-1572-1.summary b/pengine/test10/bug-1572-1.summary +index 4280f7b..6c37bb4 100644 +--- a/pengine/test10/bug-1572-1.summary ++++ b/pengine/test10/bug-1572-1.summary +@@ -11,6 +11,7 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ] + IPaddr_147_81_84_133 (ocf::heartbeat:IPaddr): Started arc-tkincaidlx.wsicorp.com + + Transition Summary: ++ * Shutdown arc-dknightlx + * Stop rsc_drbd_7788:0 (arc-dknightlx) + * Restart rsc_drbd_7788:1 (Master arc-tkincaidlx.wsicorp.com) + * Restart fs_mirror (Started arc-tkincaidlx.wsicorp.com) +diff --git a/pengine/test10/bug-1572-2.summary b/pengine/test10/bug-1572-2.summary +index 6174027..a4235a7 100644 +--- a/pengine/test10/bug-1572-2.summary ++++ b/pengine/test10/bug-1572-2.summary +@@ -11,6 +11,7 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ] + IPaddr_147_81_84_133 (ocf::heartbeat:IPaddr): Started arc-tkincaidlx.wsicorp.com + + Transition Summary: ++ * Shutdown arc-dknightlx + * Stop rsc_drbd_7788:0 (arc-dknightlx) + * Demote rsc_drbd_7788:1 (Master -> Slave arc-tkincaidlx.wsicorp.com) + * Stop fs_mirror (arc-tkincaidlx.wsicorp.com) +diff --git a/pengine/test10/bug-1573.summary b/pengine/test10/bug-1573.summary +index 0c1fe3d..8fb2820 100644 +--- a/pengine/test10/bug-1573.summary ++++ b/pengine/test10/bug-1573.summary +@@ -11,6 +11,7 @@ OFFLINE: [ xen-c ] + apache_6 (ocf::heartbeat:apache): Stopped + + Transition Summary: ++ * Shutdown xen-b + * Stop IPaddr_192_168_1_102 (xen-b) + + Executing cluster transition: +diff --git a/pengine/test10/bug-1820-1.summary b/pengine/test10/bug-1820-1.summary +index 5edcbcb..db41ac4 100644 +--- a/pengine/test10/bug-1820-1.summary ++++ b/pengine/test10/bug-1820-1.summary +@@ -8,6 +8,7 @@ Online: [ star world ] + test2 (ocf::heartbeat:Xen): Started star + + Transition Summary: ++ * Shutdown star + * Start p1 (world) + * Migrate test1 (Started star -> world) + * Migrate test2 (Started star -> world) +diff --git a/pengine/test10/bug-1820.summary b/pengine/test10/bug-1820.summary +index 592fa67..6d9c021 100644 +--- a/pengine/test10/bug-1820.summary ++++ b/pengine/test10/bug-1820.summary +@@ -7,6 +7,7 @@ Online: [ star world ] + test2 (ocf::heartbeat:Xen): Started star + + Transition Summary: ++ * Shutdown star + * Migrate test1 (Started star -> world) + * Migrate test2 (Started star -> world) + +diff --git a/pengine/test10/bug-1822.summary b/pengine/test10/bug-1822.summary +index afb9fd1..325e408 100644 +--- a/pengine/test10/bug-1822.summary ++++ b/pengine/test10/bug-1822.summary +@@ -11,6 +11,7 @@ Online: [ process1a process2b ] + master_slave_procdctl:1 (ocf::heartbeat:procdctl): Master process1a + + Transition Summary: ++ * Shutdown process1a + * Demote master_slave_Stateful:1 (Master -> Stopped process1a) + * Demote master_slave_procdctl:1 (Master -> Stopped process1a) + +diff --git a/pengine/test10/bug-5028-bottom.summary b/pengine/test10/bug-5028-bottom.summary +index fc5cf8f..b43ba4e 100644 +--- a/pengine/test10/bug-5028-bottom.summary ++++ b/pengine/test10/bug-5028-bottom.summary +@@ -7,6 +7,7 @@ Online: [ bl460g6a bl460g6b ] + dummy02 (ocf::heartbeat:Dummy-stop-NG): Started bl460g6a + + Transition Summary: ++ * Shutdown bl460g6a + * Stop dummy02 (bl460g6a) + + Executing cluster transition: +diff --git a/pengine/test10/bug-5028-detach.summary b/pengine/test10/bug-5028-detach.summary +index 5e93b2a..712bfa1 100644 +--- a/pengine/test10/bug-5028-detach.summary ++++ b/pengine/test10/bug-5028-detach.summary +@@ -10,6 +10,7 @@ Online: [ bl460g6a bl460g6b ] + dummy02 (ocf::heartbeat:Dummy-stop-NG): FAILED bl460g6a ( blocked ) + + Transition Summary: ++ * Shutdown bl460g6a + + Executing cluster transition: + * Cluster action: do_shutdown on bl460g6a +diff --git a/pengine/test10/bug-5028.summary b/pengine/test10/bug-5028.summary +index ad7657c..a85f75b 100644 +--- a/pengine/test10/bug-5028.summary ++++ b/pengine/test10/bug-5028.summary +@@ -7,6 +7,7 @@ Online: [ bl460g6a bl460g6b ] + dummy02 (ocf::heartbeat:Dummy-stop-NG): FAILED bl460g6a ( blocked ) + + Transition Summary: ++ * Shutdown bl460g6a + * Stop dummy01 (Started bl460g6a - blocked) + + Executing cluster transition: +diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary +index 4d0b10e..63ed2d7 100644 +--- a/pengine/test10/bug-5186-partial-migrate.summary ++++ b/pengine/test10/bug-5186-partial-migrate.summary +@@ -25,6 +25,7 @@ Online: [ bl460g1n6 bl460g1n8 ] + Started: [ bl460g1n6 bl460g1n8 ] + + Transition Summary: ++ * Fence bl460g1n7 + * Move prmDummy (Started bl460g1n7 -> bl460g1n6) + * Move prmVM2 (Started bl460g1n7 -> bl460g1n8) + * Move prmStonith8-1 (Started bl460g1n7 -> bl460g1n6) +diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary +index a13754f..8ea3ff0 100644 +--- a/pengine/test10/bug-cl-5247.summary ++++ b/pengine/test10/bug-cl-5247.summary +@@ -18,6 +18,7 @@ Containers: [ pgsr01:prmDB1 ] + Stopped: [ bl460g8n3 bl460g8n4 ] + + Transition Summary: ++ * Fence pgsr02 (resource: prmDB2) + * Stop prmDB2 (bl460g8n4) + * Restart prmStonith1-2 (Started bl460g8n4) + * Restart prmStonith2-2 (Started bl460g8n3) +diff --git a/pengine/test10/bug-lf-2508.summary b/pengine/test10/bug-lf-2508.summary +index af5e4e2..7b436d4 100644 +--- a/pengine/test10/bug-lf-2508.summary ++++ b/pengine/test10/bug-lf-2508.summary +@@ -35,6 +35,7 @@ Online: [ srv01 srv03 srv04 ] + Stopped: [ srv04 ] + + Transition Summary: ++ * Fence srv02 + * Start Dummy01 (srv01) + * Move Dummy02 (Started srv02 -> srv04) + * Stop prmStonith1-1:1 (srv02) +diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary +index 158eb73..ffb7c6d 100644 +--- a/pengine/test10/bug-lf-2551.summary ++++ b/pengine/test10/bug-lf-2551.summary +@@ -81,6 +81,7 @@ Online: [ hex-0 hex-7 hex-8 ] + vm-64 (ocf::heartbeat:Xen): Stopped + + Transition Summary: ++ * Fence hex-9 + * Move fencing-sbd (Started hex-9 -> hex-0) + * Move dummy1 (Started hex-9 -> hex-0) + * Stop dlm:3 (hex-9) +diff --git a/pengine/test10/bug-lf-2606.summary b/pengine/test10/bug-lf-2606.summary +index 3e74d4b..ab93bb3 100644 +--- a/pengine/test10/bug-lf-2606.summary ++++ b/pengine/test10/bug-lf-2606.summary +@@ -12,6 +12,7 @@ Online: [ node1 ] + Slaves: [ node1 ] + + Transition Summary: ++ * Fence node2 + * Stop rsc1 (node2) + * Move rsc2 (Started node2 -> node1) + * Demote rsc3:1 (Master -> Stopped node2) +diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary +index f8d2c14..8fe474a 100644 +--- a/pengine/test10/bug-rh-1097457.summary ++++ b/pengine/test10/bug-rh-1097457.summary +@@ -32,6 +32,7 @@ Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ] + Started: [ lamaVM1 lamaVM2 lamaVM3 ] + + Transition Summary: ++ * Fence lamaVM2 (resource: VM2) + * Recover VM2 (Started lama3) + * Recover FSlun3 (Started lamaVM2 -> lama2) + * Restart FAKE4 (Started lamaVM2) +diff --git a/pengine/test10/colocate-primitive-with-clone.summary b/pengine/test10/colocate-primitive-with-clone.summary +index 5e4c511..e0f685f 100644 +--- a/pengine/test10/colocate-primitive-with-clone.summary ++++ b/pengine/test10/colocate-primitive-with-clone.summary +@@ -53,6 +53,7 @@ Online: [ srv01 srv02 srv03 srv04 ] + Stopped: [ srv01 ] + + Transition Summary: ++ * Shutdown srv01 + * Start UmVIPcheck (srv04) + * Start UmIPaddr (srv04) + * Start UmDummy01 (srv04) +diff --git a/pengine/test10/concurrent-fencing.summary b/pengine/test10/concurrent-fencing.summary +index 10b2fde..a274c3b 100644 +--- a/pengine/test10/concurrent-fencing.summary ++++ b/pengine/test10/concurrent-fencing.summary +@@ -8,6 +8,9 @@ Node node3 (uuid3): UNCLEAN (offline) + lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped + + Transition Summary: ++ * Fence node3 ++ * Fence node2 ++ * Fence node1 + + Executing cluster transition: + * Fencing node3 (reboot) +diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary +index c6180ca..717c43d 100644 +--- a/pengine/test10/guest-node-host-dies.summary ++++ b/pengine/test10/guest-node-host-dies.summary +@@ -11,6 +11,9 @@ Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + + Transition Summary: ++ * Fence rhel7-1 ++ * Fence lxc2 (resource: container2) ++ * Fence lxc1 (resource: container1) + * Restart Fencing (Started rhel7-4) + * Move rsc_rhel7-1 (Started rhel7-1 -> rhel7-5) + * Recover container1 (Started rhel7-1 -> rhel7-2) +diff --git a/pengine/test10/inc12.summary b/pengine/test10/inc12.summary +index 3df5a59..5068b7e 100644 +--- a/pengine/test10/inc12.summary ++++ b/pengine/test10/inc12.summary +@@ -32,6 +32,12 @@ Online: [ c001n02 c001n03 c001n04 c001n05 c001n06 c001n07 ] + ocf_msdummy:11 (ocf::heartbeat:Stateful): Slave c001n02 + + Transition Summary: ++ * Shutdown c001n07 ++ * Shutdown c001n06 ++ * Shutdown c001n05 ++ * Shutdown c001n04 ++ * Shutdown c001n03 ++ * Shutdown c001n02 + * Stop ocf_192.168.100.181 (c001n02) + * Stop heartbeat_192.168.100.182 (c001n02) + * Stop ocf_192.168.100.183 (c001n02) +diff --git a/pengine/test10/interleave-pseudo-stop.summary b/pengine/test10/interleave-pseudo-stop.summary +index a682462..cf30da0 100644 +--- a/pengine/test10/interleave-pseudo-stop.summary ++++ b/pengine/test10/interleave-pseudo-stop.summary +@@ -17,6 +17,7 @@ Online: [ node2 ] + Started: [ node2 ] + + Transition Summary: ++ * Fence node1 + * Stop stonithclone:1 (node1) + * Stop evmsclone:1 (node1) + * Stop imagestoreclone:1 (node1) +diff --git a/pengine/test10/master-7.summary b/pengine/test10/master-7.summary +index 7472542..348b4ee 100644 +--- a/pengine/test10/master-7.summary ++++ b/pengine/test10/master-7.summary +@@ -29,6 +29,7 @@ Online: [ c001n02 c001n03 c001n08 ] + ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08 + + Transition Summary: ++ * Fence c001n01 + * Move DcIPaddr (Started c001n01 -> c001n03) + * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) + * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) +diff --git a/pengine/test10/master-8.summary b/pengine/test10/master-8.summary +index 450737b..b77c884 100644 +--- a/pengine/test10/master-8.summary ++++ b/pengine/test10/master-8.summary +@@ -29,6 +29,7 @@ Online: [ c001n02 c001n03 c001n08 ] + ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08 + + Transition Summary: ++ * Fence c001n01 + * Move DcIPaddr (Started c001n01 -> c001n03) + * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) + * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) +diff --git a/pengine/test10/master-9.summary b/pengine/test10/master-9.summary +index 4f57a79..c1cd4a1 100644 +--- a/pengine/test10/master-9.summary ++++ b/pengine/test10/master-9.summary +@@ -30,6 +30,7 @@ Online: [ ibm1 va1 ] + ocf_msdummy:7 (ocf::heartbeat:/usr/lib64/heartbeat/cts/OCFMSDummy): Stopped + + Transition Summary: ++ * Shutdown ibm1 + * Start DcIPaddr (va1 - blocked) + * Start ocf_127.0.0.11 (va1 - blocked) + * Start heartbeat_127.0.0.12 (va1 - blocked) +diff --git a/pengine/test10/migrate-fencing.summary b/pengine/test10/migrate-fencing.summary +index e8bcd45..831e49a 100644 +--- a/pengine/test10/migrate-fencing.summary ++++ b/pengine/test10/migrate-fencing.summary +@@ -22,6 +22,7 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Slaves: [ pcmk-1 pcmk-2 pcmk-3 ] + + Transition Summary: ++ * Fence pcmk-4 + * Stop FencingChild:0 (pcmk-4) + * Move r192.168.101.181 (Started pcmk-4 -> pcmk-1) + * Move r192.168.101.182 (Started pcmk-4 -> pcmk-1) +diff --git a/pengine/test10/migrate-shutdown.summary b/pengine/test10/migrate-shutdown.summary +index e634a5c..630d58d 100644 +--- a/pengine/test10/migrate-shutdown.summary ++++ b/pengine/test10/migrate-shutdown.summary +@@ -22,6 +22,10 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 pcmk-4 ] + Stopped: [ pcmk-3 ] + + Transition Summary: ++ * Shutdown pcmk-4 ++ * Shutdown pcmk-3 ++ * Shutdown pcmk-2 ++ * Shutdown pcmk-1 + * Stop Fencing (pcmk-1) + * Stop r192.168.122.105 (pcmk-2) + * Stop r192.168.122.106 (pcmk-2) +diff --git a/pengine/test10/novell-239082.summary b/pengine/test10/novell-239082.summary +index b2c28ca..b596de4 100644 +--- a/pengine/test10/novell-239082.summary ++++ b/pengine/test10/novell-239082.summary +@@ -8,6 +8,7 @@ Online: [ xen-1 xen-2 ] + Slaves: [ xen-2 ] + + Transition Summary: ++ * Shutdown xen-1 + * Move fs_1 (Started xen-1 -> xen-2) + * Promote drbd0:0 (Slave -> Master xen-2) + * Demote drbd0:1 (Master -> Stopped xen-1) +diff --git a/pengine/test10/novell-252693.summary b/pengine/test10/novell-252693.summary +index 47bad6f..23f0632 100644 +--- a/pengine/test10/novell-252693.summary ++++ b/pengine/test10/novell-252693.summary +@@ -15,6 +15,7 @@ Online: [ node1 node2 ] + sles10 (ocf::heartbeat:Xen): Started node1 + + Transition Summary: ++ * Shutdown node1 + * Stop stonithclone:1 (node1) + * Stop evmsdclone:1 (node1) + * Stop evmsclone:1 (node1) +diff --git a/pengine/test10/params-2.summary b/pengine/test10/params-2.summary +index 891ea33..f4169f2 100644 +--- a/pengine/test10/params-2.summary ++++ b/pengine/test10/params-2.summary +@@ -7,6 +7,7 @@ Online: [ node1 node2 node3 ] + rsc3 (lsb:apache): Stopped + + Transition Summary: ++ * Shutdown node1 + * Stop rsc1 (node1) + * Restart rsc2 (Started node2) + * Start rsc3 (node3) +diff --git a/pengine/test10/rec-node-11.summary b/pengine/test10/rec-node-11.summary +index 1ed0eab..eb967dd 100644 +--- a/pengine/test10/rec-node-11.summary ++++ b/pengine/test10/rec-node-11.summary +@@ -10,6 +10,7 @@ Online: [ node2 ] + rsc3 (heartbeat:apache): Started node2 + + Transition Summary: ++ * Fence node1 + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +diff --git a/pengine/test10/rec-node-12.summary b/pengine/test10/rec-node-12.summary +index 82a03ac..26f9b13 100644 +--- a/pengine/test10/rec-node-12.summary ++++ b/pengine/test10/rec-node-12.summary +@@ -15,6 +15,7 @@ Online: [ c001n01 c001n03 c001n08 ] + child_DoFencing:3 (stonith:ssh): Stopped + + Transition Summary: ++ * Fence c001n02 + * Start DcIPaddr (c001n08) + * Start rsc_c001n08 (c001n08) + * Start rsc_c001n02 (c001n01) +diff --git a/pengine/test10/rec-node-13.summary b/pengine/test10/rec-node-13.summary +index 7c6f52b..e273a79 100644 +--- a/pengine/test10/rec-node-13.summary ++++ b/pengine/test10/rec-node-13.summary +@@ -34,6 +34,7 @@ OFFLINE: [ c001n03 c001n05 ] + ocf_msdummy:11 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n07 + + Transition Summary: ++ * Fence c001n04 + * Stop ocf_msdummy:6 (c001n04) + + Executing cluster transition: +diff --git a/pengine/test10/rec-node-14.summary b/pengine/test10/rec-node-14.summary +index d97d943..532f3d9 100644 +--- a/pengine/test10/rec-node-14.summary ++++ b/pengine/test10/rec-node-14.summary +@@ -8,6 +8,9 @@ Node node3 (uuid3): UNCLEAN (offline) + lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped + + Transition Summary: ++ * Fence node3 ++ * Fence node2 ++ * Fence node1 + + Executing cluster transition: + * Fencing node1 (reboot) +diff --git a/pengine/test10/rec-node-15.summary b/pengine/test10/rec-node-15.summary +index 1c3e8ea..8d88680 100644 +--- a/pengine/test10/rec-node-15.summary ++++ b/pengine/test10/rec-node-15.summary +@@ -21,6 +21,7 @@ Online: [ sapcl01 ] + oralsnr_25 (ocf::heartbeat:oralsnr): Stopped + + Transition Summary: ++ * Fence sapcl03 + * Start stonith-1 (sapcl01) + * Move IPaddr_192_168_1_102 (Started sapcl02 -> sapcl01) + * Move LVM_12 (Started sapcl02 -> sapcl01) +diff --git a/pengine/test10/rec-node-2.summary b/pengine/test10/rec-node-2.summary +index 6dc5098..5c8db02 100644 +--- a/pengine/test10/rec-node-2.summary ++++ b/pengine/test10/rec-node-2.summary +@@ -14,6 +14,7 @@ Online: [ node2 ] + rsc6 (heartbeat:apache): Stopped + + Transition Summary: ++ * Fence node1 + * Start stonith-1 (node2) + * Start rsc1 (node2) + * Start rsc2 (node2) +diff --git a/pengine/test10/rec-node-4.summary b/pengine/test10/rec-node-4.summary +index a64580c..761573f 100644 +--- a/pengine/test10/rec-node-4.summary ++++ b/pengine/test10/rec-node-4.summary +@@ -8,6 +8,7 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + + Transition Summary: ++ * Fence node1 + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +diff --git a/pengine/test10/rec-node-6.summary b/pengine/test10/rec-node-6.summary +index cf02414..fb294fb 100644 +--- a/pengine/test10/rec-node-6.summary ++++ b/pengine/test10/rec-node-6.summary +@@ -8,6 +8,7 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 + + Transition Summary: ++ * Fence node1 + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +diff --git a/pengine/test10/rec-node-7.summary b/pengine/test10/rec-node-7.summary +index a64580c..761573f 100644 +--- a/pengine/test10/rec-node-7.summary ++++ b/pengine/test10/rec-node-7.summary +@@ -8,6 +8,7 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + + Transition Summary: ++ * Fence node1 + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +diff --git a/pengine/test10/rec-rsc-5.summary b/pengine/test10/rec-rsc-5.summary +index bfb9a3c..7bcb1a3 100644 +--- a/pengine/test10/rec-rsc-5.summary ++++ b/pengine/test10/rec-rsc-5.summary +@@ -8,6 +8,7 @@ Online: [ node1 ] + rsc2 (heartbeat:apache): Started node2 + + Transition Summary: ++ * Fence node2 + * Start stonith-1 (node1) + * Recover rsc1 (Started node2 -> node1) + * Move rsc2 (Started node2 -> node1) +diff --git a/pengine/test10/remote-fence-before-reconnect.summary b/pengine/test10/remote-fence-before-reconnect.summary +index 88ca48c..520f5cf 100644 +--- a/pengine/test10/remote-fence-before-reconnect.summary ++++ b/pengine/test10/remote-fence-before-reconnect.summary +@@ -12,6 +12,7 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + fake5 (ocf::heartbeat:Dummy): Started c7auto3 + + Transition Summary: ++ * Fence c7auto4 + * Stop c7auto4 (c7auto1) + * Move fake2 (Started c7auto4 -> c7auto1) + +diff --git a/pengine/test10/remote-fence-unclean.summary b/pengine/test10/remote-fence-unclean.summary +index cd246e4..06940fc 100644 +--- a/pengine/test10/remote-fence-unclean.summary ++++ b/pengine/test10/remote-fence-unclean.summary +@@ -11,6 +11,7 @@ Online: [ 18builder 18node1 18node2 ] + FAKE4 (ocf::heartbeat:Dummy): Started 18node1 + + Transition Summary: ++ * Fence remote1 + * Recover remote1 (Started 18node1) + * Move FAKE2 (Started remote1 -> 18builder) + * Move FAKE3 (Started 18builder -> 18node1) +diff --git a/pengine/test10/remote-fence-unclean2.summary b/pengine/test10/remote-fence-unclean2.summary +index 78ff784..96f5d69 100644 +--- a/pengine/test10/remote-fence-unclean2.summary ++++ b/pengine/test10/remote-fence-unclean2.summary +@@ -10,6 +10,7 @@ OFFLINE: [ rhel7-alt3 ] + fake (ocf::heartbeat:Dummy): Started rhel7-alt4 (UNCLEAN) + + Transition Summary: ++ * Fence rhel7-alt4 + * Stop fake (rhel7-alt4) + + Executing cluster transition: +diff --git a/pengine/test10/remote-partial-migrate2.summary b/pengine/test10/remote-partial-migrate2.summary +index 6a65975..197bd8c 100644 +--- a/pengine/test10/remote-partial-migrate2.summary ++++ b/pengine/test10/remote-partial-migrate2.summary +@@ -63,6 +63,7 @@ RemoteOFFLINE: [ pcmk_remote4 ] + FAKE50 (ocf::heartbeat:Dummy): Started pcmk_remote5 + + Transition Summary: ++ * Fence pcmk4 + * Migrate pcmk_remote2 (Started pcmk3 -> pcmk1) + * Start pcmk_remote4 (pcmk2) + * Migrate pcmk_remote5 (Started pcmk1 -> pcmk2) +diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary +index 881f449..3f29f70 100644 +--- a/pengine/test10/remote-recover-all.summary ++++ b/pengine/test10/remote-recover-all.summary +@@ -38,6 +38,9 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: ++ * Fence messaging-1 ++ * Fence galera-2 ++ * Fence controller-1 + * Stop messaging-1 (controller-1) + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +diff --git a/pengine/test10/remote-recover-connection.summary b/pengine/test10/remote-recover-connection.summary +index 57b5e01..43507af 100644 +--- a/pengine/test10/remote-recover-connection.summary ++++ b/pengine/test10/remote-recover-connection.summary +@@ -38,6 +38,7 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: ++ * Fence controller-1 + * Move messaging-1 (Started controller-1 -> controller-2) + * Move galera-0 (Started controller-1 -> controller-2) + * Move galera-2 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-recover-fail.summary b/pengine/test10/remote-recover-fail.summary +index 5953e34..ec2d701 100644 +--- a/pengine/test10/remote-recover-fail.summary ++++ b/pengine/test10/remote-recover-fail.summary +@@ -14,6 +14,7 @@ OFFLINE: [ rhel7-auto1 ] + FAKE6 (ocf::heartbeat:Dummy): Started rhel7-auto4 (UNCLEAN) + + Transition Summary: ++ * Fence rhel7-auto4 + * Recover rhel7-auto4 (Started rhel7-auto2) + * Start FAKE1 (rhel7-auto2) + * Move FAKE2 (Started rhel7-auto4 -> rhel7-auto3) +diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary +index 8bfeb43..48355a4 100644 +--- a/pengine/test10/remote-recover-no-resources.summary ++++ b/pengine/test10/remote-recover-no-resources.summary +@@ -38,6 +38,8 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: ++ * Fence messaging-1 ++ * Fence controller-1 + * Stop messaging-1 (controller-1) + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary +index 7562f12..212af4e 100644 +--- a/pengine/test10/remote-recover-unknown.summary ++++ b/pengine/test10/remote-recover-unknown.summary +@@ -38,6 +38,9 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: ++ * Fence messaging-1 ++ * Fence galera-2 ++ * Fence controller-1 + * Stop messaging-1 (controller-1) + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +diff --git a/pengine/test10/remote-recovery.summary b/pengine/test10/remote-recovery.summary +index 57b5e01..43507af 100644 +--- a/pengine/test10/remote-recovery.summary ++++ b/pengine/test10/remote-recovery.summary +@@ -38,6 +38,7 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: ++ * Fence controller-1 + * Move messaging-1 (Started controller-1 -> controller-2) + * Move galera-0 (Started controller-1 -> controller-2) + * Move galera-2 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-unclean2.summary b/pengine/test10/remote-unclean2.summary +index 0a73cbd..ad4af90 100644 +--- a/pengine/test10/remote-unclean2.summary ++++ b/pengine/test10/remote-unclean2.summary +@@ -7,6 +7,7 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ] + rhel7-auto4 (ocf::pacemaker:remote): FAILED rhel7-auto1 + + Transition Summary: ++ * Fence rhel7-auto4 + * Recover rhel7-auto4 (Started rhel7-auto1) + + Executing cluster transition: +diff --git a/pengine/test10/simple7.summary b/pengine/test10/simple7.summary +index 2c2818f..021c15f 100644 +--- a/pengine/test10/simple7.summary ++++ b/pengine/test10/simple7.summary +@@ -5,6 +5,7 @@ Online: [ node1 ] + rsc1 (heartbeat:apache): Started node1 + + Transition Summary: ++ * Shutdown node1 + * Stop rsc1 (node1) + + Executing cluster transition: +diff --git a/pengine/test10/start-then-stop-with-unfence.summary b/pengine/test10/start-then-stop-with-unfence.summary +index df7d9e3..4430cc0 100644 +--- a/pengine/test10/start-then-stop-with-unfence.summary ++++ b/pengine/test10/start-then-stop-with-unfence.summary +@@ -11,6 +11,8 @@ Online: [ rhel7-node1.example.com rhel7-node2.example.com ] + Stopped: [ rhel7-node1.example.com ] + + Transition Summary: ++ * Fence rhel7-node2.example.com ++ * Fence rhel7-node1.example.com + * Start mpath-node1 (rhel7-node1.example.com) + * Move ip1 (Started rhel7-node2.example.com -> rhel7-node1.example.com) + * Start jrummy:1 (rhel7-node1.example.com) +diff --git a/pengine/test10/stonith-0.summary b/pengine/test10/stonith-0.summary +index 0fcaf80..24008a1 100644 +--- a/pengine/test10/stonith-0.summary ++++ b/pengine/test10/stonith-0.summary +@@ -37,6 +37,8 @@ Online: [ c001n02 c001n04 c001n06 c001n07 c001n08 ] + ocf_msdummy:13 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n06 + + Transition Summary: ++ * Fence c001n05 ++ * Fence c001n03 + * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) + * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) + * Recover ocf_192.168.100.183 (Started c001n03 -> c001n02) +diff --git a/pengine/test10/stonith-1.summary b/pengine/test10/stonith-1.summary +index 589e402..8556021 100644 +--- a/pengine/test10/stonith-1.summary ++++ b/pengine/test10/stonith-1.summary +@@ -28,6 +28,7 @@ Online: [ sles-1 sles-2 sles-4 ] + ocf_msdummy:7 (ocf::heartbeat:Stateful): Stopped + + Transition Summary: ++ * Fence sles-3 + * Start r192.168.100.183 (sles-1) + * Move migrator (Started sles-3 -> sles-4) + * Move rsc_sles-3 (Started sles-3 -> sles-4) +diff --git a/pengine/test10/stonith-2.summary b/pengine/test10/stonith-2.summary +index b02c9b4..0f7cb99 100644 +--- a/pengine/test10/stonith-2.summary ++++ b/pengine/test10/stonith-2.summary +@@ -33,6 +33,7 @@ Online: [ sles-1 sles-2 sles-3 sles-4 sles-6 ] + ocf_msdummy:11 (ocf::heartbeat:Stateful): Slave sles-3 + + Transition Summary: ++ * Fence sles-5 + * Start rsc_sles-5 (sles-6) + + Executing cluster transition: +diff --git a/pengine/test10/stonith-3.summary b/pengine/test10/stonith-3.summary +index 64b21d4..cb25cc3 100644 +--- a/pengine/test10/stonith-3.summary ++++ b/pengine/test10/stonith-3.summary +@@ -8,6 +8,7 @@ Online: [ rh5node2 ] + Stopped: [ rh5node1 rh5node2 ] + + Transition Summary: ++ * Fence rh5node1 + * Start prmIpPostgreSQLDB (rh5node2) + * Start prmStonith:0 (rh5node2) + +diff --git a/pengine/test10/stonith-4.summary b/pengine/test10/stonith-4.summary +index 78118c5..dc7cd89 100644 +--- a/pengine/test10/stonith-4.summary ++++ b/pengine/test10/stonith-4.summary +@@ -14,6 +14,10 @@ OFFLINE: [ pcmk-4 pcmk-6 ] + Fencing (stonith:fence_xvm): Stopped + + Transition Summary: ++ * Fence pcmk-8 ++ * Fence pcmk-7 ++ * Fence pcmk-5 ++ * Fence pcmk-10 + * Start Fencing (pcmk-1 - blocked) + + Executing cluster transition: +diff --git a/pengine/test10/stop-failure-no-quorum.summary b/pengine/test10/stop-failure-no-quorum.summary +index 7c14ce2..d864f1a 100644 +--- a/pengine/test10/stop-failure-no-quorum.summary ++++ b/pengine/test10/stop-failure-no-quorum.summary +@@ -15,6 +15,7 @@ Online: [ pcmk-1 ] + Fencing (stonith:fence_xvm): Stopped + + Transition Summary: ++ * Fence pcmk-2 + * Start dlm:0 (pcmk-1 - blocked) + * Stop clvm:0 (pcmk-2) + * Start clvm:2 (pcmk-1 - blocked) +diff --git a/pengine/test10/stop-failure-with-fencing.summary b/pengine/test10/stop-failure-with-fencing.summary +index 72417e6..e01b6c4 100644 +--- a/pengine/test10/stop-failure-with-fencing.summary ++++ b/pengine/test10/stop-failure-with-fencing.summary +@@ -14,6 +14,7 @@ Online: [ pcmk-1 ] + Fencing (stonith:fence_xvm): Stopped + + Transition Summary: ++ * Fence pcmk-2 + * Start dlm:0 (pcmk-1 - blocked) + * Stop clvm:0 (pcmk-2) + * Start clvm:1 (pcmk-1 - blocked) +diff --git a/pengine/test10/systemhealth1.summary b/pengine/test10/systemhealth1.summary +index bbdbcaf..37b0b49 100644 +--- a/pengine/test10/systemhealth1.summary ++++ b/pengine/test10/systemhealth1.summary +@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d ++ * Fence hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealth2.summary b/pengine/test10/systemhealth2.summary +index 86c6bd4..a37ce18 100644 +--- a/pengine/test10/systemhealth2.summary ++++ b/pengine/test10/systemhealth2.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealth3.summary b/pengine/test10/systemhealth3.summary +index 86c6bd4..a37ce18 100644 +--- a/pengine/test10/systemhealth3.summary ++++ b/pengine/test10/systemhealth3.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealthm1.summary b/pengine/test10/systemhealthm1.summary +index bbdbcaf..37b0b49 100644 +--- a/pengine/test10/systemhealthm1.summary ++++ b/pengine/test10/systemhealthm1.summary +@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d ++ * Fence hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealthm2.summary b/pengine/test10/systemhealthm2.summary +index 86c6bd4..a37ce18 100644 +--- a/pengine/test10/systemhealthm2.summary ++++ b/pengine/test10/systemhealthm2.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealthm3.summary b/pengine/test10/systemhealthm3.summary +index 760023d..b10d3f2 100644 +--- a/pengine/test10/systemhealthm3.summary ++++ b/pengine/test10/systemhealthm3.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/systemhealthn1.summary b/pengine/test10/systemhealthn1.summary +index bbdbcaf..37b0b49 100644 +--- a/pengine/test10/systemhealthn1.summary ++++ b/pengine/test10/systemhealthn1.summary +@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d ++ * Fence hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealthn2.summary b/pengine/test10/systemhealthn2.summary +index 86c6bd4..a37ce18 100644 +--- a/pengine/test10/systemhealthn2.summary ++++ b/pengine/test10/systemhealthn2.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealthn3.summary b/pengine/test10/systemhealthn3.summary +index 86c6bd4..a37ce18 100644 +--- a/pengine/test10/systemhealthn3.summary ++++ b/pengine/test10/systemhealthn3.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealtho1.summary b/pengine/test10/systemhealtho1.summary +index bbdbcaf..37b0b49 100644 +--- a/pengine/test10/systemhealtho1.summary ++++ b/pengine/test10/systemhealtho1.summary +@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d ++ * Fence hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealtho2.summary b/pengine/test10/systemhealtho2.summary +index 760023d..b10d3f2 100644 +--- a/pengine/test10/systemhealtho2.summary ++++ b/pengine/test10/systemhealtho2.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/systemhealtho3.summary b/pengine/test10/systemhealtho3.summary +index 760023d..b10d3f2 100644 +--- a/pengine/test10/systemhealtho3.summary ++++ b/pengine/test10/systemhealtho3.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/systemhealthp1.summary b/pengine/test10/systemhealthp1.summary +index bbdbcaf..37b0b49 100644 +--- a/pengine/test10/systemhealthp1.summary ++++ b/pengine/test10/systemhealthp1.summary +@@ -8,6 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d ++ * Fence hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealthp2.summary b/pengine/test10/systemhealthp2.summary +index 94c63d6..6605e12 100644 +--- a/pengine/test10/systemhealthp2.summary ++++ b/pengine/test10/systemhealthp2.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) + +diff --git a/pengine/test10/systemhealthp3.summary b/pengine/test10/systemhealthp3.summary +index 760023d..b10d3f2 100644 +--- a/pengine/test10/systemhealthp3.summary ++++ b/pengine/test10/systemhealthp3.summary +@@ -8,6 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: ++ * Fence hs21d + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/ticket-clone-21.summary b/pengine/test10/ticket-clone-21.summary +index 5ae23ec..50df628 100644 +--- a/pengine/test10/ticket-clone-21.summary ++++ b/pengine/test10/ticket-clone-21.summary +@@ -7,6 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: ++ * Fence node2 ++ * Fence node1 + * Stop rsc_stonith (node1) + * Stop rsc1:0 (node1) + * Stop rsc1:1 (node2) +diff --git a/pengine/test10/ticket-clone-9.summary b/pengine/test10/ticket-clone-9.summary +index 5ae23ec..50df628 100644 +--- a/pengine/test10/ticket-clone-9.summary ++++ b/pengine/test10/ticket-clone-9.summary +@@ -7,6 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: ++ * Fence node2 ++ * Fence node1 + * Stop rsc_stonith (node1) + * Stop rsc1:0 (node1) + * Stop rsc1:1 (node2) +diff --git a/pengine/test10/ticket-group-21.summary b/pengine/test10/ticket-group-21.summary +index 9027d2c..eab14a9 100644 +--- a/pengine/test10/ticket-group-21.summary ++++ b/pengine/test10/ticket-group-21.summary +@@ -8,6 +8,7 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: ++ * Fence node2 + * Stop rsc1 (node2) + * Stop rsc2 (node2) + +diff --git a/pengine/test10/ticket-group-9.summary b/pengine/test10/ticket-group-9.summary +index 9027d2c..eab14a9 100644 +--- a/pengine/test10/ticket-group-9.summary ++++ b/pengine/test10/ticket-group-9.summary +@@ -8,6 +8,7 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: ++ * Fence node2 + * Stop rsc1 (node2) + * Stop rsc2 (node2) + +diff --git a/pengine/test10/ticket-master-21.summary b/pengine/test10/ticket-master-21.summary +index b228696..a107a38 100644 +--- a/pengine/test10/ticket-master-21.summary ++++ b/pengine/test10/ticket-master-21.summary +@@ -8,6 +8,7 @@ Online: [ node1 node2 ] + Slaves: [ node2 ] + + Transition Summary: ++ * Fence node1 + * Move rsc_stonith (Started node1 -> node2) + * Demote rsc1:0 (Master -> Stopped node1) + +diff --git a/pengine/test10/ticket-master-9.summary b/pengine/test10/ticket-master-9.summary +index b228696..a107a38 100644 +--- a/pengine/test10/ticket-master-9.summary ++++ b/pengine/test10/ticket-master-9.summary +@@ -8,6 +8,7 @@ Online: [ node1 node2 ] + Slaves: [ node2 ] + + Transition Summary: ++ * Fence node1 + * Move rsc_stonith (Started node1 -> node2) + * Demote rsc1:0 (Master -> Stopped node1) + +diff --git a/pengine/test10/ticket-primitive-21.summary b/pengine/test10/ticket-primitive-21.summary +index 6c523fe..918b743 100644 +--- a/pengine/test10/ticket-primitive-21.summary ++++ b/pengine/test10/ticket-primitive-21.summary +@@ -6,6 +6,7 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: ++ * Fence node2 + * Stop rsc1 (node2) + + Executing cluster transition: +diff --git a/pengine/test10/ticket-primitive-9.summary b/pengine/test10/ticket-primitive-9.summary +index 6c523fe..918b743 100644 +--- a/pengine/test10/ticket-primitive-9.summary ++++ b/pengine/test10/ticket-primitive-9.summary +@@ -6,6 +6,7 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: ++ * Fence node2 + * Stop rsc1 (node2) + + Executing cluster transition: +diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary +index 05f8003..03506a3 100644 +--- a/pengine/test10/unfence-definition.summary ++++ b/pengine/test10/unfence-definition.summary +@@ -12,6 +12,11 @@ Online: [ virt-1 virt-2 virt-3 ] + Stopped: [ virt-2 virt-3 virt-4 ] + + Transition Summary: ++ * Fence virt-4 ++ * Fence virt-4 ++ * Fence virt-3 ++ * Fence virt-2 ++ * Fence virt-1 + * Restart fencing (Started virt-1) + * Restart dlm:0 (Started virt-1) + * Start dlm:2 (virt-3) +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +index 41fed90..5cc4a4b 100644 +--- a/pengine/test10/unfence-parameters.summary ++++ b/pengine/test10/unfence-parameters.summary +@@ -12,6 +12,11 @@ Online: [ virt-1 virt-2 virt-3 ] + Stopped: [ virt-2 virt-3 virt-4 ] + + Transition Summary: ++ * Fence virt-4 ++ * Fence virt-4 ++ * Fence virt-3 ++ * Fence virt-2 ++ * Fence virt-1 + * Restart fencing (Started virt-1) + * Restart dlm:0 (Started virt-1) + * Restart dlm:1 (Started virt-2) +diff --git a/pengine/test10/unfence-startup.summary b/pengine/test10/unfence-startup.summary +index 76bc0fc..6cd38ad 100644 +--- a/pengine/test10/unfence-startup.summary ++++ b/pengine/test10/unfence-startup.summary +@@ -12,6 +12,11 @@ Online: [ virt-1 virt-2 virt-3 ] + Stopped: [ virt-2 virt-3 virt-4 ] + + Transition Summary: ++ * Fence virt-4 ++ * Fence virt-4 ++ * Fence virt-3 ++ * Fence virt-2 ++ * Fence virt-1 + * Start dlm:2 (virt-3) + * Start clvmd:1 (virt-2) + * Start clvmd:2 (virt-3) +diff --git a/pengine/test10/unmanaged-master.summary b/pengine/test10/unmanaged-master.summary +index 024179a..66a8748 100644 +--- a/pengine/test10/unmanaged-master.summary ++++ b/pengine/test10/unmanaged-master.summary +@@ -27,6 +27,8 @@ OFFLINE: [ pcmk-3 pcmk-4 ] + Stopped: [ pcmk-3 pcmk-4 ] + + Transition Summary: ++ * Shutdown pcmk-2 ++ * Shutdown pcmk-1 + + Executing cluster transition: + * Cluster action: do_shutdown on pcmk-2 +diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary +index 1872e9a..5f70a66 100644 +--- a/pengine/test10/whitebox-fail1.summary ++++ b/pengine/test10/whitebox-fail1.summary +@@ -14,6 +14,7 @@ Containers: [ lxc2:container2 ] + D (ocf::pacemaker:Dummy): Started 18node1 + + Transition Summary: ++ * Fence lxc1 (resource: container1) + * Recover container1 (Started 18node2) + * Recover M:4 (Started lxc1) + * Recover B (Started lxc1) +diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary +index 5db6588..2922f16 100644 +--- a/pengine/test10/whitebox-fail2.summary ++++ b/pengine/test10/whitebox-fail2.summary +@@ -14,6 +14,7 @@ Containers: [ lxc2:container2 ] + D (ocf::pacemaker:Dummy): Started 18node1 + + Transition Summary: ++ * Fence lxc1 (resource: container1) + * Recover container1 (Started 18node2) + * Recover M:4 (Started lxc1) + * Recover B (Started lxc1) +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary +index 50a3446..31cc4a5 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.summary ++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary +@@ -25,6 +25,9 @@ Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ] + vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01 + + Transition Summary: ++ * Fence lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) ++ * Fence lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) ++ * Fence kiff-01 + * Move fence-kiff-02 (Started kiff-01 -> kiff-02) + * Stop dlm:0 (kiff-01) + * Stop clvmd:0 (kiff-01) +diff --git a/pengine/test10/whitebox-ms-ordering.summary b/pengine/test10/whitebox-ms-ordering.summary +index 365cf8d..fcdef66 100644 +--- a/pengine/test10/whitebox-ms-ordering.summary ++++ b/pengine/test10/whitebox-ms-ordering.summary +@@ -9,6 +9,8 @@ Online: [ 18node1 18node2 18node3 ] + Stopped: [ 18node1 18node2 18node3 ] + + Transition Summary: ++ * Fence lxc2 (resource: container2) ++ * Fence lxc1 (resource: container1) + * Start container1 (18node1) + * Start container2 (18node1) + * Recover lxc-ms:0 (Master lxc1) +diff --git a/pengine/test10/whitebox-unexpectedly-running.summary b/pengine/test10/whitebox-unexpectedly-running.summary +index 0b5ca42..ed0a5bd 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.summary ++++ b/pengine/test10/whitebox-unexpectedly-running.summary +@@ -5,6 +5,7 @@ Online: [ 18builder ] + FAKE (ocf::pacemaker:Dummy): FAILED 18builder + + Transition Summary: ++ * Fence remote1 (resource: FAKE) + * Recover FAKE (Started 18builder) + * Start remote1 (18builder) + +diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c +index 2cc578d..aaaf0aa 100644 +--- a/tools/crm_simulate.c ++++ b/tools/crm_simulate.c +@@ -892,6 +892,7 @@ main(int argc, char **argv) + || modified ? "\n" : ""); + fflush(stdout); + ++ LogNodeActions(&data_set, TRUE); + for (gIter = data_set.resources; gIter != NULL; gIter = gIter->next) { + resource_t *rsc = (resource_t *) gIter->data; + +-- +1.8.3.1 + diff --git a/SOURCES/074-bundle-ordering.patch b/SOURCES/074-bundle-ordering.patch new file mode 100644 index 0000000..15d7335 --- /dev/null +++ b/SOURCES/074-bundle-ordering.patch @@ -0,0 +1,17008 @@ +From 557ad15ec500f8edf0356d3ffcd880965ae0de1a Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Thu, 15 Jun 2017 10:47:04 +1000 +Subject: [PATCH 1/8] Feature: PE: Implement bundle ordering + +--- + pengine/allocate.c | 8 ++-- + pengine/allocate.h | 4 +- + pengine/clone.c | 51 +++++++++++-------------- + pengine/container.c | 82 ++++++++++++++++++++++++++++++++++++++- + pengine/master.c | 108 ++++++++++++++++++++++------------------------------ + pengine/utils.c | 12 ++++++ + pengine/utils.h | 1 + + 7 files changed, 170 insertions(+), 96 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index f2987cc..8f15282 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1860,9 +1860,11 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set) + * recurring monitors to be restarted, even if just + * the connection was re-established + */ +- custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, +- action->rsc, NULL, action, +- pe_order_preserve | pe_order_runnable_left | pe_order_implies_then, data_set); ++ if(task != no_action) { ++ custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, ++ action->rsc, NULL, action, ++ pe_order_preserve | pe_order_runnable_left | pe_order_implies_then, data_set); ++ } + } else { + custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, + action->rsc, NULL, action, +diff --git a/pengine/allocate.h b/pengine/allocate.h +index d89943d..9a30b80 100644 +--- a/pengine/allocate.h ++++ b/pengine/allocate.h +@@ -179,6 +179,8 @@ gboolean update_action_flags(action_t * action, enum pe_action_flags flags, cons + gboolean update_action(action_t * action); + void complex_set_cmds(resource_t * rsc); + +- ++void master_promotion_constraints(resource_t * rsc, pe_working_set_t * data_set); ++void clone_create_pseudo_actions( ++ resource_t * rsc, GListPtr children, notify_data_t **start_notify, notify_data_t **stop_notify, pe_working_set_t * data_set); + + #endif +diff --git a/pengine/clone.c b/pengine/clone.c +index 51338d6..a51677a 100644 +--- a/pengine/clone.c ++++ b/pengine/clone.c +@@ -805,6 +805,17 @@ child_ordering_constraints(resource_t * rsc, pe_working_set_t * data_set) + void + clone_create_actions(resource_t * rsc, pe_working_set_t * data_set) + { ++ clone_variant_data_t *clone_data = NULL; ++ ++ get_clone_variant_data(clone_data, rsc); ++ clone_create_pseudo_actions(rsc, rsc->children, &clone_data->start_notify, &clone_data->stop_notify,data_set); ++ child_ordering_constraints(rsc, data_set); ++} ++ ++void ++clone_create_pseudo_actions( ++ resource_t * rsc, GListPtr children, notify_data_t **start_notify, notify_data_t **stop_notify, pe_working_set_t * data_set) ++{ + gboolean child_active = FALSE; + gboolean child_starting = FALSE; + gboolean child_stopping = FALSE; +@@ -816,14 +827,9 @@ clone_create_actions(resource_t * rsc, pe_working_set_t * data_set) + action_t *start = NULL; + action_t *started = NULL; + +- GListPtr gIter = rsc->children; +- clone_variant_data_t *clone_data = NULL; +- +- get_clone_variant_data(clone_data, rsc); +- + pe_rsc_trace(rsc, "Creating actions for %s", rsc->id); + +- for (; gIter != NULL; gIter = gIter->next) { ++ for (GListPtr gIter = children; gIter != NULL; gIter = gIter->next) { + resource_t *child_rsc = (resource_t *) gIter->data; + gboolean starting = FALSE; + gboolean stopping = FALSE; +@@ -839,42 +845,31 @@ clone_create_actions(resource_t * rsc, pe_working_set_t * data_set) + } + + /* start */ +- start = start_action(rsc, NULL, !child_starting); +- started = custom_action(rsc, started_key(rsc), +- RSC_STARTED, NULL, !child_starting, TRUE, data_set); +- +- update_action_flags(start, pe_action_pseudo | pe_action_runnable, __FUNCTION__, __LINE__); +- update_action_flags(started, pe_action_pseudo, __FUNCTION__, __LINE__); ++ start = create_pseudo_resource_op(rsc, RSC_START, !child_starting, TRUE, data_set); ++ started = create_pseudo_resource_op(rsc, RSC_STARTED, !child_starting, FALSE, data_set); + started->priority = INFINITY; + + if (child_active || child_starting) { + update_action_flags(started, pe_action_runnable, __FUNCTION__, __LINE__); + } + +- child_ordering_constraints(rsc, data_set); +- if (clone_data->start_notify == NULL) { +- clone_data->start_notify = +- create_notification_boundaries(rsc, RSC_START, start, started, data_set); ++ if (start_notify != NULL && *start_notify == NULL) { ++ *start_notify = create_notification_boundaries(rsc, RSC_START, start, started, data_set); + } + + /* stop */ +- stop = stop_action(rsc, NULL, !child_stopping); +- stopped = custom_action(rsc, stopped_key(rsc), +- RSC_STOPPED, NULL, !child_stopping, TRUE, data_set); +- ++ stop = create_pseudo_resource_op(rsc, RSC_STOP, !child_stopping, TRUE, data_set); ++ stopped = create_pseudo_resource_op(rsc, RSC_STOPPED, !child_stopping, TRUE, data_set); + stopped->priority = INFINITY; +- update_action_flags(stop, pe_action_pseudo | pe_action_runnable, __FUNCTION__, __LINE__); + if (allow_dependent_migrations) { + update_action_flags(stop, pe_action_migrate_runnable, __FUNCTION__, __LINE__); + } +- update_action_flags(stopped, pe_action_pseudo | pe_action_runnable, __FUNCTION__, __LINE__); +- if (clone_data->stop_notify == NULL) { +- clone_data->stop_notify = +- create_notification_boundaries(rsc, RSC_STOP, stop, stopped, data_set); + +- if (clone_data->stop_notify && clone_data->start_notify) { +- order_actions(clone_data->stop_notify->post_done, clone_data->start_notify->pre, +- pe_order_optional); ++ if (stop_notify != NULL && *stop_notify == NULL) { ++ *stop_notify = create_notification_boundaries(rsc, RSC_STOP, stop, stopped, data_set); ++ ++ if (*stop_notify && *start_notify) { ++ order_actions((*stop_notify)->post_done, (*start_notify)->pre, pe_order_optional); + } + } + } +diff --git a/pengine/container.c b/pengine/container.c +index 58f6fca..d648e0f 100644 +--- a/pengine/container.c ++++ b/pengine/container.c +@@ -122,6 +122,8 @@ container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + void + container_create_actions(resource_t * rsc, pe_working_set_t * data_set) + { ++ pe_action_t *action = NULL; ++ GListPtr containers = NULL; + container_variant_data_t *container_data = NULL; + + CRM_CHECK(rsc != NULL, return); +@@ -136,15 +138,32 @@ container_create_actions(resource_t * rsc, pe_working_set_t * data_set) + } + if(tuple->docker) { + tuple->docker->cmds->create_actions(tuple->docker, data_set); ++ containers = g_list_append(containers, tuple->docker); + } + if(tuple->remote) { + tuple->remote->cmds->create_actions(tuple->remote, data_set); + } + } + ++ clone_create_pseudo_actions(rsc, containers, NULL, NULL, data_set); ++ + if(container_data->child) { + container_data->child->cmds->create_actions(container_data->child, data_set); ++ ++ if(container_data->child->variant == pe_master) { ++ /* promote */ ++ action = create_pseudo_resource_op(rsc, RSC_PROMOTE, TRUE, TRUE, data_set); ++ action = create_pseudo_resource_op(rsc, RSC_PROMOTED, TRUE, TRUE, data_set); ++ action->priority = INFINITY; ++ ++ /* demote */ ++ action = create_pseudo_resource_op(rsc, RSC_DEMOTE, TRUE, TRUE, data_set); ++ action = create_pseudo_resource_op(rsc, RSC_DEMOTED, TRUE, TRUE, data_set); ++ action->priority = INFINITY; ++ } + } ++ ++ g_list_free(containers); + } + + void +@@ -155,12 +174,38 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + CRM_CHECK(rsc != NULL, return); + + get_container_variant_data(container_data, rsc); ++ ++ if(container_data->child) { ++ new_rsc_order(rsc, RSC_START, container_data->child, RSC_START, pe_order_implies_first_printed, data_set); ++ new_rsc_order(rsc, RSC_STOP, container_data->child, RSC_STOP, pe_order_implies_first_printed, data_set); ++ ++ if(container_data->child->children) { ++ new_rsc_order(container_data->child, RSC_STARTED, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set); ++ new_rsc_order(container_data->child, RSC_STOPPED, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set); ++ } else { ++ new_rsc_order(container_data->child, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set); ++ new_rsc_order(container_data->child, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set); ++ } ++ } ++ + for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { + container_grouping_t *tuple = (container_grouping_t *)gIter->data; + + CRM_ASSERT(tuple); +- if(tuple->docker) { +- tuple->docker->cmds->internal_constraints(tuple->docker, data_set); ++ CRM_ASSERT(tuple->docker); ++ ++ tuple->docker->cmds->internal_constraints(tuple->docker, data_set); ++ ++ order_start_start(rsc, tuple->docker, pe_order_runnable_left | pe_order_implies_first_printed); ++ ++ if(tuple->child) { ++ order_stop_stop(rsc, tuple->child, pe_order_implies_first_printed); ++ ++ } else { ++ order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed); ++ new_rsc_order(tuple->docker, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set); ++ new_rsc_order(tuple->docker, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, ++ data_set); + } + + if(tuple->ip) { +@@ -194,10 +239,31 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + + if(container_data->child) { + container_data->child->cmds->internal_constraints(container_data->child, data_set); ++ if(container_data->child->variant == pe_master) { ++ master_promotion_constraints(rsc, data_set); ++ ++ /* child demoted before global demoted */ ++ new_rsc_order(container_data->child, RSC_DEMOTED, rsc, RSC_DEMOTED, pe_order_implies_then_printed, data_set); ++ ++ /* global demote before child demote */ ++ new_rsc_order(rsc, RSC_DEMOTE, container_data->child, RSC_DEMOTE, pe_order_implies_first_printed, data_set); ++ ++ /* child promoted before global promoted */ ++ new_rsc_order(container_data->child, RSC_PROMOTED, rsc, RSC_PROMOTED, pe_order_implies_then_printed, data_set); ++ ++ /* global promote before child promote */ ++ new_rsc_order(rsc, RSC_PROMOTE, container_data->child, RSC_PROMOTE, pe_order_implies_first_printed, data_set); ++ } ++ ++ } else { ++// int type = pe_order_optional | pe_order_implies_then | pe_order_restart; ++// custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, ++// rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, pe_order_optional, data_set); + } + } + + ++ + static resource_t * + find_compatible_tuple_by_node(resource_t * rsc_lh, node_t * candidate, resource_t * rsc, + enum rsc_role_e filter, gboolean current) +@@ -336,6 +402,10 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum + enum pe_action_flags filter, enum pe_ordering type) + { + enum pe_graph_flags changed = pe_graph_none; ++ ++ // At the point we need to force container X to stop because ++ // resource Y needs to stop, here is where we'd implement that ++ + return changed; + } + +@@ -359,6 +429,14 @@ container_rsc_location(resource_t * rsc, rsc_to_node_t * constraint) + tuple->ip->cmds->rsc_location(tuple->ip, constraint); + } + } ++ ++ if(container_data->child && (constraint->role_filter == RSC_ROLE_SLAVE || constraint->role_filter == RSC_ROLE_MASTER)) { ++ // Translate the node into container names running on that node ++ crm_err("Applying constraint %s", constraint->id); ++ container_data->child->cmds->rsc_location(container_data->child, constraint); ++ container_data->child->rsc_location = g_list_prepend(container_data->child->rsc_location, constraint); ++ crm_err("Added %d location constraints to %s", g_list_length(container_data->child->rsc_location), container_data->child->id); ++ } + } + + void +diff --git a/pengine/master.c b/pengine/master.c +index f6fcad3..93e5186 100644 +--- a/pengine/master.c ++++ b/pengine/master.c +@@ -139,27 +139,26 @@ master_update_pseudo_status(resource_t * rsc, gboolean * demoting, gboolean * pr + } + } + +-#define apply_master_location(list) do { \ +- gIter2 = list; \ +- for(; gIter2 != NULL; gIter2 = gIter2->next) { \ +- rsc_to_node_t *cons = (rsc_to_node_t*)gIter2->data; \ +- \ +- cons_node = NULL; \ +- if(cons->role_filter == RSC_ROLE_MASTER) { \ +- pe_rsc_trace(rsc, "Applying %s to %s", \ +- cons->id, child_rsc->id); \ +- cons_node = pe_find_node_id( \ +- cons->node_list_rh, chosen->details->id); \ +- } \ +- if(cons_node != NULL) { \ +- int new_priority = merge_weights( \ +- child_rsc->priority, cons_node->weight); \ +- pe_rsc_trace(rsc, "\t%s: %d->%d (%d)", child_rsc->id, \ +- child_rsc->priority, new_priority, cons_node->weight); \ +- child_rsc->priority = new_priority; \ +- } \ +- } \ +- } while(0) ++static void apply_master_location(resource_t *child, GListPtr location_constraints, pe_node_t *chosen) ++{ ++ for(GListPtr gIter = location_constraints; gIter != NULL; gIter = gIter->next) { ++ pe_node_t *cons_node = NULL; ++ rsc_to_node_t *cons = (rsc_to_node_t*)gIter->data; ++ ++ if(cons->role_filter == RSC_ROLE_MASTER) { ++ pe_rsc_trace(child, "Applying %s to %s", cons->id, child->id); ++ cons_node = pe_find_node_id(cons->node_list_rh, chosen->details->id); ++ } ++ if(cons_node != NULL) { ++ int new_priority = merge_weights(child->priority, cons_node->weight); ++ pe_rsc_trace(child, "\t%s[%s]: %d -> %d (%d)", child->id, cons_node->details->uname, ++ child->priority, new_priority, cons_node->weight); ++ crm_err("\t%s[%s]: %d -> %d (%d)", child->id, cons_node->details->uname, ++ child->priority, new_priority, cons_node->weight); ++ child->priority = new_priority; ++ } ++ } ++} + + static node_t * + can_be_master(resource_t * rsc) +@@ -290,8 +289,7 @@ master_promotion_order(resource_t * rsc, pe_working_set_t * data_set) + pe_rsc_trace(rsc, "Merging weights for %s", rsc->id); + set_bit(rsc->flags, pe_rsc_merging); + +- gIter = rsc->children; +- for (; gIter != NULL; gIter = gIter->next) { ++ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { + resource_t *child = (resource_t *) gIter->data; + + pe_rsc_trace(rsc, "Sort index: %s = %d", child->id, child->sort_index); +@@ -639,7 +637,6 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + GHashTableIter iter; + node_t *node = NULL; + node_t *chosen = NULL; +- node_t *cons_node = NULL; + enum rsc_role_e next_role = RSC_ROLE_UNKNOWN; + + char score[33]; +@@ -672,8 +669,7 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + /* + * assign priority + */ +- gIter = rsc->children; +- for (; gIter != NULL; gIter = gIter->next) { ++ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { + GListPtr list = NULL; + resource_t *child_rsc = (resource_t *) gIter->data; + +@@ -723,11 +719,11 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + crm_err("Unknown resource role: %d for %s", next_role, child_rsc->id)); + } + +- apply_master_location(child_rsc->rsc_location); +- apply_master_location(rsc->rsc_location); ++ apply_master_location(child_rsc, child_rsc->rsc_location, chosen); ++ crm_err("Applying %d location constraints for %s", g_list_length(rsc->rsc_location), rsc->id); ++ apply_master_location(child_rsc, rsc->rsc_location, chosen); + +- gIter2 = child_rsc->rsc_cons; +- for (; gIter2 != NULL; gIter2 = gIter2->next) { ++ for (gIter2 = child_rsc->rsc_cons; gIter2 != NULL; gIter2 = gIter2->next) { + rsc_colocation_t *cons = (rsc_colocation_t *) gIter2->data; + + child_rsc->cmds->rsc_colocation_lh(child_rsc, cons->rsc_rh, cons); +@@ -746,8 +742,7 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + + /* mark the first N as masters */ + +- gIter = rsc->children; +- for (; gIter != NULL; gIter = gIter->next) { ++ for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { + resource_t *child_rsc = (resource_t *) gIter->data; + score2char_stack(child_rsc->sort_index, score, len); + +@@ -840,20 +835,9 @@ master_create_actions(resource_t * rsc, pe_working_set_t * data_set) + } + + /* promote */ +- action = promote_action(rsc, NULL, !any_promoting); +- action_complete = custom_action(rsc, promoted_key(rsc), +- RSC_PROMOTED, NULL, !any_promoting, TRUE, data_set); +- ++ action = create_pseudo_resource_op(rsc, RSC_PROMOTE, !any_promoting, TRUE, data_set); ++ action_complete = create_pseudo_resource_op(rsc, RSC_PROMOTED, !any_promoting, TRUE, data_set); + action_complete->priority = INFINITY; +- update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__); +- update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__); +- update_action_flags(action_complete, pe_action_pseudo, __FUNCTION__, __LINE__); +- update_action_flags(action_complete, pe_action_runnable, __FUNCTION__, __LINE__); +- +- if (clone_data->masters_allocated > 0) { +- update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__); +- update_action_flags(action_complete, pe_action_runnable, __FUNCTION__, __LINE__); +- } + + child_promoting_constraints(clone_data, pe_order_optional, + rsc, NULL, last_promote_rsc, data_set); +@@ -864,16 +848,10 @@ master_create_actions(resource_t * rsc, pe_working_set_t * data_set) + } + + /* demote */ +- action = demote_action(rsc, NULL, !any_demoting); +- action_complete = custom_action(rsc, demoted_key(rsc), +- RSC_DEMOTED, NULL, !any_demoting, TRUE, data_set); ++ action = create_pseudo_resource_op(rsc, RSC_DEMOTE, !any_demoting, TRUE, data_set); ++ action_complete = create_pseudo_resource_op(rsc, RSC_DEMOTED, !any_demoting, TRUE, data_set); + action_complete->priority = INFINITY; + +- update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__); +- update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__); +- update_action_flags(action_complete, pe_action_pseudo, __FUNCTION__, __LINE__); +- update_action_flags(action_complete, pe_action_runnable, __FUNCTION__, __LINE__); +- + child_demoting_constraints(clone_data, pe_order_optional, rsc, NULL, last_demote_rsc, data_set); + + if (clone_data->demote_notify == NULL) { +@@ -908,16 +886,8 @@ master_create_actions(resource_t * rsc, pe_working_set_t * data_set) + } + + void +-master_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) ++master_promotion_constraints(resource_t * rsc, pe_working_set_t * data_set) + { +- GListPtr gIter = rsc->children; +- resource_t *last_rsc = NULL; +- clone_variant_data_t *clone_data = NULL; +- +- get_clone_variant_data(clone_data, rsc); +- +- clone_internal_constraints(rsc, data_set); +- + /* global stopped before start */ + new_rsc_order(rsc, RSC_STOPPED, rsc, RSC_START, pe_order_optional, data_set); + +@@ -938,6 +908,20 @@ master_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + + /* global demoted before promote */ + new_rsc_order(rsc, RSC_DEMOTED, rsc, RSC_PROMOTE, pe_order_optional, data_set); ++} ++ ++ ++void ++master_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) ++{ ++ GListPtr gIter = rsc->children; ++ resource_t *last_rsc = NULL; ++ clone_variant_data_t *clone_data = NULL; ++ ++ get_clone_variant_data(clone_data, rsc); ++ ++ clone_internal_constraints(rsc, data_set); ++ master_promotion_constraints(rsc, data_set); + + for (; gIter != NULL; gIter = gIter->next) { + resource_t *child_rsc = (resource_t *) gIter->data; +diff --git a/pengine/utils.c b/pengine/utils.c +index a587e58..0cc6381 100644 +--- a/pengine/utils.c ++++ b/pengine/utils.c +@@ -422,3 +422,15 @@ can_run_any(GHashTable * nodes) + + return FALSE; + } ++ ++pe_action_t * ++create_pseudo_resource_op(resource_t * rsc, const char *task, bool optional, bool runnable, pe_working_set_t *data_set) ++{ ++ pe_action_t *action = custom_action(rsc, generate_op_key(rsc->id, task, 0), task, NULL, optional, TRUE, data_set); ++ update_action_flags(action, pe_action_pseudo, __FUNCTION__, __LINE__); ++ update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__); ++ if(runnable) { ++ update_action_flags(action, pe_action_runnable, __FUNCTION__, __LINE__); ++ } ++ return action; ++} +diff --git a/pengine/utils.h b/pengine/utils.h +index 79fd33d..10e7201 100644 +--- a/pengine/utils.h ++++ b/pengine/utils.h +@@ -71,6 +71,7 @@ extern void calculate_utilization(GHashTable * current_utilization, + GHashTable * utilization, gboolean plus); + + extern void process_utilization(resource_t * rsc, node_t ** prefer, pe_working_set_t * data_set); ++pe_action_t *create_pseudo_resource_op(resource_t * rsc, const char *task, bool optional, bool runnable, pe_working_set_t *data_set); + + # define STONITH_UP "stonith_up" + # define STONITH_DONE "stonith_complete" +-- +1.8.3.1 + + +From 356261e246b6aa46d6647eb16f09ff2dceeb1313 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Thu, 15 Jun 2017 10:47:13 +1000 +Subject: [PATCH 2/8] Test: PE: Implement bundle ordering + +--- + pengine/regression.sh | 6 + + pengine/test10/bundle-order-partial-start-2.dot | 51 ++ + pengine/test10/bundle-order-partial-start-2.exp | 299 ++++++++ + pengine/test10/bundle-order-partial-start-2.scores | 197 +++++ + .../test10/bundle-order-partial-start-2.summary | 75 ++ + pengine/test10/bundle-order-partial-start-2.xml | 387 ++++++++++ + pengine/test10/bundle-order-partial-start.dot | 65 ++ + pengine/test10/bundle-order-partial-start.exp | 375 ++++++++++ + pengine/test10/bundle-order-partial-start.scores | 197 +++++ + pengine/test10/bundle-order-partial-start.summary | 82 ++ + pengine/test10/bundle-order-partial-start.xml | 379 ++++++++++ + pengine/test10/bundle-order-partial-stop.dot | 194 +++++ + pengine/test10/bundle-order-partial-stop.exp | 734 ++++++++++++++++++ + pengine/test10/bundle-order-partial-stop.scores | 199 +++++ + pengine/test10/bundle-order-partial-stop.summary | 114 +++ + pengine/test10/bundle-order-partial-stop.xml | 421 +++++++++++ + pengine/test10/bundle-order-partial.dot | 2 + + pengine/test10/bundle-order-partial.exp | 1 + + pengine/test10/bundle-order-partial.scores | 197 +++++ + pengine/test10/bundle-order-partial.summary | 47 ++ + pengine/test10/bundle-order-partial.xml | 421 +++++++++++ + pengine/test10/bundle-order-startup.dot | 139 ++++ + pengine/test10/bundle-order-startup.exp | 825 +++++++++++++++++++++ + pengine/test10/bundle-order-startup.scores | 197 +++++ + pengine/test10/bundle-order-startup.summary | 126 ++++ + pengine/test10/bundle-order-startup.xml | 315 ++++++++ + pengine/test10/bundle-order-stop.dot | 194 +++++ + pengine/test10/bundle-order-stop.exp | 734 ++++++++++++++++++ + pengine/test10/bundle-order-stop.scores | 199 +++++ + pengine/test10/bundle-order-stop.summary | 114 +++ + pengine/test10/bundle-order-stop.xml | 421 +++++++++++ + 31 files changed, 7707 insertions(+) + create mode 100644 pengine/test10/bundle-order-partial-start-2.dot + create mode 100644 pengine/test10/bundle-order-partial-start-2.exp + create mode 100644 pengine/test10/bundle-order-partial-start-2.scores + create mode 100644 pengine/test10/bundle-order-partial-start-2.summary + create mode 100644 pengine/test10/bundle-order-partial-start-2.xml + create mode 100644 pengine/test10/bundle-order-partial-start.dot + create mode 100644 pengine/test10/bundle-order-partial-start.exp + create mode 100644 pengine/test10/bundle-order-partial-start.scores + create mode 100644 pengine/test10/bundle-order-partial-start.summary + create mode 100644 pengine/test10/bundle-order-partial-start.xml + create mode 100644 pengine/test10/bundle-order-partial-stop.dot + create mode 100644 pengine/test10/bundle-order-partial-stop.exp + create mode 100644 pengine/test10/bundle-order-partial-stop.scores + create mode 100644 pengine/test10/bundle-order-partial-stop.summary + create mode 100644 pengine/test10/bundle-order-partial-stop.xml + create mode 100644 pengine/test10/bundle-order-partial.dot + create mode 100644 pengine/test10/bundle-order-partial.exp + create mode 100644 pengine/test10/bundle-order-partial.scores + create mode 100644 pengine/test10/bundle-order-partial.summary + create mode 100644 pengine/test10/bundle-order-partial.xml + create mode 100644 pengine/test10/bundle-order-startup.dot + create mode 100644 pengine/test10/bundle-order-startup.exp + create mode 100644 pengine/test10/bundle-order-startup.scores + create mode 100644 pengine/test10/bundle-order-startup.summary + create mode 100644 pengine/test10/bundle-order-startup.xml + create mode 100644 pengine/test10/bundle-order-stop.dot + create mode 100644 pengine/test10/bundle-order-stop.exp + create mode 100644 pengine/test10/bundle-order-stop.scores + create mode 100644 pengine/test10/bundle-order-stop.summary + create mode 100644 pengine/test10/bundle-order-stop.xml + +diff --git a/pengine/regression.sh b/pengine/regression.sh +index df449e0..e97b54b 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -802,6 +802,12 @@ do_test container-is-remote-node "Place resource within container when container + do_test bug-rh-1097457 "Kill user defined container/contents ordering" + do_test bug-cl-5247 "Graph loop when recovering m/s resource in a container" + ++do_test bundle-order-startup "Bundle startup ordering" ++do_test bundle-order-partial-start "Bundle startup ordering when some dependancies are already running" ++do_test bundle-order-partial-start-2 "Bundle startup ordering when some dependancies and the container are already running" ++do_test bundle-order-stop "Bundle stop ordering" ++do_test bundle-order-partial-stop "Bundle startup ordering when some dependancies are already stopped" ++ + echo "" + do_test whitebox-fail1 "Fail whitebox container rsc." + do_test whitebox-fail2 "Fail whitebox container rsc lrmd connection." +diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot +new file mode 100644 +index 0000000..59a8b15 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start-2.dot +@@ -0,0 +1,51 @@ ++digraph "g" { ++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] ++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold] ++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] ++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] ++"haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle_running_0" -> "galera-bundle_start_0" [ style = bold] ++"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] ++"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_running_0" -> "galera-bundle_start_0" [ style = bold] ++"rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_promoted_0" -> "galera-bundle_start_0" [ style = bold] ++"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold] ++"redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] ++"redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++} +diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp +new file mode 100644 +index 0000000..621332c +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start-2.exp +@@ -0,0 +1,299 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-partial-start-2.scores b/pengine/test10/bundle-order-partial-start-2.scores +new file mode 100644 +index 0000000..f5e86de +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start-2.scores +@@ -0,0 +1,197 @@ ++Allocation scores: ++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:0 allocation score on undercloud: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0 ++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY ++clone_color: redis:0 allocation score on undercloud: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on undercloud: 0 ++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on undercloud: INFINITY ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on undercloud: 0 ++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:0 allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on undercloud: 0 ++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:0 allocation score on undercloud: 0 ++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on undercloud: 0 ++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on undercloud: 0 ++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on undercloud: 0 ++galera:0 promotion score on galera-bundle-0: -1 ++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on undercloud: INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: galera:0 allocation score on galera-bundle-0: INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:0 allocation score on undercloud: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on undercloud: INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on undercloud: INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-0: INFINITY ++native_color: redis:0 allocation score on undercloud: -INFINITY ++redis:0 promotion score on redis-bundle-0: 3000 +diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary +new file mode 100644 +index 0000000..859ca25 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start-2.summary +@@ -0,0 +1,75 @@ ++ ++Current cluster status: ++Online: [ undercloud ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped undercloud ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Stopped undercloud ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Slave undercloud ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud ++ ++Transition Summary: ++ * Start rabbitmq:0 (rabbitmq-bundle-0) ++ * Start galera:0 (galera-bundle-0) ++ * Promote redis:0 (Slave -> Master redis-bundle-0) ++ * Start haproxy-bundle-docker-0 (undercloud) ++ ++Executing cluster transition: ++ * Resource action: haproxy-bundle-docker-0 monitor on undercloud ++ * Pseudo action: haproxy-bundle_start_0 ++ * Pseudo action: redis-bundle_promote_0 ++ * Pseudo action: redis-bundle-master_promote_0 ++ * Pseudo action: rabbitmq-bundle_start_0 ++ * Resource action: redis promote on redis-bundle-0 ++ * Resource action: haproxy-bundle-docker-0 start on undercloud ++ * Pseudo action: haproxy-bundle_running_0 ++ * Pseudo action: redis-bundle-master_promoted_0 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 ++ * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 ++ * Resource action: redis monitor=20000 on redis-bundle-0 ++ * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud ++ * Pseudo action: redis-bundle_promoted_0 ++ * Pseudo action: rabbitmq-bundle-clone_running_0 ++ * Pseudo action: rabbitmq-bundle_running_0 ++ * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 ++ * Pseudo action: galera-bundle_start_0 ++ * Pseudo action: galera-bundle-master_start_0 ++ * Resource action: galera:0 start on galera-bundle-0 ++ * Pseudo action: galera-bundle-master_running_0 ++ * Pseudo action: galera-bundle_running_0 ++ * Resource action: galera:0 monitor=30000 on galera-bundle-0 ++ * Resource action: galera:0 monitor=20000 on galera-bundle-0 ++ ++Revised cluster status: ++Online: [ undercloud ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Slave undercloud ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud ++ +diff --git a/pengine/test10/bundle-order-partial-start-2.xml b/pengine/test10/bundle-order-partial-start-2.xml +new file mode 100644 +index 0000000..9c4d40c +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start-2.xml +@@ -0,0 +1,387 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot +new file mode 100644 +index 0000000..06c3620 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start.dot +@@ -0,0 +1,65 @@ ++digraph "g" { ++"galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_monitor_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] ++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold] ++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] ++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] ++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] ++"haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle_running_0" -> "galera-bundle_start_0" [ style = bold] ++"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] ++"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_running_0" -> "galera-bundle_start_0" [ style = bold] ++"rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_promoted_0" -> "galera-bundle_start_0" [ style = bold] ++"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold] ++"redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] ++"redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++} +diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp +new file mode 100644 +index 0000000..b48bb03 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start.exp +@@ -0,0 +1,375 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-partial-start.scores b/pengine/test10/bundle-order-partial-start.scores +new file mode 100644 +index 0000000..ea702d0 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start.scores +@@ -0,0 +1,197 @@ ++Allocation scores: ++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:0 allocation score on undercloud: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0 ++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY ++clone_color: redis:0 allocation score on undercloud: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on undercloud: 0 ++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on undercloud: 0 ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on undercloud: 0 ++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on undercloud: 0 ++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:0 allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on undercloud: 0 ++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:0 allocation score on undercloud: 0 ++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on undercloud: 0 ++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on undercloud: 0 ++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on undercloud: 0 ++galera:0 promotion score on galera-bundle-0: -1 ++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on undercloud: 10000 ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on undercloud: 0 ++native_color: galera:0 allocation score on galera-bundle-0: INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:0 allocation score on undercloud: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on undercloud: INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on undercloud: INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-0: INFINITY ++native_color: redis:0 allocation score on undercloud: -INFINITY ++redis:0 promotion score on redis-bundle-0: 3000 +diff --git a/pengine/test10/bundle-order-partial-start.summary b/pengine/test10/bundle-order-partial-start.summary +new file mode 100644 +index 0000000..e56e55f +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start.summary +@@ -0,0 +1,82 @@ ++ ++Current cluster status: ++Online: [ undercloud ] ++Containers: [ rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped undercloud ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Stopped ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Slave undercloud ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud ++ ++Transition Summary: ++ * Start rabbitmq:0 (rabbitmq-bundle-0) ++ * Start galera-bundle-docker-0 (undercloud) ++ * Start galera-bundle-0 (undercloud) ++ * Start galera:0 (galera-bundle-0) ++ * Promote redis:0 (Slave -> Master redis-bundle-0) ++ * Start haproxy-bundle-docker-0 (undercloud) ++ ++Executing cluster transition: ++ * Resource action: galera-bundle-docker-0 monitor on undercloud ++ * Resource action: haproxy-bundle-docker-0 monitor on undercloud ++ * Pseudo action: haproxy-bundle_start_0 ++ * Pseudo action: redis-bundle_promote_0 ++ * Pseudo action: redis-bundle-master_promote_0 ++ * Pseudo action: rabbitmq-bundle_start_0 ++ * Resource action: redis promote on redis-bundle-0 ++ * Resource action: haproxy-bundle-docker-0 start on undercloud ++ * Pseudo action: haproxy-bundle_running_0 ++ * Pseudo action: redis-bundle-master_promoted_0 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 ++ * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 ++ * Resource action: redis monitor=20000 on redis-bundle-0 ++ * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud ++ * Pseudo action: redis-bundle_promoted_0 ++ * Pseudo action: rabbitmq-bundle-clone_running_0 ++ * Pseudo action: rabbitmq-bundle_running_0 ++ * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 ++ * Pseudo action: galera-bundle_start_0 ++ * Resource action: galera-bundle-docker-0 start on undercloud ++ * Resource action: galera-bundle-0 start on undercloud ++ * Pseudo action: galera-bundle-master_start_0 ++ * Resource action: galera:0 start on galera-bundle-0 ++ * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud ++ * Resource action: galera-bundle-0 monitor=60000 on undercloud ++ * Pseudo action: galera-bundle-master_running_0 ++ * Pseudo action: galera-bundle_running_0 ++ * Resource action: galera:0 monitor=30000 on galera-bundle-0 ++ * Resource action: galera:0 monitor=20000 on galera-bundle-0 ++ ++Revised cluster status: ++Online: [ undercloud ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Slave undercloud ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud ++ +diff --git a/pengine/test10/bundle-order-partial-start.xml b/pengine/test10/bundle-order-partial-start.xml +new file mode 100644 +index 0000000..907dbfb +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-start.xml +@@ -0,0 +1,379 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-partial-stop.dot b/pengine/test10/bundle-order-partial-stop.dot +new file mode 100644 +index 0000000..235b634 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-stop.dot +@@ -0,0 +1,194 @@ ++digraph "g" { ++"Cancel galera_monitor_10000 galera-bundle-0" -> "galera_demote_0 galera-bundle-0" [ style = bold] ++"Cancel galera_monitor_10000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"Cancel redis_monitor_20000 redis-bundle-0" -> "redis_demote_0 redis-bundle-0" [ style = bold] ++"Cancel redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"do_shutdown undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"galera-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"galera-bundle-0_stop_0 undercloud" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] ++"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_demoted_0" -> "galera-bundle-master_start_0" [ style = dashed] ++"galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold] ++"galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold] ++"galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed] ++"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold] ++"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = dashed] ++"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold] ++"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold] ++"galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold] ++"galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold] ++"galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] ++"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_stopped_0" -> "redis-bundle_stop_0" [ style = bold] ++"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold] ++"galera_demote_0 galera-bundle-0" -> "galera-bundle-master_demoted_0" [ style = bold] ++"galera_demote_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] ++"galera_demote_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] ++"galera_demote_0 galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed] ++"galera_start_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] ++"galera_start_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] ++"galera_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_stop_0 galera-bundle-0" -> "all_stopped" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"haproxy-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"haproxy-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle_stopped_0" [ style = bold] ++"haproxy-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle_stop_0" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.247_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.248_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.249_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.250_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.253_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.254_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"ip-192.168.122.247_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.247_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.247_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.247_stop_0 undercloud" -> "ip-192.168.122.247_start_0 " [ style = dashed] ++"ip-192.168.122.247_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.248_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.248_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.248_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.248_stop_0 undercloud" -> "ip-192.168.122.248_start_0 " [ style = dashed] ++"ip-192.168.122.248_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.249_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.249_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.249_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.249_stop_0 undercloud" -> "ip-192.168.122.249_start_0 " [ style = dashed] ++"ip-192.168.122.249_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.250_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.250_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.250_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.250_stop_0 undercloud" -> "ip-192.168.122.250_start_0 " [ style = dashed] ++"ip-192.168.122.250_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.253_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.253_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.253_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.253_stop_0 undercloud" -> "ip-192.168.122.253_start_0 " [ style = dashed] ++"ip-192.168.122.253_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.254_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.254_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.254_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.254_stop_0 undercloud" -> "ip-192.168.122.254_start_0 " [ style = dashed] ++"ip-192.168.122.254_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "openstack-cinder-volume_stopped_0" [ style = bold] ++"openstack-cinder-volume-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"openstack-cinder-volume_stop_0" -> "openstack-cinder-volume-docker-0_stop_0 undercloud" [ style = bold] ++"openstack-cinder-volume_stop_0" [ style=bold color="green" fontcolor="orange"] ++"openstack-cinder-volume_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-clone_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "all_stopped" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-0_stop_0 undercloud" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"redis-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"redis-bundle-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] ++"redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_start_0" [ style = dashed] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold] ++"redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] ++"redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed] ++"redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_stop_0" -> "redis-bundle-master_stopped_0" [ style = bold] ++"redis-bundle-master_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_stopped_0" -> "redis-bundle-master_start_0" [ style = dashed] ++"redis-bundle-master_stopped_0" -> "redis-bundle_stopped_0" [ style = bold] ++"redis-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] ++"redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold] ++"redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold] ++"redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed] ++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] ++"redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_stopped_0" -> "haproxy-bundle_stop_0" [ style = bold] ++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis_demote_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold] ++"redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold] ++"redis_demote_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis_demote_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis_demote_0 redis-bundle-0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"redis_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_monitor_45000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_monitor_60000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = dashed] ++"redis_start_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis_start_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis_start_0 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_stop_0 redis-bundle-0" -> "all_stopped" [ style = bold] ++"redis_stop_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold] ++"redis_stop_0 redis-bundle-0" -> "redis-bundle-master_stopped_0" [ style = bold] ++"redis_stop_0 redis-bundle-0" -> "redis_start_0 redis-bundle-0" [ style = dashed] ++"redis_stop_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++} +diff --git a/pengine/test10/bundle-order-partial-stop.exp b/pengine/test10/bundle-order-partial-stop.exp +new file mode 100644 +index 0000000..d085433 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-stop.exp +@@ -0,0 +1,734 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-partial-stop.scores b/pengine/test10/bundle-order-partial-stop.scores +new file mode 100644 +index 0000000..a662f42 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-stop.scores +@@ -0,0 +1,199 @@ ++Allocation scores: ++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:0 allocation score on undercloud: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0 ++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY ++clone_color: redis:0 allocation score on undercloud: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on undercloud: 0 ++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on undercloud: INFINITY ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on undercloud: 0 ++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:0 allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on undercloud: 0 ++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++container_color: rabbitmq:0 allocation score on undercloud: 0 ++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on undercloud: 0 ++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on undercloud: 0 ++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on undercloud: 0 ++galera:0 promotion score on galera-bundle-0: 100 ++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on undercloud: INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on undercloud: -INFINITY ++native_color: galera:0 allocation score on galera-bundle-0: INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:0 allocation score on undercloud: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on undercloud: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: -INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on undercloud: INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on undercloud: -INFINITY ++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-0: INFINITY ++native_color: redis:0 allocation score on undercloud: -INFINITY ++redis:0 promotion score on redis-bundle-0: 1 +diff --git a/pengine/test10/bundle-order-partial-stop.summary b/pengine/test10/bundle-order-partial-stop.summary +new file mode 100644 +index 0000000..bd6f937 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-stop.summary +@@ -0,0 +1,114 @@ ++ ++Current cluster status: ++Online: [ undercloud ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Master undercloud ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud ++ ++Transition Summary: ++ * Shutdown undercloud ++ * Stop rabbitmq-bundle-docker-0 (undercloud) ++ * Stop rabbitmq-bundle-0 (undercloud) ++ * Stop rabbitmq:0 (Started rabbitmq-bundle-0) ++ * Stop galera-bundle-docker-0 (undercloud) ++ * Stop galera-bundle-0 (undercloud) ++ * Demote galera:0 (Master -> Slave galera-bundle-0) ++ * Restart galera:0 (Slave galera-bundle-0) ++ * Stop redis-bundle-docker-0 (undercloud) ++ * Stop redis-bundle-0 (undercloud) ++ * Demote redis:0 (Master -> Slave redis-bundle-0) ++ * Restart redis:0 (Slave redis-bundle-0) ++ * Stop ip-192.168.122.254 (undercloud) ++ * Stop ip-192.168.122.250 (undercloud) ++ * Stop ip-192.168.122.249 (undercloud) ++ * Stop ip-192.168.122.253 (undercloud) ++ * Stop ip-192.168.122.247 (undercloud) ++ * Stop ip-192.168.122.248 (undercloud) ++ * Stop haproxy-bundle-docker-0 (undercloud) ++ * Stop openstack-cinder-volume-docker-0 (undercloud) ++ ++Executing cluster transition: ++ * Resource action: galera cancel=10000 on galera-bundle-0 ++ * Resource action: redis cancel=20000 on redis-bundle-0 ++ * Pseudo action: openstack-cinder-volume_stop_0 ++ * Pseudo action: redis-bundle_demote_0 ++ * Pseudo action: redis-bundle-master_demote_0 ++ * Pseudo action: galera-bundle_demote_0 ++ * Pseudo action: galera-bundle-master_demote_0 ++ * Pseudo action: rabbitmq-bundle_stop_0 ++ * Resource action: galera demote on galera-bundle-0 ++ * Resource action: redis demote on redis-bundle-0 ++ * Resource action: openstack-cinder-volume-docker-0 stop on undercloud ++ * Pseudo action: openstack-cinder-volume_stopped_0 ++ * Pseudo action: redis-bundle-master_demoted_0 ++ * Pseudo action: galera-bundle-master_demoted_0 ++ * Pseudo action: rabbitmq-bundle-clone_stop_0 ++ * Resource action: rabbitmq stop on rabbitmq-bundle-0 ++ * Resource action: rabbitmq-bundle-0 stop on undercloud ++ * Pseudo action: redis-bundle_demoted_0 ++ * Pseudo action: galera-bundle_demoted_0 ++ * Pseudo action: galera-bundle_stop_0 ++ * Pseudo action: rabbitmq-bundle-clone_stopped_0 ++ * Pseudo action: rabbitmq-bundle_stopped_0 ++ * Resource action: rabbitmq-bundle-docker-0 stop on undercloud ++ * Pseudo action: galera-bundle-master_stop_0 ++ * Resource action: galera stop on galera-bundle-0 ++ * Resource action: galera-bundle-0 stop on undercloud ++ * Pseudo action: galera-bundle-master_stopped_0 ++ * Pseudo action: galera-bundle_stopped_0 ++ * Resource action: galera-bundle-docker-0 stop on undercloud ++ * Pseudo action: redis-bundle_stop_0 ++ * Pseudo action: redis-bundle-master_stop_0 ++ * Resource action: redis stop on redis-bundle-0 ++ * Resource action: redis-bundle-0 stop on undercloud ++ * Pseudo action: redis-bundle-master_stopped_0 ++ * Pseudo action: redis-bundle_stopped_0 ++ * Pseudo action: redis-bundle_start_0 ++ * Resource action: redis-bundle-docker-0 stop on undercloud ++ * Pseudo action: haproxy-bundle_stop_0 ++ * Resource action: haproxy-bundle-docker-0 stop on undercloud ++ * Pseudo action: haproxy-bundle_stopped_0 ++ * Resource action: ip-192.168.122.254 stop on undercloud ++ * Resource action: ip-192.168.122.250 stop on undercloud ++ * Resource action: ip-192.168.122.249 stop on undercloud ++ * Resource action: ip-192.168.122.253 stop on undercloud ++ * Resource action: ip-192.168.122.247 stop on undercloud ++ * Resource action: ip-192.168.122.248 stop on undercloud ++ * Cluster action: do_shutdown on undercloud ++ * Pseudo action: all_stopped ++ ++Revised cluster status: ++Online: [ undercloud ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Stopped ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Stopped ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Stopped ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Stopped ++ +diff --git a/pengine/test10/bundle-order-partial-stop.xml b/pengine/test10/bundle-order-partial-stop.xml +new file mode 100644 +index 0000000..467082a +--- /dev/null ++++ b/pengine/test10/bundle-order-partial-stop.xml +@@ -0,0 +1,421 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-partial.dot b/pengine/test10/bundle-order-partial.dot +new file mode 100644 +index 0000000..4b30191 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial.dot +@@ -0,0 +1,2 @@ ++digraph "g" { ++} +diff --git a/pengine/test10/bundle-order-partial.exp b/pengine/test10/bundle-order-partial.exp +new file mode 100644 +index 0000000..56e315f +--- /dev/null ++++ b/pengine/test10/bundle-order-partial.exp +@@ -0,0 +1 @@ ++ +diff --git a/pengine/test10/bundle-order-partial.scores b/pengine/test10/bundle-order-partial.scores +new file mode 100644 +index 0000000..be91c96 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial.scores +@@ -0,0 +1,197 @@ ++Allocation scores: ++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:0 allocation score on undercloud: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0 ++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY ++clone_color: redis:0 allocation score on undercloud: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on undercloud: 0 ++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on undercloud: INFINITY ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on undercloud: 0 ++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:0 allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on undercloud: 0 ++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++container_color: rabbitmq:0 allocation score on undercloud: 0 ++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on undercloud: 0 ++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on undercloud: 0 ++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on undercloud: 0 ++galera:0 promotion score on galera-bundle-0: 100 ++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on undercloud: INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: galera:0 allocation score on galera-bundle-0: INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:0 allocation score on undercloud: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on undercloud: INFINITY ++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on undercloud: INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on undercloud: INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY ++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-0: INFINITY ++native_color: redis:0 allocation score on undercloud: -INFINITY ++redis:0 promotion score on redis-bundle-0: 1 +diff --git a/pengine/test10/bundle-order-partial.summary b/pengine/test10/bundle-order-partial.summary +new file mode 100644 +index 0000000..6155230 +--- /dev/null ++++ b/pengine/test10/bundle-order-partial.summary +@@ -0,0 +1,47 @@ ++ ++Current cluster status: ++Online: [ undercloud ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Master undercloud ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud ++ ++Transition Summary: ++ ++Executing cluster transition: ++ ++Revised cluster status: ++Online: [ undercloud ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Master undercloud ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud ++ +diff --git a/pengine/test10/bundle-order-partial.xml b/pengine/test10/bundle-order-partial.xml +new file mode 100644 +index 0000000..eda20bb +--- /dev/null ++++ b/pengine/test10/bundle-order-partial.xml +@@ -0,0 +1,421 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-startup.dot b/pengine/test10/bundle-order-startup.dot +new file mode 100644 +index 0000000..73947a5 +--- /dev/null ++++ b/pengine/test10/bundle-order-startup.dot +@@ -0,0 +1,139 @@ ++digraph "g" { ++"galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_monitor_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] ++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold] ++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] ++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] ++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] ++"haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle_running_0" -> "redis-bundle_start_0" [ style = bold] ++"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] ++"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"ip-192.168.122.247_monitor_0 undercloud" -> "ip-192.168.122.247_start_0 undercloud" [ style = bold] ++"ip-192.168.122.247_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.247_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.247_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold] ++"ip-192.168.122.247_start_0 undercloud" -> "ip-192.168.122.247_monitor_10000 undercloud" [ style = bold] ++"ip-192.168.122.247_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.248_monitor_0 undercloud" -> "ip-192.168.122.248_start_0 undercloud" [ style = bold] ++"ip-192.168.122.248_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.248_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.248_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold] ++"ip-192.168.122.248_start_0 undercloud" -> "ip-192.168.122.248_monitor_10000 undercloud" [ style = bold] ++"ip-192.168.122.248_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.249_monitor_0 undercloud" -> "ip-192.168.122.249_start_0 undercloud" [ style = bold] ++"ip-192.168.122.249_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.249_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.249_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold] ++"ip-192.168.122.249_start_0 undercloud" -> "ip-192.168.122.249_monitor_10000 undercloud" [ style = bold] ++"ip-192.168.122.249_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.250_monitor_0 undercloud" -> "ip-192.168.122.250_start_0 undercloud" [ style = bold] ++"ip-192.168.122.250_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.250_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.250_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold] ++"ip-192.168.122.250_start_0 undercloud" -> "ip-192.168.122.250_monitor_10000 undercloud" [ style = bold] ++"ip-192.168.122.250_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.253_monitor_0 undercloud" -> "ip-192.168.122.253_start_0 undercloud" [ style = bold] ++"ip-192.168.122.253_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.253_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.253_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold] ++"ip-192.168.122.253_start_0 undercloud" -> "ip-192.168.122.253_monitor_10000 undercloud" [ style = bold] ++"ip-192.168.122.253_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.254_monitor_0 undercloud" -> "ip-192.168.122.254_start_0 undercloud" [ style = bold] ++"ip-192.168.122.254_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.254_monitor_10000 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.254_start_0 undercloud" -> "haproxy-bundle_start_0" [ style = bold] ++"ip-192.168.122.254_start_0 undercloud" -> "ip-192.168.122.254_monitor_10000 undercloud" [ style = bold] ++"ip-192.168.122.254_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"openstack-cinder-volume-docker-0_monitor_0 undercloud" -> "openstack-cinder-volume-docker-0_start_0 undercloud" [ style = bold] ++"openstack-cinder-volume-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"openstack-cinder-volume-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"openstack-cinder-volume-docker-0_start_0 undercloud" -> "openstack-cinder-volume-docker-0_monitor_60000 undercloud" [ style = bold] ++"openstack-cinder-volume-docker-0_start_0 undercloud" -> "openstack-cinder-volume_running_0" [ style = bold] ++"openstack-cinder-volume-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"openstack-cinder-volume_running_0" [ style=bold color="green" fontcolor="orange"] ++"openstack-cinder-volume_start_0" -> "openstack-cinder-volume-docker-0_start_0 undercloud" [ style = bold] ++"openstack-cinder-volume_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq-bundle-0_monitor_60000 undercloud" [ style = bold] ++"rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-docker-0_monitor_0 undercloud" -> "rabbitmq-bundle-docker-0_start_0 undercloud" [ style = bold] ++"rabbitmq-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle-0_start_0 undercloud" [ style = bold] ++"rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-0_start_0 undercloud" [ style = bold] ++"rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_start_0 undercloud" -> "redis-bundle-0_monitor_60000 undercloud" [ style = bold] ++"redis-bundle-0_start_0 undercloud" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold] ++"redis-bundle-0_start_0 undercloud" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold] ++"redis-bundle-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold] ++"redis-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_monitor_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold] ++"redis-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-0_start_0 undercloud" [ style = bold] ++"redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"redis-bundle-docker-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold] ++"redis-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold] ++"redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_running_0" -> "galera-bundle_start_0" [ style = bold] ++"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_start_0" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold] ++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold] ++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis:0_monitor_45000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:0_monitor_60000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:0_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = bold] ++"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold] ++"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold] ++"redis:0_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++} +diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp +new file mode 100644 +index 0000000..51d108d +--- /dev/null ++++ b/pengine/test10/bundle-order-startup.exp +@@ -0,0 +1,825 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-startup.scores b/pengine/test10/bundle-order-startup.scores +new file mode 100644 +index 0000000..565b47e +--- /dev/null ++++ b/pengine/test10/bundle-order-startup.scores +@@ -0,0 +1,197 @@ ++Allocation scores: ++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:0 allocation score on undercloud: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0 ++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY ++clone_color: redis:0 allocation score on undercloud: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on undercloud: 0 ++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on undercloud: 0 ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on undercloud: 0 ++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on undercloud: 0 ++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:0 allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on undercloud: 0 ++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: 0 ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: 0 ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:0 allocation score on undercloud: 0 ++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on undercloud: 0 ++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on undercloud: 0 ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on undercloud: 0 ++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on undercloud: 0 ++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on undercloud: 0 ++galera:0 promotion score on galera-bundle-0: -1 ++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on undercloud: 10000 ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on undercloud: 0 ++native_color: galera:0 allocation score on galera-bundle-0: INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:0 allocation score on undercloud: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on undercloud: 0 ++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on undercloud: 0 ++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on undercloud: 0 ++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on undercloud: 0 ++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on undercloud: 0 ++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on undercloud: 0 ++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on undercloud: 0 ++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: 0 ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on undercloud: 10000 ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: 0 ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on undercloud: 10000 ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on undercloud: 0 ++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-0: INFINITY ++native_color: redis:0 allocation score on undercloud: -INFINITY ++redis:0 promotion score on redis-bundle-0: -1 +diff --git a/pengine/test10/bundle-order-startup.summary b/pengine/test10/bundle-order-startup.summary +new file mode 100644 +index 0000000..b5c2091 +--- /dev/null ++++ b/pengine/test10/bundle-order-startup.summary +@@ -0,0 +1,126 @@ ++ ++Current cluster status: ++Online: [ undercloud ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Stopped ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Stopped ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Stopped ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Stopped ++ ++Transition Summary: ++ * Start rabbitmq-bundle-docker-0 (undercloud) ++ * Start rabbitmq-bundle-0 (undercloud) ++ * Start rabbitmq:0 (rabbitmq-bundle-0) ++ * Start galera-bundle-docker-0 (undercloud) ++ * Start galera-bundle-0 (undercloud) ++ * Start galera:0 (galera-bundle-0) ++ * Start redis-bundle-docker-0 (undercloud) ++ * Start redis-bundle-0 (undercloud) ++ * Start redis:0 (redis-bundle-0) ++ * Start ip-192.168.122.254 (undercloud) ++ * Start ip-192.168.122.250 (undercloud) ++ * Start ip-192.168.122.249 (undercloud) ++ * Start ip-192.168.122.253 (undercloud) ++ * Start ip-192.168.122.247 (undercloud) ++ * Start ip-192.168.122.248 (undercloud) ++ * Start haproxy-bundle-docker-0 (undercloud) ++ * Start openstack-cinder-volume-docker-0 (undercloud) ++ ++Executing cluster transition: ++ * Resource action: rabbitmq-bundle-docker-0 monitor on undercloud ++ * Resource action: galera-bundle-docker-0 monitor on undercloud ++ * Resource action: redis-bundle-docker-0 monitor on undercloud ++ * Resource action: ip-192.168.122.254 monitor on undercloud ++ * Resource action: ip-192.168.122.250 monitor on undercloud ++ * Resource action: ip-192.168.122.249 monitor on undercloud ++ * Resource action: ip-192.168.122.253 monitor on undercloud ++ * Resource action: ip-192.168.122.247 monitor on undercloud ++ * Resource action: ip-192.168.122.248 monitor on undercloud ++ * Resource action: haproxy-bundle-docker-0 monitor on undercloud ++ * Resource action: openstack-cinder-volume-docker-0 monitor on undercloud ++ * Pseudo action: openstack-cinder-volume_start_0 ++ * Pseudo action: rabbitmq-bundle_start_0 ++ * Resource action: rabbitmq-bundle-docker-0 start on undercloud ++ * Resource action: rabbitmq-bundle-0 start on undercloud ++ * Resource action: ip-192.168.122.254 start on undercloud ++ * Resource action: ip-192.168.122.250 start on undercloud ++ * Resource action: ip-192.168.122.249 start on undercloud ++ * Resource action: ip-192.168.122.253 start on undercloud ++ * Resource action: ip-192.168.122.247 start on undercloud ++ * Resource action: ip-192.168.122.248 start on undercloud ++ * Resource action: openstack-cinder-volume-docker-0 start on undercloud ++ * Pseudo action: openstack-cinder-volume_running_0 ++ * Pseudo action: haproxy-bundle_start_0 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 ++ * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 ++ * Resource action: rabbitmq-bundle-docker-0 monitor=60000 on undercloud ++ * Resource action: rabbitmq-bundle-0 monitor=60000 on undercloud ++ * Resource action: ip-192.168.122.254 monitor=10000 on undercloud ++ * Resource action: ip-192.168.122.250 monitor=10000 on undercloud ++ * Resource action: ip-192.168.122.249 monitor=10000 on undercloud ++ * Resource action: ip-192.168.122.253 monitor=10000 on undercloud ++ * Resource action: ip-192.168.122.247 monitor=10000 on undercloud ++ * Resource action: ip-192.168.122.248 monitor=10000 on undercloud ++ * Resource action: haproxy-bundle-docker-0 start on undercloud ++ * Resource action: openstack-cinder-volume-docker-0 monitor=60000 on undercloud ++ * Pseudo action: haproxy-bundle_running_0 ++ * Pseudo action: redis-bundle_start_0 ++ * Pseudo action: rabbitmq-bundle-clone_running_0 ++ * Pseudo action: rabbitmq-bundle_running_0 ++ * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 ++ * Resource action: redis-bundle-docker-0 start on undercloud ++ * Resource action: redis-bundle-0 start on undercloud ++ * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud ++ * Pseudo action: redis-bundle-master_start_0 ++ * Resource action: redis:0 start on redis-bundle-0 ++ * Resource action: redis-bundle-docker-0 monitor=60000 on undercloud ++ * Resource action: redis-bundle-0 monitor=60000 on undercloud ++ * Pseudo action: redis-bundle-master_running_0 ++ * Pseudo action: redis-bundle_running_0 ++ * Pseudo action: galera-bundle_start_0 ++ * Resource action: galera-bundle-docker-0 start on undercloud ++ * Resource action: galera-bundle-0 start on undercloud ++ * Resource action: redis:0 monitor=60000 on redis-bundle-0 ++ * Resource action: redis:0 monitor=45000 on redis-bundle-0 ++ * Pseudo action: galera-bundle-master_start_0 ++ * Resource action: galera:0 start on galera-bundle-0 ++ * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud ++ * Resource action: galera-bundle-0 monitor=60000 on undercloud ++ * Pseudo action: galera-bundle-master_running_0 ++ * Pseudo action: galera-bundle_running_0 ++ * Resource action: galera:0 monitor=30000 on galera-bundle-0 ++ * Resource action: galera:0 monitor=20000 on galera-bundle-0 ++ ++Revised cluster status: ++Online: [ undercloud ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Slave undercloud ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Slave undercloud ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud ++ +diff --git a/pengine/test10/bundle-order-startup.xml b/pengine/test10/bundle-order-startup.xml +new file mode 100644 +index 0000000..3e2e8f6 +--- /dev/null ++++ b/pengine/test10/bundle-order-startup.xml +@@ -0,0 +1,315 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-stop.dot b/pengine/test10/bundle-order-stop.dot +new file mode 100644 +index 0000000..235b634 +--- /dev/null ++++ b/pengine/test10/bundle-order-stop.dot +@@ -0,0 +1,194 @@ ++digraph "g" { ++"Cancel galera_monitor_10000 galera-bundle-0" -> "galera_demote_0 galera-bundle-0" [ style = bold] ++"Cancel galera_monitor_10000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"Cancel redis_monitor_20000 redis-bundle-0" -> "redis_demote_0 redis-bundle-0" [ style = bold] ++"Cancel redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"do_shutdown undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"galera-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"galera-bundle-0_stop_0 undercloud" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] ++"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_demoted_0" -> "galera-bundle-master_start_0" [ style = dashed] ++"galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold] ++"galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold] ++"galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed] ++"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold] ++"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = dashed] ++"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold] ++"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold] ++"galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold] ++"galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold] ++"galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] ++"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_stopped_0" -> "redis-bundle_stop_0" [ style = bold] ++"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold] ++"galera_demote_0 galera-bundle-0" -> "galera-bundle-master_demoted_0" [ style = bold] ++"galera_demote_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] ++"galera_demote_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] ++"galera_demote_0 galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed] ++"galera_start_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] ++"galera_start_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] ++"galera_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_stop_0 galera-bundle-0" -> "all_stopped" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"haproxy-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"haproxy-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle_stopped_0" [ style = bold] ++"haproxy-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle_stop_0" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.247_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.248_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.249_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.250_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.253_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.122.254_stop_0 undercloud" [ style = bold] ++"haproxy-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"ip-192.168.122.247_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.247_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.247_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.247_stop_0 undercloud" -> "ip-192.168.122.247_start_0 " [ style = dashed] ++"ip-192.168.122.247_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.248_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.248_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.248_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.248_stop_0 undercloud" -> "ip-192.168.122.248_start_0 " [ style = dashed] ++"ip-192.168.122.248_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.249_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.249_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.249_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.249_stop_0 undercloud" -> "ip-192.168.122.249_start_0 " [ style = dashed] ++"ip-192.168.122.249_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.250_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.250_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.250_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.250_stop_0 undercloud" -> "ip-192.168.122.250_start_0 " [ style = dashed] ++"ip-192.168.122.250_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.253_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.253_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.253_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.253_stop_0 undercloud" -> "ip-192.168.122.253_start_0 " [ style = dashed] ++"ip-192.168.122.253_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.122.254_start_0 " [ style=dashed color="red" fontcolor="black"] ++"ip-192.168.122.254_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"ip-192.168.122.254_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"ip-192.168.122.254_stop_0 undercloud" -> "ip-192.168.122.254_start_0 " [ style = dashed] ++"ip-192.168.122.254_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"openstack-cinder-volume-docker-0_stop_0 undercloud" -> "openstack-cinder-volume_stopped_0" [ style = bold] ++"openstack-cinder-volume-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"openstack-cinder-volume_stop_0" -> "openstack-cinder-volume-docker-0_stop_0 undercloud" [ style = bold] ++"openstack-cinder-volume_stop_0" [ style=bold color="green" fontcolor="orange"] ++"openstack-cinder-volume_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-clone_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "all_stopped" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-0_stop_0 undercloud" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"redis-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"redis-bundle-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] ++"redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_start_0" [ style = dashed] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold] ++"redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] ++"redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed] ++"redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_start_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_stop_0" -> "redis-bundle-master_stopped_0" [ style = bold] ++"redis-bundle-master_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_stopped_0" -> "redis-bundle-master_start_0" [ style = dashed] ++"redis-bundle-master_stopped_0" -> "redis-bundle_stopped_0" [ style = bold] ++"redis-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] ++"redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold] ++"redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold] ++"redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed] ++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] ++"redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_stopped_0" -> "haproxy-bundle_stop_0" [ style = bold] ++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis_demote_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold] ++"redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold] ++"redis_demote_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis_demote_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis_demote_0 redis-bundle-0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"redis_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_monitor_45000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_monitor_60000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = dashed] ++"redis_start_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis_start_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis_start_0 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_stop_0 redis-bundle-0" -> "all_stopped" [ style = bold] ++"redis_stop_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold] ++"redis_stop_0 redis-bundle-0" -> "redis-bundle-master_stopped_0" [ style = bold] ++"redis_stop_0 redis-bundle-0" -> "redis_start_0 redis-bundle-0" [ style = dashed] ++"redis_stop_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++} +diff --git a/pengine/test10/bundle-order-stop.exp b/pengine/test10/bundle-order-stop.exp +new file mode 100644 +index 0000000..d085433 +--- /dev/null ++++ b/pengine/test10/bundle-order-stop.exp +@@ -0,0 +1,734 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-stop.scores b/pengine/test10/bundle-order-stop.scores +new file mode 100644 +index 0000000..a662f42 +--- /dev/null ++++ b/pengine/test10/bundle-order-stop.scores +@@ -0,0 +1,199 @@ ++Allocation scores: ++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera-bundle-master allocation score on undercloud: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:0 allocation score on undercloud: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on undercloud: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0 ++clone_color: redis-bundle-master allocation score on undercloud: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY ++clone_color: redis:0 allocation score on undercloud: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on undercloud: 0 ++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on undercloud: INFINITY ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on undercloud: 0 ++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:0 allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle allocation score on undercloud: 0 ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on undercloud: 0 ++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on undercloud: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++container_color: rabbitmq:0 allocation score on undercloud: 0 ++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on undercloud: 0 ++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on undercloud: INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on undercloud: 0 ++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on undercloud: 0 ++galera:0 promotion score on galera-bundle-0: 100 ++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on undercloud: INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on undercloud: -INFINITY ++native_color: galera:0 allocation score on galera-bundle-0: INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:0 allocation score on undercloud: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.247 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.247 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.248 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.248 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.249 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.249 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.250 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.250 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.253 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.253 allocation score on undercloud: -INFINITY ++native_color: ip-192.168.122.254 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.122.254 allocation score on undercloud: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on undercloud: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on undercloud: INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on undercloud: -INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:0 allocation score on undercloud: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on undercloud: INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on undercloud: -INFINITY ++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-0: INFINITY ++native_color: redis:0 allocation score on undercloud: -INFINITY ++redis:0 promotion score on redis-bundle-0: 1 +diff --git a/pengine/test10/bundle-order-stop.summary b/pengine/test10/bundle-order-stop.summary +new file mode 100644 +index 0000000..bd6f937 +--- /dev/null ++++ b/pengine/test10/bundle-order-stop.summary +@@ -0,0 +1,114 @@ ++ ++Current cluster status: ++Online: [ undercloud ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 redis-bundle-0:redis-bundle-docker-0 ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started undercloud ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Master undercloud ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Master undercloud ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Started undercloud ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Started undercloud ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started undercloud ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started undercloud ++ ++Transition Summary: ++ * Shutdown undercloud ++ * Stop rabbitmq-bundle-docker-0 (undercloud) ++ * Stop rabbitmq-bundle-0 (undercloud) ++ * Stop rabbitmq:0 (Started rabbitmq-bundle-0) ++ * Stop galera-bundle-docker-0 (undercloud) ++ * Stop galera-bundle-0 (undercloud) ++ * Demote galera:0 (Master -> Slave galera-bundle-0) ++ * Restart galera:0 (Slave galera-bundle-0) ++ * Stop redis-bundle-docker-0 (undercloud) ++ * Stop redis-bundle-0 (undercloud) ++ * Demote redis:0 (Master -> Slave redis-bundle-0) ++ * Restart redis:0 (Slave redis-bundle-0) ++ * Stop ip-192.168.122.254 (undercloud) ++ * Stop ip-192.168.122.250 (undercloud) ++ * Stop ip-192.168.122.249 (undercloud) ++ * Stop ip-192.168.122.253 (undercloud) ++ * Stop ip-192.168.122.247 (undercloud) ++ * Stop ip-192.168.122.248 (undercloud) ++ * Stop haproxy-bundle-docker-0 (undercloud) ++ * Stop openstack-cinder-volume-docker-0 (undercloud) ++ ++Executing cluster transition: ++ * Resource action: galera cancel=10000 on galera-bundle-0 ++ * Resource action: redis cancel=20000 on redis-bundle-0 ++ * Pseudo action: openstack-cinder-volume_stop_0 ++ * Pseudo action: redis-bundle_demote_0 ++ * Pseudo action: redis-bundle-master_demote_0 ++ * Pseudo action: galera-bundle_demote_0 ++ * Pseudo action: galera-bundle-master_demote_0 ++ * Pseudo action: rabbitmq-bundle_stop_0 ++ * Resource action: galera demote on galera-bundle-0 ++ * Resource action: redis demote on redis-bundle-0 ++ * Resource action: openstack-cinder-volume-docker-0 stop on undercloud ++ * Pseudo action: openstack-cinder-volume_stopped_0 ++ * Pseudo action: redis-bundle-master_demoted_0 ++ * Pseudo action: galera-bundle-master_demoted_0 ++ * Pseudo action: rabbitmq-bundle-clone_stop_0 ++ * Resource action: rabbitmq stop on rabbitmq-bundle-0 ++ * Resource action: rabbitmq-bundle-0 stop on undercloud ++ * Pseudo action: redis-bundle_demoted_0 ++ * Pseudo action: galera-bundle_demoted_0 ++ * Pseudo action: galera-bundle_stop_0 ++ * Pseudo action: rabbitmq-bundle-clone_stopped_0 ++ * Pseudo action: rabbitmq-bundle_stopped_0 ++ * Resource action: rabbitmq-bundle-docker-0 stop on undercloud ++ * Pseudo action: galera-bundle-master_stop_0 ++ * Resource action: galera stop on galera-bundle-0 ++ * Resource action: galera-bundle-0 stop on undercloud ++ * Pseudo action: galera-bundle-master_stopped_0 ++ * Pseudo action: galera-bundle_stopped_0 ++ * Resource action: galera-bundle-docker-0 stop on undercloud ++ * Pseudo action: redis-bundle_stop_0 ++ * Pseudo action: redis-bundle-master_stop_0 ++ * Resource action: redis stop on redis-bundle-0 ++ * Resource action: redis-bundle-0 stop on undercloud ++ * Pseudo action: redis-bundle-master_stopped_0 ++ * Pseudo action: redis-bundle_stopped_0 ++ * Pseudo action: redis-bundle_start_0 ++ * Resource action: redis-bundle-docker-0 stop on undercloud ++ * Pseudo action: haproxy-bundle_stop_0 ++ * Resource action: haproxy-bundle-docker-0 stop on undercloud ++ * Pseudo action: haproxy-bundle_stopped_0 ++ * Resource action: ip-192.168.122.254 stop on undercloud ++ * Resource action: ip-192.168.122.250 stop on undercloud ++ * Resource action: ip-192.168.122.249 stop on undercloud ++ * Resource action: ip-192.168.122.253 stop on undercloud ++ * Resource action: ip-192.168.122.247 stop on undercloud ++ * Resource action: ip-192.168.122.248 stop on undercloud ++ * Cluster action: do_shutdown on undercloud ++ * Pseudo action: all_stopped ++ ++Revised cluster status: ++Online: [ undercloud ] ++ ++ Docker container: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped ++ Docker container: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Stopped ++ Docker container: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Stopped ++ ip-192.168.122.254 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.250 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.249 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.253 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.247 (ocf::heartbeat:IPaddr2): Stopped ++ ip-192.168.122.248 (ocf::heartbeat:IPaddr2): Stopped ++ Docker container: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Stopped ++ +diff --git a/pengine/test10/bundle-order-stop.xml b/pengine/test10/bundle-order-stop.xml +new file mode 100644 +index 0000000..467082a +--- /dev/null ++++ b/pengine/test10/bundle-order-stop.xml +@@ -0,0 +1,421 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +-- +1.8.3.1 + + +From 00c0cda9226b04a9a95e78b9bff63419bf3b4e47 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Thu, 15 Jun 2017 14:05:45 +1000 +Subject: [PATCH 3/8] PE: Basic inter-bundle ordering when both sides have + children + +--- + pengine/container.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++- + 1 file changed, 62 insertions(+), 1 deletion(-) + +diff --git a/pengine/container.c b/pengine/container.c +index d648e0f..a39aebf 100644 +--- a/pengine/container.c ++++ b/pengine/container.c +@@ -396,15 +396,76 @@ container_action_flags(action_t * action, node_t * node) + return flags; + } + +- + enum pe_graph_flags + container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags, + enum pe_action_flags filter, enum pe_ordering type) + { ++ gboolean current = FALSE; + enum pe_graph_flags changed = pe_graph_none; ++ container_variant_data_t *first_data = NULL; ++ container_variant_data_t *then_data = NULL; ++ + + // At the point we need to force container X to stop because + // resource Y needs to stop, here is where we'd implement that ++ crm_trace("%s -> %s", first->uuid, then->uuid); ++ if(first->rsc == NULL || then->rsc == NULL) { ++ return changed; ++ ++ } else if(first->rsc->variant != then->rsc->variant) { ++ return changed; // For now ++ } ++ ++ /* Fix this - lazy */ ++ if (crm_ends_with(first->uuid, "_stopped_0") ++ || crm_ends_with(first->uuid, "_demoted_0")) { ++ current = TRUE; ++ } ++ ++ get_container_variant_data(first_data, first->rsc); ++ get_container_variant_data(then_data, then->rsc); ++ ++ if(first_data->child == NULL || then_data->child == NULL) { ++ return changed; // For now ++ } ++ ++ for (GListPtr gIter = then_data->tuples; gIter != NULL; gIter = gIter->next) { ++ container_grouping_t *tuple = (container_grouping_t *)gIter->data; ++ ++ resource_t *first_child = find_compatible_child(tuple->docker, first_data->child, RSC_ROLE_UNKNOWN, current); ++ if (first_child == NULL && current) { ++ crm_trace("Ignore"); ++ ++ } else if (first_child == NULL) { ++ crm_debug("No match found for %s (%d / %s / %s)", tuple->child->id, current, first->uuid, then->uuid); ++ ++ /* Me no like this hack - but what else can we do? ++ * ++ * If there is no-one active or about to be active ++ * on the same node as then_child, then they must ++ * not be allowed to start ++ */ ++ if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) { ++ pe_rsc_info(then->rsc, "Inhibiting %s from being active", tuple->child->id); ++ if(assign_node(tuple->child, NULL, TRUE)) { ++ changed |= pe_graph_updated_then; ++ } ++ } ++ ++ } else { ++ enum action_tasks task = get_complex_task(first_child, first->task, TRUE); ++ pe_action_t *first_action = find_first_action(first_child->actions, NULL, task2text(task), node); ++ pe_action_t *then_action = find_first_action(tuple->child->actions, NULL, then->task, node); ++ ++ if (order_actions(first_action, then_action, type)) { ++ crm_debug("Created constraint for %s -> %s", first_action->uuid, then_action->uuid); ++ changed |= (pe_graph_updated_first | pe_graph_updated_then); ++ } ++ changed |= tuple->child->cmds->update_actions(first_action, then_action, node, ++ first_child->cmds->action_flags(first_action, node), ++ filter, type); ++ } ++ } + + return changed; + } +-- +1.8.3.1 + + +From 45fa91e049629a22fd69310e34235b5422b87146 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Fri, 16 Jun 2017 13:33:55 +1000 +Subject: [PATCH 4/8] Fix: PE: Functional inter-bundle ordering when either + side has no child + +Also, force containers to restart if they are found active before one of +their dependancies. +--- + pengine/clone.c | 34 ++--- + pengine/container.c | 85 ++++++++---- + pengine/graph.c | 11 +- + pengine/master.c | 5 +- + pengine/test10/bundle-order-partial-start-2.dot | 21 +++ + pengine/test10/bundle-order-partial-start-2.exp | 149 ++++++++++++++++++--- + .../test10/bundle-order-partial-start-2.summary | 9 ++ + pengine/test10/bundle-order-partial-start.dot | 1 + + pengine/test10/bundle-order-partial-start.exp | 3 + + pengine/test10/bundle-order-partial-stop.dot | 19 ++- + pengine/test10/bundle-order-partial-stop.exp | 47 +++---- + pengine/test10/bundle-order-partial-stop.summary | 1 - + pengine/test10/bundle-order-startup.dot | 2 + + pengine/test10/bundle-order-startup.exp | 6 + + pengine/test10/bundle-order-stop.dot | 19 ++- + pengine/test10/bundle-order-stop.exp | 47 +++---- + pengine/test10/bundle-order-stop.summary | 1 - + pengine/utils.h | 4 +- + 18 files changed, 331 insertions(+), 133 deletions(-) + +diff --git a/pengine/clone.c b/pengine/clone.c +index a51677a..a79271e 100644 +--- a/pengine/clone.c ++++ b/pengine/clone.c +@@ -946,7 +946,7 @@ assign_node(resource_t * rsc, node_t * node, gboolean force) + + static resource_t * + find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc, +- enum rsc_role_e filter, gboolean current) ++ GListPtr children, enum rsc_role_e filter, gboolean current) + { + GListPtr gIter = NULL; + +@@ -958,8 +958,7 @@ find_compatible_child_by_node(resource_t * local_child, node_t * local_node, res + crm_trace("Looking for compatible child from %s for %s on %s", + local_child->id, rsc->id, local_node->details->uname); + +- gIter = rsc->children; +- for (; gIter != NULL; gIter = gIter->next) { ++ for (gIter = children; gIter != NULL; gIter = gIter->next) { + resource_t *child_rsc = (resource_t *) gIter->data; + + if(is_child_compatible(child_rsc, local_node, filter, current)) { +@@ -1003,8 +1002,7 @@ is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e + } + + resource_t * +-find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_e filter, +- gboolean current) ++find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr children, enum rsc_role_e filter, gboolean current) + { + resource_t *pair = NULL; + GListPtr gIter = NULL; +@@ -1013,7 +1011,7 @@ find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_ + + local_node = local_child->fns->location(local_child, NULL, current); + if (local_node) { +- return find_compatible_child_by_node(local_child, local_node, rsc, filter, current); ++ return find_compatible_child_by_node(local_child, local_node, rsc, children, filter, current); + } + + scratch = g_hash_table_get_values(local_child->allowed_nodes); +@@ -1023,7 +1021,7 @@ find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_ + for (; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + +- pair = find_compatible_child_by_node(local_child, node, rsc, filter, current); ++ pair = find_compatible_child_by_node(local_child, node, rsc, children, filter, current); + if (pair) { + goto done; + } +@@ -1084,7 +1082,7 @@ clone_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation + } else if (do_interleave) { + resource_t *rh_child = NULL; + +- rh_child = find_compatible_child(rsc_lh, rsc_rh, RSC_ROLE_UNKNOWN, FALSE); ++ rh_child = find_compatible_child(rsc_lh, rsc_rh, rsc_rh->children, RSC_ROLE_UNKNOWN, FALSE); + + if (rh_child) { + pe_rsc_debug(rsc_rh, "Pairing %s with %s", rsc_lh->id, rh_child->id); +@@ -1167,7 +1165,7 @@ clone_child_action(action_t * action) + } + + enum pe_action_flags +-clone_action_flags(action_t * action, node_t * node) ++summary_action_flags(action_t * action, GListPtr children, node_t * node) + { + GListPtr gIter = NULL; + gboolean any_runnable = FALSE; +@@ -1176,15 +1174,13 @@ clone_action_flags(action_t * action, node_t * node) + enum pe_action_flags flags = (pe_action_optional | pe_action_runnable | pe_action_pseudo); + const char *task_s = task2text(task); + +- gIter = action->rsc->children; +- for (; gIter != NULL; gIter = gIter->next) { ++ for (gIter = children; gIter != NULL; gIter = gIter->next) { + action_t *child_action = NULL; + resource_t *child = (resource_t *) gIter->data; + +- child_action = +- find_first_action(child->actions, NULL, task_s, child->children ? NULL : node); +- pe_rsc_trace(action->rsc, "Checking for %s in %s on %s", task_s, child->id, +- node ? node->details->uname : "none"); ++ child_action = find_first_action(child->actions, NULL, task_s, child->children ? NULL : node); ++ pe_rsc_trace(action->rsc, "Checking for %s in %s on %s (%s)", task_s, child->id, ++ node ? node->details->uname : "none", child_action?child_action->uuid:"NA"); + if (child_action) { + enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node); + +@@ -1223,6 +1219,12 @@ clone_action_flags(action_t * action, node_t * node) + return flags; + } + ++enum pe_action_flags ++clone_action_flags(action_t * action, node_t * node) ++{ ++ return summary_action_flags(action, action->rsc->children, node); ++} ++ + static enum pe_graph_flags + clone_update_actions_interleave(action_t * first, action_t * then, node_t * node, + enum pe_action_flags flags, enum pe_action_flags filter, +@@ -1246,7 +1248,7 @@ clone_update_actions_interleave(action_t * first, action_t * then, node_t * node + resource_t *then_child = (resource_t *) gIter->data; + + CRM_ASSERT(then_child != NULL); +- first_child = find_compatible_child(then_child, first->rsc, RSC_ROLE_UNKNOWN, current); ++ first_child = find_compatible_child(then_child, first->rsc, first->rsc->children, RSC_ROLE_UNKNOWN, current); + if (first_child == NULL && current) { + crm_trace("Ignore"); + +diff --git a/pengine/container.c b/pengine/container.c +index a39aebf..f3ea797 100644 +--- a/pengine/container.c ++++ b/pengine/container.c +@@ -42,6 +42,19 @@ gint sort_clone_instance(gconstpointer a, gconstpointer b, gpointer data_set); + void distribute_children(resource_t *rsc, GListPtr children, GListPtr nodes, + int max, int per_host_max, pe_working_set_t * data_set); + ++static GListPtr get_container_list(resource_t *rsc) ++{ ++ GListPtr containers = NULL; ++ container_variant_data_t *data = NULL; ++ ++ get_container_variant_data(data, rsc); ++ for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) { ++ container_grouping_t *tuple = (container_grouping_t *)gIter->data; ++ containers = g_list_append(containers, tuple->docker); ++ } ++ return containers; ++} ++ + node_t * + container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + { +@@ -54,11 +67,7 @@ container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + get_container_variant_data(container_data, rsc); + + set_bit(rsc->flags, pe_rsc_allocating); +- +- for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { +- container_grouping_t *tuple = (container_grouping_t *)gIter->data; +- containers = g_list_append(containers, tuple->docker); +- } ++ containers = get_container_list(rsc); + + dump_node_scores(show_scores ? 0 : scores_log_level, rsc, __FUNCTION__, rsc->allowed_nodes); + +@@ -128,6 +137,7 @@ container_create_actions(resource_t * rsc, pe_working_set_t * data_set) + + CRM_CHECK(rsc != NULL, return); + ++ containers = get_container_list(rsc); + get_container_variant_data(container_data, rsc); + for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { + container_grouping_t *tuple = (container_grouping_t *)gIter->data; +@@ -138,7 +148,6 @@ container_create_actions(resource_t * rsc, pe_working_set_t * data_set) + } + if(tuple->docker) { + tuple->docker->cmds->create_actions(tuple->docker, data_set); +- containers = g_list_append(containers, tuple->docker); + } + if(tuple->remote) { + tuple->remote->cmds->create_actions(tuple->remote, data_set); +@@ -392,7 +401,18 @@ container_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc, rsc_colocatio + enum pe_action_flags + container_action_flags(action_t * action, node_t * node) + { +- enum pe_action_flags flags = (pe_action_optional | pe_action_runnable | pe_action_pseudo); ++ enum pe_action_flags flags = 0; ++ container_variant_data_t *data = NULL; ++ ++ get_container_variant_data(data, action->rsc); ++ if(data->child) { ++ flags = summary_action_flags(action, data->child->children, node); ++ ++ } else { ++ GListPtr containers = get_container_list(action->rsc); ++ flags = summary_action_flags(action, containers, node); ++ g_list_free(containers); ++ } + return flags; + } + +@@ -402,9 +422,8 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum + { + gboolean current = FALSE; + enum pe_graph_flags changed = pe_graph_none; +- container_variant_data_t *first_data = NULL; + container_variant_data_t *then_data = NULL; +- ++ GListPtr containers = NULL; + + // At the point we need to force container X to stop because + // resource Y needs to stop, here is where we'd implement that +@@ -422,22 +441,21 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum + current = TRUE; + } + +- get_container_variant_data(first_data, first->rsc); + get_container_variant_data(then_data, then->rsc); +- +- if(first_data->child == NULL || then_data->child == NULL) { +- return changed; // For now +- } ++ containers = get_container_list(first->rsc); + + for (GListPtr gIter = then_data->tuples; gIter != NULL; gIter = gIter->next) { + container_grouping_t *tuple = (container_grouping_t *)gIter->data; + +- resource_t *first_child = find_compatible_child(tuple->docker, first_data->child, RSC_ROLE_UNKNOWN, current); ++ /* We can't do the then_data->child->children trick here, ++ * since the node's wont match ++ */ ++ resource_t *first_child = find_compatible_child(tuple->docker, first->rsc, containers, RSC_ROLE_UNKNOWN, current); + if (first_child == NULL && current) { + crm_trace("Ignore"); + + } else if (first_child == NULL) { +- crm_debug("No match found for %s (%d / %s / %s)", tuple->child->id, current, first->uuid, then->uuid); ++ crm_debug("No match found for %s (%d / %s / %s)", tuple->docker->id, current, first->uuid, then->uuid); + + /* Me no like this hack - but what else can we do? + * +@@ -446,27 +464,45 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum + * not be allowed to start + */ + if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) { +- pe_rsc_info(then->rsc, "Inhibiting %s from being active", tuple->child->id); +- if(assign_node(tuple->child, NULL, TRUE)) { ++ pe_rsc_info(then->rsc, "Inhibiting %s from being active", tuple->docker->id); ++ if(assign_node(tuple->docker, NULL, TRUE)) { + changed |= pe_graph_updated_then; + } + } + + } else { + enum action_tasks task = get_complex_task(first_child, first->task, TRUE); ++ ++ /* Potentially we might want to invovle first_data->child ++ * if present, however we mostly just need the "you need ++ * to stop" signal to flow back up the ordering chain via ++ * the docker resources which are always present ++ * ++ * Almost certain to break if first->task or then->task is ++ * promote or demote ++ */ + pe_action_t *first_action = find_first_action(first_child->actions, NULL, task2text(task), node); +- pe_action_t *then_action = find_first_action(tuple->child->actions, NULL, then->task, node); ++ pe_action_t *then_action = find_first_action(tuple->docker->actions, NULL, then->task, node); + + if (order_actions(first_action, then_action, type)) { +- crm_debug("Created constraint for %s -> %s", first_action->uuid, then_action->uuid); ++ crm_debug("Created constraint for %s (%d) -> %s (%d) %.6x", ++ first_action->uuid, is_set(first_action->flags, pe_action_optional), ++ then_action->uuid, is_set(then_action->flags, pe_action_optional), type); + changed |= (pe_graph_updated_first | pe_graph_updated_then); + } +- changed |= tuple->child->cmds->update_actions(first_action, then_action, node, +- first_child->cmds->action_flags(first_action, node), +- filter, type); ++ if(first_action && then_action) { ++ changed |= tuple->docker->cmds->update_actions(first_action, then_action, node, ++ first_child->cmds->action_flags(first_action, node), ++ filter, type); ++ } else { ++ crm_err("Nothing found either for %s (%p) or %s (%p) %s", ++ first_child->id, first_action, ++ tuple->docker->id, then_action, task2text(task)); ++ } + } + } + ++ g_list_free(containers); + return changed; + } + +@@ -492,11 +528,8 @@ container_rsc_location(resource_t * rsc, rsc_to_node_t * constraint) + } + + if(container_data->child && (constraint->role_filter == RSC_ROLE_SLAVE || constraint->role_filter == RSC_ROLE_MASTER)) { +- // Translate the node into container names running on that node +- crm_err("Applying constraint %s", constraint->id); + container_data->child->cmds->rsc_location(container_data->child, constraint); + container_data->child->rsc_location = g_list_prepend(container_data->child->rsc_location, constraint); +- crm_err("Added %d location constraints to %s", g_list_length(container_data->child->rsc_location), container_data->child->id); + } + } + +diff --git a/pengine/graph.c b/pengine/graph.c +index c93745b..b774c71 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -231,7 +231,9 @@ graph_update_action(action_t * first, action_t * then, node_t * node, + pe_action_optional, pe_order_implies_first); + + } else if (is_set(first_flags, pe_action_optional) == FALSE) { +- pe_rsc_trace(first->rsc, "first unrunnable: %s then %s", first->uuid, then->uuid); ++ pe_rsc_trace(first->rsc, "first unrunnable: %s (%d) then %s (%d)", ++ first->uuid, is_set(first_flags, pe_action_optional), ++ then->uuid, is_set(then_flags, pe_action_optional)); + if (update_action_flags(first, pe_action_runnable | pe_action_clear, __FUNCTION__, __LINE__)) { + changed |= pe_graph_updated_first; + } +@@ -240,7 +242,9 @@ graph_update_action(action_t * first, action_t * then, node_t * node, + if (changed) { + pe_rsc_trace(then->rsc, "implies left: %s then %s: changed", first->uuid, then->uuid); + } else { +- crm_trace("implies left: %s then %s", first->uuid, then->uuid); ++ crm_trace("implies left: %s (%d) then %s (%d)", ++ first->uuid, is_set(first_flags, pe_action_optional), ++ then->uuid, is_set(then_flags, pe_action_optional)); + } + } + +@@ -607,7 +611,8 @@ update_action(action_t * then) + } + + if (changed & pe_graph_disable) { +- crm_trace("Disabled constraint %s -> %s", other->action->uuid, then->uuid); ++ crm_trace("Disabled constraint %s -> %s in favor of %s -> %s", ++ other->action->uuid, then->uuid, first->uuid, then->uuid); + clear_bit(changed, pe_graph_disable); + other->type = pe_order_none; + } +diff --git a/pengine/master.c b/pengine/master.c +index 93e5186..c15e740 100644 +--- a/pengine/master.c ++++ b/pengine/master.c +@@ -153,8 +153,6 @@ static void apply_master_location(resource_t *child, GListPtr location_constrain + int new_priority = merge_weights(child->priority, cons_node->weight); + pe_rsc_trace(child, "\t%s[%s]: %d -> %d (%d)", child->id, cons_node->details->uname, + child->priority, new_priority, cons_node->weight); +- crm_err("\t%s[%s]: %d -> %d (%d)", child->id, cons_node->details->uname, +- child->priority, new_priority, cons_node->weight); + child->priority = new_priority; + } + } +@@ -720,7 +718,6 @@ master_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + } + + apply_master_location(child_rsc, child_rsc->rsc_location, chosen); +- crm_err("Applying %d location constraints for %s", g_list_length(rsc->rsc_location), rsc->id); + apply_master_location(child_rsc, rsc->rsc_location, chosen); + + for (gIter2 = child_rsc->rsc_cons; gIter2 != NULL; gIter2 = gIter2->next) { +@@ -1025,7 +1022,7 @@ master_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocatio + g_list_free(rhs); + + } else if (constraint->role_lh == RSC_ROLE_MASTER) { +- resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE); ++ resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, rsc_rh->children, constraint->role_rh, FALSE); + + if (rh_child == NULL && constraint->score >= INFINITY) { + pe_rsc_trace(rsc_lh, "%s can't be promoted %s", rsc_lh->id, constraint->id); +diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot +index 59a8b15..20afbe6 100644 +--- a/pengine/test10/bundle-order-partial-start-2.dot ++++ b/pengine/test10/bundle-order-partial-start-2.dot +@@ -1,10 +1,30 @@ + digraph "g" { ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"galera-bundle-0_stop_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] ++"galera-bundle-0_stop_0 undercloud" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] + "galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] + "galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold] + "galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold] + "galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] + "galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] + "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] + "galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] +@@ -16,6 +36,7 @@ digraph "g" { + "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp +index 621332c..08f5084 100644 +--- a/pengine/test10/bundle-order-partial-start-2.exp ++++ b/pengine/test10/bundle-order-partial-start-2.exp +@@ -34,6 +34,9 @@ + + + ++ ++ ++ + + + +@@ -47,6 +50,9 @@ + + + ++ ++ ++ + + + +@@ -60,12 +66,104 @@ + + + ++ ++ ++ ++ ++ ++ + + + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -77,7 +175,7 @@ + + + +- ++ + + + +@@ -90,7 +188,7 @@ + + + +- ++ + + + +@@ -103,7 +201,7 @@ + + + +- ++ + + + +@@ -119,7 +217,7 @@ + + + +- ++ + + + +@@ -128,7 +226,7 @@ + + + +- ++ + + + +@@ -140,7 +238,7 @@ + + + +- ++ + + + +@@ -148,7 +246,7 @@ + + + +- ++ + + + +@@ -160,7 +258,7 @@ + + + +- ++ + + + +@@ -168,7 +266,7 @@ + + + +- ++ + + + +@@ -180,7 +278,7 @@ + + + +- ++ + + + +@@ -192,7 +290,7 @@ + + + +- ++ + + + +@@ -207,7 +305,7 @@ + + + +- ++ + + + +@@ -219,7 +317,7 @@ + + + +- ++ + + + +@@ -231,7 +329,7 @@ + + + +- ++ + + + +@@ -249,7 +347,7 @@ + + + +- ++ + + + +@@ -264,7 +362,7 @@ + + + +- ++ + + + +@@ -276,7 +374,7 @@ + + + +- ++ + + + +@@ -288,7 +386,7 @@ + + + +- ++ + + + +@@ -296,4 +394,19 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary +index 859ca25..5e3927c 100644 +--- a/pengine/test10/bundle-order-partial-start-2.summary ++++ b/pengine/test10/bundle-order-partial-start-2.summary +@@ -22,21 +22,26 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq- + + Transition Summary: + * Start rabbitmq:0 (rabbitmq-bundle-0) ++ * Restart galera-bundle-docker-0 (Started undercloud) ++ * Restart galera-bundle-0 (Started undercloud) + * Start galera:0 (galera-bundle-0) + * Promote redis:0 (Slave -> Master redis-bundle-0) + * Start haproxy-bundle-docker-0 (undercloud) + + Executing cluster transition: ++ * Resource action: galera-bundle-0 stop on undercloud + * Resource action: haproxy-bundle-docker-0 monitor on undercloud + * Pseudo action: haproxy-bundle_start_0 + * Pseudo action: redis-bundle_promote_0 + * Pseudo action: redis-bundle-master_promote_0 + * Pseudo action: rabbitmq-bundle_start_0 ++ * Resource action: galera-bundle-docker-0 stop on undercloud + * Resource action: redis promote on redis-bundle-0 + * Resource action: haproxy-bundle-docker-0 start on undercloud + * Pseudo action: haproxy-bundle_running_0 + * Pseudo action: redis-bundle-master_promoted_0 + * Pseudo action: rabbitmq-bundle-clone_start_0 ++ * Pseudo action: all_stopped + * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 + * Resource action: redis monitor=20000 on redis-bundle-0 + * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud +@@ -45,6 +50,10 @@ Executing cluster transition: + * Pseudo action: rabbitmq-bundle_running_0 + * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 + * Pseudo action: galera-bundle_start_0 ++ * Resource action: galera-bundle-docker-0 start on undercloud ++ * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud ++ * Resource action: galera-bundle-0 start on undercloud ++ * Resource action: galera-bundle-0 monitor=60000 on undercloud + * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera:0 start on galera-bundle-0 + * Pseudo action: galera-bundle-master_running_0 +diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot +index 06c3620..c36ff04 100644 +--- a/pengine/test10/bundle-order-partial-start.dot ++++ b/pengine/test10/bundle-order-partial-start.dot +@@ -30,6 +30,7 @@ digraph "g" { + "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp +index b48bb03..4a5c01c 100644 +--- a/pengine/test10/bundle-order-partial-start.exp ++++ b/pengine/test10/bundle-order-partial-start.exp +@@ -103,6 +103,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-partial-stop.dot b/pengine/test10/bundle-order-partial-stop.dot +index 235b634..af2493c 100644 +--- a/pengine/test10/bundle-order-partial-stop.dot ++++ b/pengine/test10/bundle-order-partial-stop.dot +@@ -11,6 +11,7 @@ digraph "g" { + "galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold] + "galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] + "galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] +@@ -19,6 +20,7 @@ digraph "g" { + "galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold] + "galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold] + "galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = dashed] + "galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] + "galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed] + "galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed] +@@ -32,11 +34,16 @@ digraph "g" { + "galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold] + "galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold] + "galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demoted_0" -> "galera-bundle_start_0" [ style = dashed] + "galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold] + "galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed] ++"galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] + "galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] + "galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] + "galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = dashed] + "galera-bundle_stopped_0" -> "redis-bundle_stop_0" [ style = bold] + "galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] + "galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold] +@@ -110,6 +117,7 @@ digraph "g" { + "rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] + "rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold] + "rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed] + "rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] +@@ -123,6 +131,7 @@ digraph "g" { + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] + "rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] +@@ -142,6 +151,7 @@ digraph "g" { + "redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"redis-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] + "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] +@@ -150,6 +160,7 @@ digraph "g" { + "redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold] + "redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] + "redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = dashed] + "redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed] + "redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed] +@@ -163,16 +174,18 @@ digraph "g" { + "redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] + "redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold] + "redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] +-"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = dashed] + "redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold] + "redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_running_0" -> "galera-bundle_start_0" [ style = dashed] ++"redis-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed] +-"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] + "redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] + "redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle_stopped_0" -> "haproxy-bundle_stop_0" [ style = bold] +-"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = dashed] + "redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] + "redis_demote_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold] + "redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold] +diff --git a/pengine/test10/bundle-order-partial-stop.exp b/pengine/test10/bundle-order-partial-stop.exp +index d085433..02a7372 100644 +--- a/pengine/test10/bundle-order-partial-stop.exp ++++ b/pengine/test10/bundle-order-partial-stop.exp +@@ -173,6 +173,9 @@ + + + ++ ++ ++ + + + +@@ -283,6 +286,9 @@ + + + ++ ++ ++ + + + +@@ -505,22 +511,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -535,7 +526,7 @@ + + + +- ++ + + + +@@ -543,7 +534,7 @@ + + + +- ++ + + + +@@ -558,7 +549,7 @@ + + + +- ++ + + + +@@ -570,7 +561,7 @@ + + + +- ++ + + + +@@ -585,7 +576,7 @@ + + + +- ++ + + + +@@ -600,7 +591,7 @@ + + + +- ++ + + + +@@ -612,7 +603,7 @@ + + + +- ++ + + + +@@ -624,7 +615,7 @@ + + + +- ++ + + + +@@ -639,7 +630,7 @@ + + + +- ++ + + + +@@ -651,7 +642,7 @@ + + + +- ++ + + + +@@ -663,7 +654,7 @@ + + + +- ++ + + + +@@ -671,7 +662,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-stop.summary b/pengine/test10/bundle-order-partial-stop.summary +index bd6f937..c341f4c 100644 +--- a/pengine/test10/bundle-order-partial-stop.summary ++++ b/pengine/test10/bundle-order-partial-stop.summary +@@ -78,7 +78,6 @@ Executing cluster transition: + * Resource action: redis-bundle-0 stop on undercloud + * Pseudo action: redis-bundle-master_stopped_0 + * Pseudo action: redis-bundle_stopped_0 +- * Pseudo action: redis-bundle_start_0 + * Resource action: redis-bundle-docker-0 stop on undercloud + * Pseudo action: haproxy-bundle_stop_0 + * Resource action: haproxy-bundle-docker-0 stop on undercloud +diff --git a/pengine/test10/bundle-order-startup.dot b/pengine/test10/bundle-order-startup.dot +index 73947a5..4e60d79 100644 +--- a/pengine/test10/bundle-order-startup.dot ++++ b/pengine/test10/bundle-order-startup.dot +@@ -32,6 +32,7 @@ digraph "g" { + "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] ++"haproxy-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "haproxy-bundle_running_0" -> "redis-bundle_start_0" [ style = bold] + "haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] +@@ -116,6 +117,7 @@ digraph "g" { + "redis-bundle-docker-0_monitor_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold] + "redis-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] + "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-0_start_0 undercloud" [ style = bold] + "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_monitor_60000 undercloud" [ style = bold] + "redis-bundle-docker-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp +index 51d108d..66ecb08 100644 +--- a/pengine/test10/bundle-order-startup.exp ++++ b/pengine/test10/bundle-order-startup.exp +@@ -176,6 +176,9 @@ + + + ++ ++ ++ + + + +@@ -291,6 +294,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-stop.dot b/pengine/test10/bundle-order-stop.dot +index 235b634..af2493c 100644 +--- a/pengine/test10/bundle-order-stop.dot ++++ b/pengine/test10/bundle-order-stop.dot +@@ -11,6 +11,7 @@ digraph "g" { + "galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold] + "galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] + "galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] +@@ -19,6 +20,7 @@ digraph "g" { + "galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold] + "galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold] + "galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = dashed] + "galera-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] + "galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = dashed] + "galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed] +@@ -32,11 +34,16 @@ digraph "g" { + "galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold] + "galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold] + "galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demoted_0" -> "galera-bundle_start_0" [ style = dashed] + "galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold] + "galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed] ++"galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] + "galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] + "galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] + "galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = dashed] + "galera-bundle_stopped_0" -> "redis-bundle_stop_0" [ style = bold] + "galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] + "galera_demote_0 galera-bundle-0" -> "galera-bundle-0_stop_0 undercloud" [ style = bold] +@@ -110,6 +117,7 @@ digraph "g" { + "rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] + "rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold] + "rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed] + "rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] +@@ -123,6 +131,7 @@ digraph "g" { + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] + "rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] +@@ -142,6 +151,7 @@ digraph "g" { + "redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"redis-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] + "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] +@@ -150,6 +160,7 @@ digraph "g" { + "redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold] + "redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] + "redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = dashed] + "redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed] + "redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed] +@@ -163,16 +174,18 @@ digraph "g" { + "redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] + "redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold] + "redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] +-"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = dashed] + "redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold] + "redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_running_0" -> "galera-bundle_start_0" [ style = dashed] ++"redis-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed] +-"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] + "redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] + "redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle_stopped_0" -> "haproxy-bundle_stop_0" [ style = bold] +-"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = dashed] + "redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] + "redis_demote_0 redis-bundle-0" -> "redis-bundle-0_stop_0 undercloud" [ style = bold] + "redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold] +diff --git a/pengine/test10/bundle-order-stop.exp b/pengine/test10/bundle-order-stop.exp +index d085433..02a7372 100644 +--- a/pengine/test10/bundle-order-stop.exp ++++ b/pengine/test10/bundle-order-stop.exp +@@ -173,6 +173,9 @@ + + + ++ ++ ++ + + + +@@ -283,6 +286,9 @@ + + + ++ ++ ++ + + + +@@ -505,22 +511,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -535,7 +526,7 @@ + + + +- ++ + + + +@@ -543,7 +534,7 @@ + + + +- ++ + + + +@@ -558,7 +549,7 @@ + + + +- ++ + + + +@@ -570,7 +561,7 @@ + + + +- ++ + + + +@@ -585,7 +576,7 @@ + + + +- ++ + + + +@@ -600,7 +591,7 @@ + + + +- ++ + + + +@@ -612,7 +603,7 @@ + + + +- ++ + + + +@@ -624,7 +615,7 @@ + + + +- ++ + + + +@@ -639,7 +630,7 @@ + + + +- ++ + + + +@@ -651,7 +642,7 @@ + + + +- ++ + + + +@@ -663,7 +654,7 @@ + + + +- ++ + + + +@@ -671,7 +662,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-stop.summary b/pengine/test10/bundle-order-stop.summary +index bd6f937..c341f4c 100644 +--- a/pengine/test10/bundle-order-stop.summary ++++ b/pengine/test10/bundle-order-stop.summary +@@ -78,7 +78,6 @@ Executing cluster transition: + * Resource action: redis-bundle-0 stop on undercloud + * Pseudo action: redis-bundle-master_stopped_0 + * Pseudo action: redis-bundle_stopped_0 +- * Pseudo action: redis-bundle_start_0 + * Resource action: redis-bundle-docker-0 stop on undercloud + * Pseudo action: haproxy-bundle_stop_0 + * Resource action: haproxy-bundle-docker-0 stop on undercloud +diff --git a/pengine/utils.h b/pengine/utils.h +index 10e7201..1421166 100644 +--- a/pengine/utils.h ++++ b/pengine/utils.h +@@ -50,11 +50,11 @@ extern void log_action(unsigned int log_level, const char *pre_text, + action_t * action, gboolean details); + + extern gboolean can_run_any(GHashTable * nodes); +-extern resource_t *find_compatible_child(resource_t * local_child, resource_t * rsc, ++extern resource_t *find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr children, + enum rsc_role_e filter, gboolean current); + gboolean is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e filter, gboolean current); + bool assign_node(resource_t * rsc, node_t * node, gboolean force); +- ++enum pe_action_flags summary_action_flags(action_t * action, GListPtr children, node_t * node); + + enum filter_colocation_res { + influence_nothing = 0, +-- +1.8.3.1 + + +From e083fa923274c908a8ef4b124a5d7e53caabd7f5 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Fri, 16 Jun 2017 14:47:37 +1000 +Subject: [PATCH 5/8] Fix: PE: Default to non-interleaved bundle ordering for + safety + +--- + pengine/container.c | 89 +++++++++++++++++++----- + pengine/test10/bundle-order-partial-start-2.dot | 2 +- + pengine/test10/bundle-order-partial-start-2.exp | 6 +- + pengine/test10/bundle-order-partial-start.dot | 2 +- + pengine/test10/bundle-order-partial-start.exp | 6 +- + pengine/test10/bundle-order-partial-stop.dot | 8 ++- + pengine/test10/bundle-order-partial-stop.exp | 22 ++++-- + pengine/test10/bundle-order-partial-stop.summary | 6 +- + pengine/test10/bundle-order-startup.dot | 5 +- + pengine/test10/bundle-order-startup.exp | 15 ++-- + pengine/test10/bundle-order-stop.dot | 8 ++- + pengine/test10/bundle-order-stop.exp | 22 ++++-- + pengine/test10/bundle-order-stop.summary | 6 +- + 13 files changed, 143 insertions(+), 54 deletions(-) + +diff --git a/pengine/container.c b/pengine/container.c +index f3ea797..adee92a 100644 +--- a/pengine/container.c ++++ b/pengine/container.c +@@ -209,13 +209,10 @@ container_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + + if(tuple->child) { + order_stop_stop(rsc, tuple->child, pe_order_implies_first_printed); +- +- } else { +- order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed); +- new_rsc_order(tuple->docker, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set); +- new_rsc_order(tuple->docker, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, +- data_set); + } ++ order_stop_stop(rsc, tuple->docker, pe_order_implies_first_printed); ++ new_rsc_order(tuple->docker, RSC_START, rsc, RSC_STARTED, pe_order_implies_then_printed, data_set); ++ new_rsc_order(tuple->docker, RSC_STOP, rsc, RSC_STOPPED, pe_order_implies_then_printed, data_set); + + if(tuple->ip) { + tuple->ip->cmds->internal_constraints(tuple->ip, data_set); +@@ -416,8 +413,8 @@ container_action_flags(action_t * action, node_t * node) + return flags; + } + +-enum pe_graph_flags +-container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags, ++static enum pe_graph_flags ++container_update_interleave_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags, + enum pe_action_flags filter, enum pe_ordering type) + { + gboolean current = FALSE; +@@ -425,22 +422,18 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum + container_variant_data_t *then_data = NULL; + GListPtr containers = NULL; + +- // At the point we need to force container X to stop because +- // resource Y needs to stop, here is where we'd implement that +- crm_trace("%s -> %s", first->uuid, then->uuid); +- if(first->rsc == NULL || then->rsc == NULL) { +- return changed; +- +- } else if(first->rsc->variant != then->rsc->variant) { +- return changed; // For now +- } +- + /* Fix this - lazy */ + if (crm_ends_with(first->uuid, "_stopped_0") + || crm_ends_with(first->uuid, "_demoted_0")) { + current = TRUE; + } + ++ /* Eventually we may want to allow interleaving between bundles ++ * and clones, but for now assert both sides are bundles ++ */ ++ CRM_ASSERT(first->rsc->variant == pe_container); ++ CRM_ASSERT(then->rsc->variant == pe_container); ++ + get_container_variant_data(then_data, then->rsc); + containers = get_container_list(first->rsc); + +@@ -506,6 +499,66 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum + return changed; + } + ++enum pe_graph_flags ++container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags, ++ enum pe_action_flags filter, enum pe_ordering type) ++{ ++ bool interleave = FALSE; ++ enum pe_graph_flags changed = pe_graph_none; ++ ++ crm_trace("%s -> %s", first->uuid, then->uuid); ++ ++ if(first->rsc == NULL || then->rsc == NULL) { ++ return changed; ++ ++ } else if(first->rsc->variant == then->rsc->variant) { ++ // When and how to turn on interleaving? ++ // interleave = TRUE; ++ } ++ ++ if(interleave) { ++ changed = container_update_interleave_actions(first, then, node, flags, filter, type); ++ ++ } else { ++ GListPtr gIter = then->rsc->children; ++ GListPtr containers = NULL; ++ ++ // Handle the 'primitive' ordering case ++ changed |= native_update_actions(first, then, node, flags, filter, type); ++ ++ // Now any children (or containers in the case of a bundle) ++ if(then->rsc->variant == pe_container) { ++ containers = get_container_list(then->rsc); ++ gIter = containers; ++ } ++ ++ for (; gIter != NULL; gIter = gIter->next) { ++ resource_t *then_child = (resource_t *) gIter->data; ++ enum pe_graph_flags then_child_changed = pe_graph_none; ++ action_t *then_child_action = find_first_action(then_child->actions, NULL, then->task, node); ++ ++ if (then_child_action) { ++ enum pe_action_flags then_child_flags = then_child->cmds->action_flags(then_child_action, node); ++ ++ if (is_set(then_child_flags, pe_action_runnable)) { ++ then_child_changed |= ++ then_child->cmds->update_actions(first, then_child_action, node, flags, filter, type); ++ } ++ changed |= then_child_changed; ++ if (then_child_changed & pe_graph_updated_then) { ++ for (GListPtr lpc = then_child_action->actions_after; lpc != NULL; lpc = lpc->next) { ++ action_wrapper_t *next = (action_wrapper_t *) lpc->data; ++ update_action(next->action); ++ } ++ } ++ } ++ } ++ ++ g_list_free(containers); ++ } ++ return changed; ++} ++ + void + container_rsc_location(resource_t * rsc, rsc_to_node_t * constraint) + { +diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot +index 20afbe6..b5ecaf5 100644 +--- a/pengine/test10/bundle-order-partial-start-2.dot ++++ b/pengine/test10/bundle-order-partial-start-2.dot +@@ -13,6 +13,7 @@ digraph "g" { + "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] + "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle_running_0" [ style = bold] + "galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] + "galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] +@@ -36,7 +37,6 @@ digraph "g" { + "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] +-"haproxy-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp +index 08f5084..a2740d7 100644 +--- a/pengine/test10/bundle-order-partial-start-2.exp ++++ b/pengine/test10/bundle-order-partial-start-2.exp +@@ -90,9 +90,6 @@ + + + +- +- +- + + + +@@ -325,6 +322,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot +index c36ff04..756acfd 100644 +--- a/pengine/test10/bundle-order-partial-start.dot ++++ b/pengine/test10/bundle-order-partial-start.dot +@@ -10,6 +10,7 @@ digraph "g" { + "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] + "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle_running_0" [ style = bold] + "galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] + "galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] +@@ -30,7 +31,6 @@ digraph "g" { + "haproxy-bundle-docker-0_monitor_0 undercloud" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] +-"haproxy-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp +index 4a5c01c..ebd5785 100644 +--- a/pengine/test10/bundle-order-partial-start.exp ++++ b/pengine/test10/bundle-order-partial-start.exp +@@ -103,9 +103,6 @@ + + + +- +- +- + + + +@@ -306,6 +303,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-partial-stop.dot b/pengine/test10/bundle-order-partial-stop.dot +index af2493c..c0e6616 100644 +--- a/pengine/test10/bundle-order-partial-stop.dot ++++ b/pengine/test10/bundle-order-partial-stop.dot +@@ -11,7 +11,7 @@ digraph "g" { + "galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] +-"galera-bundle-docker-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" -> "galera-bundle_stopped_0" [ style = bold] + "galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] + "galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] +@@ -40,6 +40,7 @@ digraph "g" { + "galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed] + "galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold] + "galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] + "galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] + "galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] +@@ -130,9 +131,11 @@ digraph "g" { + "rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "rabbitmq-bundle_stopped_0" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold] + "rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] +@@ -151,7 +154,7 @@ digraph "g" { + "redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] +-"redis-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"redis-bundle-docker-0_stop_0 undercloud" -> "redis-bundle_stopped_0" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] + "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] +@@ -181,6 +184,7 @@ digraph "g" { + "redis-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed] + "redis-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle_stop_0" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold] + "redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] + "redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] + "redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/bundle-order-partial-stop.exp b/pengine/test10/bundle-order-partial-stop.exp +index 02a7372..27ae320 100644 +--- a/pengine/test10/bundle-order-partial-stop.exp ++++ b/pengine/test10/bundle-order-partial-stop.exp +@@ -26,6 +26,9 @@ + + + ++ ++ ++ + + + +@@ -99,6 +102,9 @@ + + + ++ ++ ++ + + + +@@ -173,10 +179,10 @@ + + + +- ++ + + +- ++ + + + +@@ -286,9 +292,6 @@ + + + +- +- +- + + + +@@ -492,6 +495,9 @@ + + + ++ ++ ++ + + + +@@ -599,6 +605,9 @@ + + + ++ ++ ++ + + + +@@ -650,6 +659,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-partial-stop.summary b/pengine/test10/bundle-order-partial-stop.summary +index c341f4c..b30a237 100644 +--- a/pengine/test10/bundle-order-partial-stop.summary ++++ b/pengine/test10/bundle-order-partial-stop.summary +@@ -64,21 +64,21 @@ Executing cluster transition: + * Pseudo action: galera-bundle_demoted_0 + * Pseudo action: galera-bundle_stop_0 + * Pseudo action: rabbitmq-bundle-clone_stopped_0 +- * Pseudo action: rabbitmq-bundle_stopped_0 + * Resource action: rabbitmq-bundle-docker-0 stop on undercloud + * Pseudo action: galera-bundle-master_stop_0 ++ * Pseudo action: rabbitmq-bundle_stopped_0 + * Resource action: galera stop on galera-bundle-0 + * Resource action: galera-bundle-0 stop on undercloud + * Pseudo action: galera-bundle-master_stopped_0 +- * Pseudo action: galera-bundle_stopped_0 + * Resource action: galera-bundle-docker-0 stop on undercloud ++ * Pseudo action: galera-bundle_stopped_0 + * Pseudo action: redis-bundle_stop_0 + * Pseudo action: redis-bundle-master_stop_0 + * Resource action: redis stop on redis-bundle-0 + * Resource action: redis-bundle-0 stop on undercloud + * Pseudo action: redis-bundle-master_stopped_0 +- * Pseudo action: redis-bundle_stopped_0 + * Resource action: redis-bundle-docker-0 stop on undercloud ++ * Pseudo action: redis-bundle_stopped_0 + * Pseudo action: haproxy-bundle_stop_0 + * Resource action: haproxy-bundle-docker-0 stop on undercloud + * Pseudo action: haproxy-bundle_stopped_0 +diff --git a/pengine/test10/bundle-order-startup.dot b/pengine/test10/bundle-order-startup.dot +index 4e60d79..66053b9 100644 +--- a/pengine/test10/bundle-order-startup.dot ++++ b/pengine/test10/bundle-order-startup.dot +@@ -10,6 +10,7 @@ digraph "g" { + "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] + "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle_running_0" [ style = bold] + "galera-bundle-docker-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] + "galera-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] +@@ -32,7 +33,6 @@ digraph "g" { + "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle-docker-0_monitor_60000 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" -> "haproxy-bundle_running_0" [ style = bold] +-"haproxy-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "haproxy-bundle_running_0" -> "redis-bundle_start_0" [ style = bold] + "haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] +@@ -98,6 +98,7 @@ digraph "g" { + "rabbitmq-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle-0_start_0 undercloud" [ style = bold] + "rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq-bundle_running_0" [ style = bold] + "rabbitmq-bundle-docker-0_start_0 undercloud" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] +@@ -117,9 +118,9 @@ digraph "g" { + "redis-bundle-docker-0_monitor_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold] + "redis-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] +-"redis-bundle-docker-0_start_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] + "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-0_start_0 undercloud" [ style = bold] + "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-docker-0_monitor_60000 undercloud" [ style = bold] ++"redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle_running_0" [ style = bold] + "redis-bundle-docker-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold] + "redis-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp +index 66ecb08..2128d7c 100644 +--- a/pengine/test10/bundle-order-startup.exp ++++ b/pengine/test10/bundle-order-startup.exp +@@ -176,9 +176,6 @@ + + + +- +- +- + + + +@@ -294,9 +291,6 @@ + + + +- +- +- + + + +@@ -714,6 +708,9 @@ + + + ++ ++ ++ + + + +@@ -765,6 +762,9 @@ + + + ++ ++ ++ + + + +@@ -816,6 +816,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-stop.dot b/pengine/test10/bundle-order-stop.dot +index af2493c..c0e6616 100644 +--- a/pengine/test10/bundle-order-stop.dot ++++ b/pengine/test10/bundle-order-stop.dot +@@ -11,7 +11,7 @@ digraph "g" { + "galera-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "galera-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] +-"galera-bundle-docker-0_stop_0 undercloud" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_stop_0 undercloud" -> "galera-bundle_stopped_0" [ style = bold] + "galera-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] + "galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] +@@ -40,6 +40,7 @@ digraph "g" { + "galera-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = dashed] + "galera-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] ++"galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 undercloud" [ style = bold] + "galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] + "galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] + "galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] +@@ -130,9 +131,11 @@ digraph "g" { + "rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 undercloud" -> "rabbitmq-bundle_stopped_0" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold] + "rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] +@@ -151,7 +154,7 @@ digraph "g" { + "redis-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] +-"redis-bundle-docker-0_stop_0 undercloud" -> "haproxy-bundle-docker-0_stop_0 undercloud" [ style = bold] ++"redis-bundle-docker-0_stop_0 undercloud" -> "redis-bundle_stopped_0" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] + "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] +@@ -181,6 +184,7 @@ digraph "g" { + "redis-bundle_running_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = dashed] + "redis-bundle_start_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle_stop_0" -> "redis-bundle-docker-0_stop_0 undercloud" [ style = bold] + "redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] + "redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] + "redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/bundle-order-stop.exp b/pengine/test10/bundle-order-stop.exp +index 02a7372..27ae320 100644 +--- a/pengine/test10/bundle-order-stop.exp ++++ b/pengine/test10/bundle-order-stop.exp +@@ -26,6 +26,9 @@ + + + ++ ++ ++ + + + +@@ -99,6 +102,9 @@ + + + ++ ++ ++ + + + +@@ -173,10 +179,10 @@ + + + +- ++ + + +- ++ + + + +@@ -286,9 +292,6 @@ + + + +- +- +- + + + +@@ -492,6 +495,9 @@ + + + ++ ++ ++ + + + +@@ -599,6 +605,9 @@ + + + ++ ++ ++ + + + +@@ -650,6 +659,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-stop.summary b/pengine/test10/bundle-order-stop.summary +index c341f4c..b30a237 100644 +--- a/pengine/test10/bundle-order-stop.summary ++++ b/pengine/test10/bundle-order-stop.summary +@@ -64,21 +64,21 @@ Executing cluster transition: + * Pseudo action: galera-bundle_demoted_0 + * Pseudo action: galera-bundle_stop_0 + * Pseudo action: rabbitmq-bundle-clone_stopped_0 +- * Pseudo action: rabbitmq-bundle_stopped_0 + * Resource action: rabbitmq-bundle-docker-0 stop on undercloud + * Pseudo action: galera-bundle-master_stop_0 ++ * Pseudo action: rabbitmq-bundle_stopped_0 + * Resource action: galera stop on galera-bundle-0 + * Resource action: galera-bundle-0 stop on undercloud + * Pseudo action: galera-bundle-master_stopped_0 +- * Pseudo action: galera-bundle_stopped_0 + * Resource action: galera-bundle-docker-0 stop on undercloud ++ * Pseudo action: galera-bundle_stopped_0 + * Pseudo action: redis-bundle_stop_0 + * Pseudo action: redis-bundle-master_stop_0 + * Resource action: redis stop on redis-bundle-0 + * Resource action: redis-bundle-0 stop on undercloud + * Pseudo action: redis-bundle-master_stopped_0 +- * Pseudo action: redis-bundle_stopped_0 + * Resource action: redis-bundle-docker-0 stop on undercloud ++ * Pseudo action: redis-bundle_stopped_0 + * Pseudo action: haproxy-bundle_stop_0 + * Resource action: haproxy-bundle-docker-0 stop on undercloud + * Pseudo action: haproxy-bundle_stopped_0 +-- +1.8.3.1 + + +From a189db3a8f42ee4817b9b0e1fc128ea64e945130 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 20 Jun 2017 16:19:02 +1000 +Subject: [PATCH 6/8] Feature: PE: Generic ordering for anything with bundles + +--- + pengine/allocate.c | 4 +- + pengine/allocate.h | 3 - + pengine/clone.c | 220 +++------------------------------------- + pengine/container.c | 287 +++++++++++++++++++++++++++++++++++++++++----------- + pengine/master.c | 2 +- + pengine/utils.h | 10 +- + 6 files changed, 250 insertions(+), 276 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 8f15282..3a95be6 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -79,7 +79,7 @@ resource_alloc_functions_t resource_class_alloc_functions[] = { + clone_rsc_colocation_rh, + clone_rsc_location, + clone_action_flags, +- clone_update_actions, ++ container_update_actions, + clone_expand, + clone_append_meta, + }, +@@ -93,7 +93,7 @@ resource_alloc_functions_t resource_class_alloc_functions[] = { + master_rsc_colocation_rh, + clone_rsc_location, + clone_action_flags, +- clone_update_actions, ++ container_update_actions, + clone_expand, + master_append_meta, + }, +diff --git a/pengine/allocate.h b/pengine/allocate.h +index 9a30b80..4ca5d65 100644 +--- a/pengine/allocate.h ++++ b/pengine/allocate.h +@@ -168,9 +168,6 @@ extern enum pe_graph_flags native_update_actions(action_t * first, action_t * th + extern enum pe_graph_flags group_update_actions(action_t * first, action_t * then, node_t * node, + enum pe_action_flags flags, + enum pe_action_flags filter, enum pe_ordering type); +-extern enum pe_graph_flags clone_update_actions(action_t * first, action_t * then, node_t * node, +- enum pe_action_flags flags, +- enum pe_action_flags filter, enum pe_ordering type); + extern enum pe_graph_flags container_update_actions(action_t * first, action_t * then, node_t * node, + enum pe_action_flags flags, + enum pe_action_flags filter, enum pe_ordering type); +diff --git a/pengine/clone.c b/pengine/clone.c +index a79271e..28f368e 100644 +--- a/pengine/clone.c ++++ b/pengine/clone.c +@@ -944,34 +944,6 @@ assign_node(resource_t * rsc, node_t * node, gboolean force) + return changed; + } + +-static resource_t * +-find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc, +- GListPtr children, enum rsc_role_e filter, gboolean current) +-{ +- GListPtr gIter = NULL; +- +- if (local_node == NULL) { +- crm_err("Can't colocate unrunnable child %s with %s", local_child->id, rsc->id); +- return NULL; +- } +- +- crm_trace("Looking for compatible child from %s for %s on %s", +- local_child->id, rsc->id, local_node->details->uname); +- +- for (gIter = children; gIter != NULL; gIter = gIter->next) { +- resource_t *child_rsc = (resource_t *) gIter->data; +- +- if(is_child_compatible(child_rsc, local_node, filter, current)) { +- crm_trace("Pairing %s with %s on %s", +- local_child->id, child_rsc->id, local_node->details->uname); +- return child_rsc; +- } +- } +- +- crm_trace("Can't pair %s with %s", local_child->id, rsc->id); +- return NULL; +-} +- + gboolean + is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e filter, gboolean current) + { +@@ -1002,7 +974,7 @@ is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e + } + + resource_t * +-find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr children, enum rsc_role_e filter, gboolean current) ++find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_e filter, gboolean current) + { + resource_t *pair = NULL; + GListPtr gIter = NULL; +@@ -1011,7 +983,7 @@ find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr child + + local_node = local_child->fns->location(local_child, NULL, current); + if (local_node) { +- return find_compatible_child_by_node(local_child, local_node, rsc, children, filter, current); ++ return find_compatible_child_by_node(local_child, local_node, rsc, filter, current); + } + + scratch = g_hash_table_get_values(local_child->allowed_nodes); +@@ -1021,7 +993,7 @@ find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr child + for (; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + +- pair = find_compatible_child_by_node(local_child, node, rsc, children, filter, current); ++ pair = find_compatible_child_by_node(local_child, node, rsc, filter, current); + if (pair) { + goto done; + } +@@ -1048,29 +1020,26 @@ clone_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation + { + GListPtr gIter = NULL; + gboolean do_interleave = FALSE; +- clone_variant_data_t *clone_data = NULL; +- clone_variant_data_t *clone_data_lh = NULL; ++ const char *interleave_s = NULL; + + CRM_CHECK(constraint != NULL, return); + CRM_CHECK(rsc_lh != NULL, pe_err("rsc_lh was NULL for %s", constraint->id); return); + CRM_CHECK(rsc_rh != NULL, pe_err("rsc_rh was NULL for %s", constraint->id); return); + CRM_CHECK(rsc_lh->variant == pe_native, return); + +- get_clone_variant_data(clone_data, constraint->rsc_rh); + pe_rsc_trace(rsc_rh, "Processing constraint %s: %s -> %s %d", + constraint->id, rsc_lh->id, rsc_rh->id, constraint->score); + +- if (pe_rsc_is_clone(constraint->rsc_lh)) { +- +- get_clone_variant_data(clone_data_lh, constraint->rsc_lh); +- if (clone_data_lh->interleave +- && clone_data->clone_node_max != clone_data_lh->clone_node_max) { +- crm_config_err("Cannot interleave " XML_CIB_TAG_INCARNATION " %s and %s because" +- " they do not support the same number of" " resources per node", ++ /* only the LHS side needs to be labeled as interleave */ ++ interleave_s = g_hash_table_lookup(constraint->rsc_lh->meta, XML_RSC_ATTR_INTERLEAVE); ++ if(crm_is_true(interleave_s) && constraint->rsc_lh->variant > pe_group) { ++ // TODO: Do we actually care about multiple RH copies sharing a LH copy anymore? ++ if (copies_per_node(constraint->rsc_lh) != copies_per_node(constraint->rsc_rh)) { ++ crm_config_err("Cannot interleave %s and %s because" ++ " they do not support the same number of copies per node", + constraint->rsc_lh->id, constraint->rsc_rh->id); + +- /* only the LHS side needs to be labeled as interleave */ +- } else if (clone_data_lh->interleave) { ++ } else { + do_interleave = TRUE; + } + } +@@ -1082,7 +1051,7 @@ clone_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation + } else if (do_interleave) { + resource_t *rh_child = NULL; + +- rh_child = find_compatible_child(rsc_lh, rsc_rh, rsc_rh->children, RSC_ROLE_UNKNOWN, FALSE); ++ rh_child = find_compatible_child(rsc_lh, rsc_rh, RSC_ROLE_UNKNOWN, FALSE); + + if (rh_child) { + pe_rsc_debug(rsc_rh, "Pairing %s with %s", rsc_lh->id, rh_child->id); +@@ -1125,7 +1094,7 @@ clone_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation + } + } + +-static enum action_tasks ++enum action_tasks + clone_child_action(action_t * action) + { + enum action_tasks result = no_action; +@@ -1225,167 +1194,6 @@ clone_action_flags(action_t * action, node_t * node) + return summary_action_flags(action, action->rsc->children, node); + } + +-static enum pe_graph_flags +-clone_update_actions_interleave(action_t * first, action_t * then, node_t * node, +- enum pe_action_flags flags, enum pe_action_flags filter, +- enum pe_ordering type) +-{ +- gboolean current = FALSE; +- resource_t *first_child = NULL; +- GListPtr gIter = then->rsc->children; +- enum pe_graph_flags changed = pe_graph_none; /*pe_graph_disable */ +- +- enum action_tasks task = clone_child_action(first); +- const char *first_task = task2text(task); +- +- /* Fix this - lazy */ +- if (crm_ends_with(first->uuid, "_stopped_0") +- || crm_ends_with(first->uuid, "_demoted_0")) { +- current = TRUE; +- } +- +- for (; gIter != NULL; gIter = gIter->next) { +- resource_t *then_child = (resource_t *) gIter->data; +- +- CRM_ASSERT(then_child != NULL); +- first_child = find_compatible_child(then_child, first->rsc, first->rsc->children, RSC_ROLE_UNKNOWN, current); +- if (first_child == NULL && current) { +- crm_trace("Ignore"); +- +- } else if (first_child == NULL) { +- crm_debug("No match found for %s (%d / %s / %s)", then_child->id, current, first->uuid, +- then->uuid); +- +- /* Me no like this hack - but what else can we do? +- * +- * If there is no-one active or about to be active +- * on the same node as then_child, then they must +- * not be allowed to start +- */ +- if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) { +- pe_rsc_info(then->rsc, "Inhibiting %s from being active", then_child->id); +- if(assign_node(then_child, NULL, TRUE)) { +- changed |= pe_graph_updated_then; +- } +- } +- +- } else { +- action_t *first_action = NULL; +- action_t *then_action = NULL; +- +- pe_rsc_debug(then->rsc, "Pairing %s with %s", first_child->id, then_child->id); +- +- first_action = find_first_action(first_child->actions, NULL, first_task, node); +- then_action = find_first_action(then_child->actions, NULL, then->task, node); +- +- if (first_action == NULL) { +- if (is_not_set(first_child->flags, pe_rsc_orphan) +- && crm_str_eq(first_task, RSC_STOP, TRUE) == FALSE +- && crm_str_eq(first_task, RSC_DEMOTE, TRUE) == FALSE) { +- crm_err("Internal error: No action found for %s in %s (first)", +- first_task, first_child->id); +- +- } else { +- crm_trace("No action found for %s in %s%s (first)", +- first_task, first_child->id, +- is_set(first_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : ""); +- } +- continue; +- } +- +- /* We're only interested if 'then' is neither stopping nor being demoted */ +- if (then_action == NULL) { +- if (is_not_set(then_child->flags, pe_rsc_orphan) +- && crm_str_eq(then->task, RSC_STOP, TRUE) == FALSE +- && crm_str_eq(then->task, RSC_DEMOTE, TRUE) == FALSE) { +- crm_err("Internal error: No action found for %s in %s (then)", +- then->task, then_child->id); +- +- } else { +- crm_trace("No action found for %s in %s%s (then)", +- then->task, then_child->id, +- is_set(then_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : ""); +- } +- continue; +- } +- +- if (order_actions(first_action, then_action, type)) { +- crm_debug("Created constraint for %s -> %s", first_action->uuid, then_action->uuid); +- changed |= (pe_graph_updated_first | pe_graph_updated_then); +- } +- changed |= +- then_child->cmds->update_actions(first_action, then_action, node, +- first_child->cmds->action_flags(first_action, +- node), filter, +- type); +- } +- } +- return changed; +-} +- +-enum pe_graph_flags +-clone_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags, +- enum pe_action_flags filter, enum pe_ordering type) +-{ +- const char *rsc = "none"; +- gboolean interleave = FALSE; +- enum pe_graph_flags changed = pe_graph_none; +- +- if (first->rsc != then->rsc +- && pe_rsc_is_clone(first->rsc) +- && pe_rsc_is_clone(then->rsc)) { +- clone_variant_data_t *clone_data = NULL; +- +- if (crm_ends_with(then->uuid, "_stop_0") +- || crm_ends_with(then->uuid, "_demote_0")) { +- get_clone_variant_data(clone_data, first->rsc); +- rsc = first->rsc->id; +- } else { +- get_clone_variant_data(clone_data, then->rsc); +- rsc = then->rsc->id; +- } +- interleave = clone_data->interleave; +- } +- +- crm_trace("Interleave %s -> %s: %s (based on %s)", +- first->uuid, then->uuid, interleave ? "yes" : "no", rsc); +- +- if (interleave) { +- changed = clone_update_actions_interleave(first, then, node, flags, filter, type); +- +- } else if (then->rsc) { +- GListPtr gIter = then->rsc->children; +- +- changed |= native_update_actions(first, then, node, flags, filter, type); +- +- for (; gIter != NULL; gIter = gIter->next) { +- enum pe_graph_flags child_changed = pe_graph_none; +- GListPtr lpc = NULL; +- resource_t *child = (resource_t *) gIter->data; +- action_t *child_action = find_first_action(child->actions, NULL, then->task, node); +- +- if (child_action) { +- enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node); +- +- if (is_set(child_flags, pe_action_runnable)) { +- +- child_changed |= +- child->cmds->update_actions(first, child_action, node, flags, filter, type); +- } +- changed |= child_changed; +- if (child_changed & pe_graph_updated_then) { +- for (lpc = child_action->actions_after; lpc != NULL; lpc = lpc->next) { +- action_wrapper_t *other = (action_wrapper_t *) lpc->data; +- update_action(other->action); +- } +- } +- } +- } +- } +- +- return changed; +-} +- + void + clone_rsc_location(resource_t * rsc, rsc_to_node_t * constraint) + { +diff --git a/pengine/container.c b/pengine/container.c +index adee92a..f742660 100644 +--- a/pengine/container.c ++++ b/pengine/container.c +@@ -47,14 +47,33 @@ static GListPtr get_container_list(resource_t *rsc) + GListPtr containers = NULL; + container_variant_data_t *data = NULL; + +- get_container_variant_data(data, rsc); +- for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) { +- container_grouping_t *tuple = (container_grouping_t *)gIter->data; +- containers = g_list_append(containers, tuple->docker); ++ if(rsc->variant == pe_container) { ++ get_container_variant_data(data, rsc); ++ for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) { ++ container_grouping_t *tuple = (container_grouping_t *)gIter->data; ++ containers = g_list_append(containers, tuple->docker); ++ } + } + return containers; + } + ++static GListPtr get_containers_or_children(resource_t *rsc) ++{ ++ GListPtr containers = NULL; ++ container_variant_data_t *data = NULL; ++ ++ if(rsc->variant == pe_container) { ++ get_container_variant_data(data, rsc); ++ for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) { ++ container_grouping_t *tuple = (container_grouping_t *)gIter->data; ++ containers = g_list_append(containers, tuple->docker); ++ } ++ return containers; ++ } else { ++ return rsc->children; ++ } ++} ++ + node_t * + container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + { +@@ -337,6 +356,39 @@ container_rsc_colocation_lh(resource_t * rsc, resource_t * rsc_rh, rsc_colocatio + CRM_ASSERT(FALSE); + } + ++int copies_per_node(resource_t * rsc) ++{ ++ /* Strictly speaking, there should be a 'copies_per_node' addition ++ * to the resource function table and each case would be a ++ * function. However that would be serious overkill to return an ++ * int. In fact, it seems to me that both function tables ++ * could/should be replaced by resources.{c,h} full of ++ * rsc_{some_operation} functions containing a switch as below ++ * which calls out to functions named {variant}_{some_operation} ++ * as needed. ++ */ ++ switch(rsc->variant) { ++ case pe_unknown: ++ return 0; ++ case pe_native: ++ case pe_group: ++ return 1; ++ case pe_clone: ++ case pe_master: ++ { ++ const char *max_clones_node = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INCARNATION_NODEMAX); ++ return crm_parse_int(max_clones_node, "1"); ++ } ++ case pe_container: ++ { ++ container_variant_data_t *data = NULL; ++ get_container_variant_data(data, rsc); ++ return data->replicas_per_host; ++ } ++ } ++ return 0; ++} ++ + void + container_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc, rsc_colocation_t * constraint) + { +@@ -346,6 +398,7 @@ container_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc, rsc_colocatio + CRM_CHECK(constraint != NULL, return); + CRM_CHECK(rsc_lh != NULL, pe_err("rsc_lh was NULL for %s", constraint->id); return); + CRM_CHECK(rsc != NULL, pe_err("rsc was NULL for %s", constraint->id); return); ++ CRM_ASSERT(rsc_lh->variant == pe_native); + + if (is_set(rsc->flags, pe_rsc_provisional)) { + pe_rsc_trace(rsc, "%s is still provisional", rsc->id); +@@ -413,14 +466,65 @@ container_action_flags(action_t * action, node_t * node) + return flags; + } + ++resource_t * ++find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc, ++ enum rsc_role_e filter, gboolean current) ++{ ++ GListPtr gIter = NULL; ++ GListPtr children = NULL; ++ ++ if (local_node == NULL) { ++ crm_err("Can't colocate unrunnable child %s with %s", local_child->id, rsc->id); ++ return NULL; ++ } ++ ++ crm_trace("Looking for compatible child from %s for %s on %s", ++ local_child->id, rsc->id, local_node->details->uname); ++ ++ children = get_containers_or_children(rsc); ++ for (gIter = children; gIter != NULL; gIter = gIter->next) { ++ resource_t *child_rsc = (resource_t *) gIter->data; ++ ++ if(is_child_compatible(child_rsc, local_node, filter, current)) { ++ crm_trace("Pairing %s with %s on %s", ++ local_child->id, child_rsc->id, local_node->details->uname); ++ return child_rsc; ++ } ++ } ++ ++ crm_trace("Can't pair %s with %s", local_child->id, rsc->id); ++ if(children != rsc->children) { ++ g_list_free(children); ++ } ++ return NULL; ++} ++ ++static container_grouping_t * ++tuple_for_docker(resource_t *rsc, resource_t *docker, node_t *node) ++{ ++ if(rsc->variant == pe_container) { ++ container_variant_data_t *data = NULL; ++ get_container_variant_data(data, rsc); ++ for (GListPtr gIter = data->tuples; gIter != NULL; gIter = gIter->next) { ++ container_grouping_t *tuple = (container_grouping_t *)gIter->data; ++ if(tuple->child ++ && docker == tuple->docker ++ && node->details == tuple->node->details) { ++ return tuple; ++ } ++ } ++ } ++ return NULL; ++} ++ + static enum pe_graph_flags + container_update_interleave_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags, + enum pe_action_flags filter, enum pe_ordering type) + { ++ GListPtr gIter = NULL; ++ GListPtr children = NULL; + gboolean current = FALSE; + enum pe_graph_flags changed = pe_graph_none; +- container_variant_data_t *then_data = NULL; +- GListPtr containers = NULL; + + /* Fix this - lazy */ + if (crm_ends_with(first->uuid, "_stopped_0") +@@ -428,27 +532,15 @@ container_update_interleave_actions(action_t * first, action_t * then, node_t * + current = TRUE; + } + +- /* Eventually we may want to allow interleaving between bundles +- * and clones, but for now assert both sides are bundles +- */ +- CRM_ASSERT(first->rsc->variant == pe_container); +- CRM_ASSERT(then->rsc->variant == pe_container); +- +- get_container_variant_data(then_data, then->rsc); +- containers = get_container_list(first->rsc); +- +- for (GListPtr gIter = then_data->tuples; gIter != NULL; gIter = gIter->next) { +- container_grouping_t *tuple = (container_grouping_t *)gIter->data; +- +- /* We can't do the then_data->child->children trick here, +- * since the node's wont match +- */ +- resource_t *first_child = find_compatible_child(tuple->docker, first->rsc, containers, RSC_ROLE_UNKNOWN, current); ++ children = get_containers_or_children(then->rsc); ++ for (gIter = children; gIter != NULL; gIter = gIter->next) { ++ resource_t *then_child = (resource_t *) gIter->data; ++ resource_t *first_child = find_compatible_child(then_child, first->rsc, RSC_ROLE_UNKNOWN, current); + if (first_child == NULL && current) { + crm_trace("Ignore"); + + } else if (first_child == NULL) { +- crm_debug("No match found for %s (%d / %s / %s)", tuple->docker->id, current, first->uuid, then->uuid); ++ crm_debug("No match found for %s (%d / %s / %s)", then_child->id, current, first->uuid, then->uuid); + + /* Me no like this hack - but what else can we do? + * +@@ -457,25 +549,76 @@ container_update_interleave_actions(action_t * first, action_t * then, node_t * + * not be allowed to start + */ + if (type & (pe_order_runnable_left | pe_order_implies_then) /* Mandatory */ ) { +- pe_rsc_info(then->rsc, "Inhibiting %s from being active", tuple->docker->id); +- if(assign_node(tuple->docker, NULL, TRUE)) { ++ pe_rsc_info(then->rsc, "Inhibiting %s from being active", then_child->id); ++ if(assign_node(then_child, NULL, TRUE)) { + changed |= pe_graph_updated_then; + } + } + + } else { +- enum action_tasks task = get_complex_task(first_child, first->task, TRUE); ++ pe_action_t *first_action = NULL; ++ pe_action_t *then_action = NULL; + +- /* Potentially we might want to invovle first_data->child +- * if present, however we mostly just need the "you need +- * to stop" signal to flow back up the ordering chain via +- * the docker resources which are always present +- * +- * Almost certain to break if first->task or then->task is +- * promote or demote +- */ +- pe_action_t *first_action = find_first_action(first_child->actions, NULL, task2text(task), node); +- pe_action_t *then_action = find_first_action(tuple->docker->actions, NULL, then->task, node); ++ enum action_tasks task = clone_child_action(first); ++ const char *first_task = task2text(task); ++ ++ container_grouping_t *first_tuple = tuple_for_docker(first->rsc, first_child, node); ++ container_grouping_t *then_tuple = tuple_for_docker(then->rsc, then_child, node); ++ ++ if(strstr(first->task, "stop") && first_tuple && first_tuple->child) { ++ /* Except for 'stopped' we should be looking at the ++ * in-container resource, actions for the child will ++ * happen later and are therefor more likely to align ++ * with the user's intent. ++ */ ++ first_action = find_first_action(first_tuple->child->actions, NULL, task2text(task), node); ++ } else { ++ first_action = find_first_action(first_child->actions, NULL, task2text(task), node); ++ } ++ ++ if(strstr(then->task, "mote") && then_tuple && then_tuple->child) { ++ /* Promote/demote actions will never be found for the ++ * docker resource, look in the child instead ++ * ++ * Alternatively treat: ++ * 'XXXX then promote YYYY' as 'XXXX then start container for YYYY', and ++ * 'demote XXXX then stop YYYY' as 'stop container for XXXX then stop YYYY' ++ */ ++ then_action = find_first_action(then_tuple->child->actions, NULL, then->task, node); ++ } else { ++ then_action = find_first_action(then_child->actions, NULL, then->task, node); ++ } ++ ++ if (first_action == NULL) { ++ if (is_not_set(first_child->flags, pe_rsc_orphan) ++ && crm_str_eq(first_task, RSC_STOP, TRUE) == FALSE ++ && crm_str_eq(first_task, RSC_DEMOTE, TRUE) == FALSE) { ++ crm_err("Internal error: No action found for %s in %s (first)", ++ first_task, first_child->id); ++ ++ } else { ++ crm_trace("No action found for %s in %s%s (first)", ++ first_task, first_child->id, ++ is_set(first_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : ""); ++ } ++ continue; ++ } ++ ++ /* We're only interested if 'then' is neither stopping nor being demoted */ ++ if (then_action == NULL) { ++ if (is_not_set(then_child->flags, pe_rsc_orphan) ++ && crm_str_eq(then->task, RSC_STOP, TRUE) == FALSE ++ && crm_str_eq(then->task, RSC_DEMOTE, TRUE) == FALSE) { ++ crm_err("Internal error: No action found for %s in %s (then)", ++ then->task, then_child->id); ++ ++ } else { ++ crm_trace("No action found for %s in %s%s (then)", ++ then->task, then_child->id, ++ is_set(then_child->flags, pe_rsc_orphan) ? " (ORPHAN)" : ""); ++ } ++ continue; ++ } + + if (order_actions(first_action, then_action, type)) { + crm_debug("Created constraint for %s (%d) -> %s (%d) %.6x", +@@ -484,55 +627,75 @@ container_update_interleave_actions(action_t * first, action_t * then, node_t * + changed |= (pe_graph_updated_first | pe_graph_updated_then); + } + if(first_action && then_action) { +- changed |= tuple->docker->cmds->update_actions(first_action, then_action, node, +- first_child->cmds->action_flags(first_action, node), +- filter, type); ++ changed |= then_child->cmds->update_actions(first_action, then_action, node, ++ first_child->cmds->action_flags(first_action, node), ++ filter, type); + } else { + crm_err("Nothing found either for %s (%p) or %s (%p) %s", + first_child->id, first_action, +- tuple->docker->id, then_action, task2text(task)); ++ then_child->id, then_action, task2text(task)); + } + } + } + +- g_list_free(containers); ++ if(children != then->rsc->children) { ++ g_list_free(children); ++ } + return changed; + } + ++bool can_interleave_actions(pe_action_t *first, pe_action_t *then) ++{ ++ bool interleave = FALSE; ++ resource_t *rsc = NULL; ++ const char *interleave_s = NULL; ++ ++ if(first->rsc == NULL || then->rsc == NULL) { ++ crm_trace("Not interleaving %s with %s (both must be resources)", first->uuid, then->uuid); ++ return FALSE; ++ } else if(first->rsc == then->rsc) { ++ crm_trace("Not interleaving %s with %s (must belong to different resources)", first->uuid, then->uuid); ++ return FALSE; ++ } else if(first->rsc->variant < pe_clone || then->rsc->variant < pe_clone) { ++ crm_trace("Not interleaving %s with %s (both sides must be clones, masters, or bundles)", first->uuid, then->uuid); ++ return FALSE; ++ } ++ ++ if (crm_ends_with(then->uuid, "_stop_0") || crm_ends_with(then->uuid, "_demote_0")) { ++ rsc = first->rsc; ++ } else { ++ rsc = then->rsc; ++ } ++ ++ interleave_s = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_INTERLEAVE); ++ interleave = crm_is_true(interleave_s); ++ crm_trace("Interleave %s -> %s: %s (based on %s)", ++ first->uuid, then->uuid, interleave ? "yes" : "no", rsc->id); ++ ++ return interleave; ++} ++ + enum pe_graph_flags + container_update_actions(action_t * first, action_t * then, node_t * node, enum pe_action_flags flags, + enum pe_action_flags filter, enum pe_ordering type) + { +- bool interleave = FALSE; + enum pe_graph_flags changed = pe_graph_none; + + crm_trace("%s -> %s", first->uuid, then->uuid); + +- if(first->rsc == NULL || then->rsc == NULL) { +- return changed; +- +- } else if(first->rsc->variant == then->rsc->variant) { +- // When and how to turn on interleaving? +- // interleave = TRUE; +- } +- +- if(interleave) { ++ if(can_interleave_actions(first, then)) { + changed = container_update_interleave_actions(first, then, node, flags, filter, type); + +- } else { +- GListPtr gIter = then->rsc->children; +- GListPtr containers = NULL; ++ } else if(then->rsc) { ++ GListPtr gIter = NULL; ++ GListPtr children = NULL; + + // Handle the 'primitive' ordering case + changed |= native_update_actions(first, then, node, flags, filter, type); + + // Now any children (or containers in the case of a bundle) +- if(then->rsc->variant == pe_container) { +- containers = get_container_list(then->rsc); +- gIter = containers; +- } +- +- for (; gIter != NULL; gIter = gIter->next) { ++ children = get_containers_or_children(then->rsc); ++ for (gIter = children; gIter != NULL; gIter = gIter->next) { + resource_t *then_child = (resource_t *) gIter->data; + enum pe_graph_flags then_child_changed = pe_graph_none; + action_t *then_child_action = find_first_action(then_child->actions, NULL, then->task, node); +@@ -554,7 +717,9 @@ container_update_actions(action_t * first, action_t * then, node_t * node, enum + } + } + +- g_list_free(containers); ++ if(children != then->rsc->children) { ++ g_list_free(children); ++ } + } + return changed; + } +diff --git a/pengine/master.c b/pengine/master.c +index c15e740..8c39f49 100644 +--- a/pengine/master.c ++++ b/pengine/master.c +@@ -1022,7 +1022,7 @@ master_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocatio + g_list_free(rhs); + + } else if (constraint->role_lh == RSC_ROLE_MASTER) { +- resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, rsc_rh->children, constraint->role_rh, FALSE); ++ resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE); + + if (rh_child == NULL && constraint->score >= INFINITY) { + pe_rsc_trace(rsc_lh, "%s can't be promoted %s", rsc_lh->id, constraint->id); +diff --git a/pengine/utils.h b/pengine/utils.h +index 1421166..7a788f7 100644 +--- a/pengine/utils.h ++++ b/pengine/utils.h +@@ -49,12 +49,16 @@ void native_deallocate(resource_t * rsc); + extern void log_action(unsigned int log_level, const char *pre_text, + action_t * action, gboolean details); + +-extern gboolean can_run_any(GHashTable * nodes); +-extern resource_t *find_compatible_child(resource_t * local_child, resource_t * rsc, GListPtr children, +- enum rsc_role_e filter, gboolean current); ++gboolean can_run_any(GHashTable * nodes); ++bool can_interleave_actions(pe_action_t *first, pe_action_t *then); ++resource_t *find_compatible_child(resource_t * local_child, resource_t * rsc, enum rsc_role_e filter, gboolean current); ++resource_t *find_compatible_child_by_node(resource_t * local_child, node_t * local_node, resource_t * rsc, ++ enum rsc_role_e filter, gboolean current); + gboolean is_child_compatible(resource_t *child_rsc, node_t * local_node, enum rsc_role_e filter, gboolean current); + bool assign_node(resource_t * rsc, node_t * node, gboolean force); + enum pe_action_flags summary_action_flags(action_t * action, GListPtr children, node_t * node); ++enum action_tasks clone_child_action(action_t * action); ++int copies_per_node(resource_t * rsc); + + enum filter_colocation_res { + influence_nothing = 0, +-- +1.8.3.1 + + +From 879a52ce9f4288fde5c19d2fb677e5ad51e6c4af Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Fri, 23 Jun 2017 13:16:57 +1000 +Subject: [PATCH 7/8] Test: PE: Generic ordering for anything with bundles + +--- + lib/pengine/container.c | 14 +- + pengine/regression.sh | 5 +- + pengine/test10/bundle-order-startup-clone-2.dot | 335 ++++ + pengine/test10/bundle-order-startup-clone-2.exp | 1697 ++++++++++++++++++++ + pengine/test10/bundle-order-startup-clone-2.scores | 584 +++++++ + .../test10/bundle-order-startup-clone-2.summary | 179 +++ + pengine/test10/bundle-order-startup-clone-2.xml | 190 +++ + pengine/test10/bundle-order-startup-clone.dot | 119 ++ + pengine/test10/bundle-order-startup-clone.exp | 327 ++++ + pengine/test10/bundle-order-startup-clone.scores | 174 ++ + pengine/test10/bundle-order-startup-clone.summary | 69 + + pengine/test10/bundle-order-startup-clone.xml | 187 +++ + pengine/test10/bundle-order-stop-clone.dot | 80 + + pengine/test10/bundle-order-stop-clone.exp | 345 ++++ + pengine/test10/bundle-order-stop-clone.scores | 591 +++++++ + pengine/test10/bundle-order-stop-clone.summary | 75 + + pengine/test10/bundle-order-stop-clone.xml | 398 +++++ + 17 files changed, 5366 insertions(+), 3 deletions(-) + create mode 100644 pengine/test10/bundle-order-startup-clone-2.dot + create mode 100644 pengine/test10/bundle-order-startup-clone-2.exp + create mode 100644 pengine/test10/bundle-order-startup-clone-2.scores + create mode 100644 pengine/test10/bundle-order-startup-clone-2.summary + create mode 100644 pengine/test10/bundle-order-startup-clone-2.xml + create mode 100644 pengine/test10/bundle-order-startup-clone.dot + create mode 100644 pengine/test10/bundle-order-startup-clone.exp + create mode 100644 pengine/test10/bundle-order-startup-clone.scores + create mode 100644 pengine/test10/bundle-order-startup-clone.summary + create mode 100644 pengine/test10/bundle-order-startup-clone.xml + create mode 100644 pengine/test10/bundle-order-stop-clone.dot + create mode 100644 pengine/test10/bundle-order-stop-clone.exp + create mode 100644 pengine/test10/bundle-order-stop-clone.scores + create mode 100644 pengine/test10/bundle-order-stop-clone.summary + create mode 100644 pengine/test10/bundle-order-stop-clone.xml + +diff --git a/lib/pengine/container.c b/lib/pengine/container.c +index cf81253..6e98e6f 100644 +--- a/lib/pengine/container.c ++++ b/lib/pengine/container.c +@@ -727,8 +727,6 @@ container_unpack(resource_t * rsc, pe_working_set_t * data_set) + } + + container_data->child = new_rsc; +- container_data->child->orig_xml = xml_obj; // Also the trigger for common_free() +- // to free xml_resource as container_data->child->xml + + mount = calloc(1, sizeof(container_mount_t)); + mount->source = strdup(DEFAULT_REMOTE_KEY_LOCATION); +@@ -1035,17 +1033,25 @@ tuple_free(container_grouping_t *tuple) + + if(tuple->ip) { + tuple->ip->fns->free(tuple->ip); ++ tuple->ip->xml = NULL; ++ free_xml(tuple->ip->xml); + tuple->ip = NULL; + } + if(tuple->child) { ++ free_xml(tuple->child->xml); ++ tuple->child->xml = NULL; + tuple->child->fns->free(tuple->child); + tuple->child = NULL; + } + if(tuple->docker) { ++ free_xml(tuple->docker->xml); ++ tuple->docker->xml = NULL; + tuple->docker->fns->free(tuple->docker); + tuple->docker = NULL; + } + if(tuple->remote) { ++ free_xml(tuple->remote->xml); ++ tuple->remote->xml = NULL; + tuple->remote->fns->free(tuple->remote); + tuple->remote = NULL; + } +@@ -1073,9 +1079,13 @@ container_free(resource_t * rsc) + free(container_data->docker_run_command); + free(container_data->docker_host_options); + ++ if(container_data->child) { ++ free_xml(container_data->child->xml); ++ } + g_list_free_full(container_data->tuples, (GDestroyNotify)tuple_free); + g_list_free_full(container_data->mounts, (GDestroyNotify)mount_free); + g_list_free_full(container_data->ports, (GDestroyNotify)port_free); ++ g_list_free(rsc->children); + common_free(rsc); + } + +diff --git a/pengine/regression.sh b/pengine/regression.sh +index e97b54b..b844bb9 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -23,7 +23,6 @@ core=`dirname $0` + create_mode="true" + info Generating test outputs for these tests... + # do_test file description +- + info Done. + echo "" + +@@ -808,6 +807,10 @@ do_test bundle-order-partial-start-2 "Bundle startup ordering when some dependan + do_test bundle-order-stop "Bundle stop ordering" + do_test bundle-order-partial-stop "Bundle startup ordering when some dependancies are already stopped" + ++do_test bundle-order-startup-clone "Prevent startup because bundle isn't promoted" ++do_test bundle-order-startup-clone-2 "Bundle startup with clones" ++do_test bundle-order-stop-clone "Stop bundle because clone is stopping" ++ + echo "" + do_test whitebox-fail1 "Fail whitebox container rsc." + do_test whitebox-fail2 "Fail whitebox container rsc lrmd connection." +diff --git a/pengine/test10/bundle-order-startup-clone-2.dot b/pengine/test10/bundle-order-startup-clone-2.dot +new file mode 100644 +index 0000000..af90261 +--- /dev/null ++++ b/pengine/test10/bundle-order-startup-clone-2.dot +@@ -0,0 +1,335 @@ ++digraph "g" { ++"galera-bundle-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = bold] ++"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_start_0 metal-2" -> "galera-bundle-1_monitor_60000 metal-2" [ style = bold] ++"galera-bundle-1_start_0 metal-2" -> "galera:1_monitor_20000 galera-bundle-1" [ style = bold] ++"galera-bundle-1_start_0 metal-2" -> "galera:1_monitor_30000 galera-bundle-1" [ style = bold] ++"galera-bundle-1_start_0 metal-2" -> "galera:1_start_0 galera-bundle-1" [ style = bold] ++"galera-bundle-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_start_0 metal-3" -> "galera-bundle-2_monitor_60000 metal-3" [ style = bold] ++"galera-bundle-2_start_0 metal-3" -> "galera:2_monitor_20000 galera-bundle-2" [ style = bold] ++"galera-bundle-2_start_0 metal-3" -> "galera:2_monitor_30000 galera-bundle-2" [ style = bold] ++"galera-bundle-2_start_0 metal-3" -> "galera:2_start_0 galera-bundle-2" [ style = bold] ++"galera-bundle-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-docker-0_monitor_60000 metal-1" [ style = bold] ++"galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle_running_0" [ style = bold] ++"galera-bundle-docker-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-docker-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle-docker-1_monitor_60000 metal-2" [ style = bold] ++"galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle_running_0" [ style = bold] ++"galera-bundle-docker-1_start_0 metal-2" -> "galera:1_start_0 galera-bundle-1" [ style = bold] ++"galera-bundle-docker-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle-docker-2_monitor_60000 metal-3" [ style = bold] ++"galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle_running_0" [ style = bold] ++"galera-bundle-docker-2_start_0 metal-3" -> "galera:2_start_0 galera-bundle-2" [ style = bold] ++"galera-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] ++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold] ++"galera-bundle-master_start_0" -> "galera:0_start_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_start_0" -> "galera:1_start_0 galera-bundle-1" [ style = bold] ++"galera-bundle-master_start_0" -> "galera:2_start_0 galera-bundle-2" [ style = bold] ++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] ++"galera-bundle_start_0" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] ++"galera-bundle_start_0" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] ++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] ++"galera:0_start_0 galera-bundle-0" -> "galera:1_start_0 galera-bundle-1" [ style = bold] ++"galera:0_start_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"galera:1_monitor_20000 galera-bundle-1" [ style=bold color="green" fontcolor="black"] ++"galera:1_monitor_30000 galera-bundle-1" [ style=bold color="green" fontcolor="black"] ++"galera:1_start_0 galera-bundle-1" -> "galera-bundle-master_running_0" [ style = bold] ++"galera:1_start_0 galera-bundle-1" -> "galera:1_monitor_20000 galera-bundle-1" [ style = bold] ++"galera:1_start_0 galera-bundle-1" -> "galera:1_monitor_30000 galera-bundle-1" [ style = bold] ++"galera:1_start_0 galera-bundle-1" -> "galera:2_start_0 galera-bundle-2" [ style = bold] ++"galera:1_start_0 galera-bundle-1" [ style=bold color="green" fontcolor="black"] ++"galera:2_monitor_20000 galera-bundle-2" [ style=bold color="green" fontcolor="black"] ++"galera:2_monitor_30000 galera-bundle-2" [ style=bold color="green" fontcolor="black"] ++"galera:2_start_0 galera-bundle-2" -> "galera-bundle-master_running_0" [ style = bold] ++"galera:2_start_0 galera-bundle-2" -> "galera:2_monitor_20000 galera-bundle-2" [ style = bold] ++"galera:2_start_0 galera-bundle-2" -> "galera:2_monitor_30000 galera-bundle-2" [ style = bold] ++"galera:2_start_0 galera-bundle-2" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 metal-1" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 metal-2" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 metal-3" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-0_start_0 metal-1" -> "haproxy-bundle-docker-0_monitor_60000 metal-1" [ style = bold] ++"haproxy-bundle-docker-0_start_0 metal-1" -> "haproxy-bundle_running_0" [ style = bold] ++"haproxy-bundle-docker-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-1_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle-docker-1_monitor_0 metal-1" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle-docker-1_monitor_0 metal-1" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-1_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle-docker-1_monitor_0 metal-2" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle-docker-1_monitor_0 metal-2" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-1_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle-docker-1_monitor_0 metal-3" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle-docker-1_monitor_0 metal-3" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-1_start_0 metal-2" -> "haproxy-bundle-docker-1_monitor_60000 metal-2" [ style = bold] ++"haproxy-bundle-docker-1_start_0 metal-2" -> "haproxy-bundle_running_0" [ style = bold] ++"haproxy-bundle-docker-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-2_monitor_0 metal-1" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle-docker-2_monitor_0 metal-1" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle-docker-2_monitor_0 metal-1" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-2_monitor_0 metal-2" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle-docker-2_monitor_0 metal-2" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle-docker-2_monitor_0 metal-2" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-2_monitor_0 metal-3" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle-docker-2_monitor_0 metal-3" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle-docker-2_monitor_0 metal-3" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle-docker-2_start_0 metal-3" -> "haproxy-bundle-docker-2_monitor_60000 metal-3" [ style = bold] ++"haproxy-bundle-docker-2_start_0 metal-3" -> "haproxy-bundle_running_0" [ style = bold] ++"haproxy-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"haproxy-bundle_running_0" -> "storage-clone_start_0" [ style = bold] ++"haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 metal-1" [ style = bold] ++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] ++"haproxy-bundle_start_0" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] ++"haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_start_0 metal-1" -> "redis-bundle-0_monitor_60000 metal-1" [ style = bold] ++"redis-bundle-0_start_0 metal-1" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold] ++"redis-bundle-0_start_0 metal-1" -> "redis:0_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-0_start_0 metal-1" -> "redis:0_start_0 redis-bundle-0" [ style = bold] ++"redis-bundle-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-1_start_0 metal-2" -> "redis-bundle-1_monitor_60000 metal-2" [ style = bold] ++"redis-bundle-1_start_0 metal-2" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold] ++"redis-bundle-1_start_0 metal-2" -> "redis:1_promote_0 redis-bundle-1" [ style = bold] ++"redis-bundle-1_start_0 metal-2" -> "redis:1_start_0 redis-bundle-1" [ style = bold] ++"redis-bundle-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_start_0 metal-3" -> "redis-bundle-2_monitor_60000 metal-3" [ style = bold] ++"redis-bundle-2_start_0 metal-3" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold] ++"redis-bundle-2_start_0 metal-3" -> "redis:2_promote_0 redis-bundle-2" [ style = bold] ++"redis-bundle-2_start_0 metal-3" -> "redis:2_start_0 redis-bundle-2" [ style = bold] ++"redis-bundle-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle-docker-0_monitor_60000 metal-1" [ style = bold] ++"redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-docker-0_start_0 metal-1" -> "redis:0_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-docker-0_start_0 metal-1" -> "redis:0_start_0 redis-bundle-0" [ style = bold] ++"redis-bundle-docker-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle-docker-1_monitor_60000 metal-2" [ style = bold] ++"redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-docker-1_start_0 metal-2" -> "redis:1_promote_0 redis-bundle-1" [ style = bold] ++"redis-bundle-docker-1_start_0 metal-2" -> "redis:1_start_0 redis-bundle-1" [ style = bold] ++"redis-bundle-docker-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle-docker-2_monitor_60000 metal-3" [ style = bold] ++"redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-docker-2_start_0 metal-3" -> "redis:2_promote_0 redis-bundle-2" [ style = bold] ++"redis-bundle-docker-2_start_0 metal-3" -> "redis:2_start_0 redis-bundle-2" [ style = bold] ++"redis-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_promote_0" -> "redis:0_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_promote_0" -> "redis:1_promote_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_promote_0" -> "redis:2_promote_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold] ++"redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_start_0" -> "redis:1_start_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_start_0" -> "redis:2_start_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_promoted_0" -> "storage-clone_start_0" [ style = bold] ++"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_running_0" -> "redis-bundle_promote_0" [ style = bold] ++"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_start_0" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] ++"redis-bundle_start_0" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] ++"redis-bundle_start_0" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold] ++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis:0_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:0_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold] ++"redis:0_promote_0 redis-bundle-0" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold] ++"redis:0_promote_0 redis-bundle-0" -> "redis:1_promote_0 redis-bundle-1" [ style = bold] ++"redis:0_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:0_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = bold] ++"redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold] ++"redis:0_start_0 redis-bundle-0" -> "redis:0_promote_0 redis-bundle-0" [ style = bold] ++"redis:0_start_0 redis-bundle-0" -> "redis:1_start_0 redis-bundle-1" [ style = bold] ++"redis:0_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:1_monitor_20000 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis:1_promote_0 redis-bundle-1" -> "redis-bundle-master_promoted_0" [ style = bold] ++"redis:1_promote_0 redis-bundle-1" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold] ++"redis:1_promote_0 redis-bundle-1" -> "redis:2_promote_0 redis-bundle-2" [ style = bold] ++"redis:1_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis:1_start_0 redis-bundle-1" -> "redis-bundle-master_running_0" [ style = bold] ++"redis:1_start_0 redis-bundle-1" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold] ++"redis:1_start_0 redis-bundle-1" -> "redis:1_promote_0 redis-bundle-1" [ style = bold] ++"redis:1_start_0 redis-bundle-1" -> "redis:2_start_0 redis-bundle-2" [ style = bold] ++"redis:1_start_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis:2_monitor_20000 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis:2_promote_0 redis-bundle-2" -> "redis-bundle-master_promoted_0" [ style = bold] ++"redis:2_promote_0 redis-bundle-2" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold] ++"redis:2_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis:2_start_0 redis-bundle-2" -> "redis-bundle-master_running_0" [ style = bold] ++"redis:2_start_0 redis-bundle-2" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold] ++"redis:2_start_0 redis-bundle-2" -> "redis:2_promote_0 redis-bundle-2" [ style = bold] ++"redis:2_start_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"storage-clone_confirmed-post_notify_running_0" -> "galera-bundle_start_0" [ style = bold] ++"storage-clone_confirmed-post_notify_running_0" -> "storage:0_monitor_30000 metal-1" [ style = bold] ++"storage-clone_confirmed-post_notify_running_0" -> "storage:1_monitor_30000 metal-2" [ style = bold] ++"storage-clone_confirmed-post_notify_running_0" -> "storage:2_monitor_30000 metal-3" [ style = bold] ++"storage-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_post_notify_running_0" [ style = bold] ++"storage-clone_confirmed-pre_notify_start_0" -> "storage-clone_start_0" [ style = bold] ++"storage-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"storage-clone_post_notify_running_0" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold] ++"storage-clone_post_notify_running_0" -> "storage:0_post_notify_start_0 metal-1" [ style = bold] ++"storage-clone_post_notify_running_0" -> "storage:1_post_notify_start_0 metal-2" [ style = bold] ++"storage-clone_post_notify_running_0" -> "storage:2_post_notify_start_0 metal-3" [ style = bold] ++"storage-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"storage-clone_pre_notify_start_0" -> "storage-clone_confirmed-pre_notify_start_0" [ style = bold] ++"storage-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"storage-clone_running_0" -> "storage-clone_post_notify_running_0" [ style = bold] ++"storage-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"storage-clone_start_0" -> "storage-clone_running_0" [ style = bold] ++"storage-clone_start_0" -> "storage:0_start_0 metal-1" [ style = bold] ++"storage-clone_start_0" -> "storage:1_start_0 metal-2" [ style = bold] ++"storage-clone_start_0" -> "storage:2_start_0 metal-3" [ style = bold] ++"storage-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"storage:0_monitor_0 metal-1" -> "storage-clone_start_0" [ style = bold] ++"storage:0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"storage:0_monitor_30000 metal-1" [ style=bold color="green" fontcolor="black"] ++"storage:0_post_notify_start_0 metal-1" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold] ++"storage:0_post_notify_start_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"storage:0_start_0 metal-1" -> "storage-clone_running_0" [ style = bold] ++"storage:0_start_0 metal-1" -> "storage:0_monitor_30000 metal-1" [ style = bold] ++"storage:0_start_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"storage:1_monitor_0 metal-2" -> "storage-clone_start_0" [ style = bold] ++"storage:1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"storage:1_monitor_30000 metal-2" [ style=bold color="green" fontcolor="black"] ++"storage:1_post_notify_start_0 metal-2" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold] ++"storage:1_post_notify_start_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"storage:1_start_0 metal-2" -> "storage-clone_running_0" [ style = bold] ++"storage:1_start_0 metal-2" -> "storage:1_monitor_30000 metal-2" [ style = bold] ++"storage:1_start_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"storage:2_monitor_0 metal-3" -> "storage-clone_start_0" [ style = bold] ++"storage:2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"storage:2_monitor_30000 metal-3" [ style=bold color="green" fontcolor="black"] ++"storage:2_post_notify_start_0 metal-3" -> "storage-clone_confirmed-post_notify_running_0" [ style = bold] ++"storage:2_post_notify_start_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"storage:2_start_0 metal-3" -> "storage-clone_running_0" [ style = bold] ++"storage:2_start_0 metal-3" -> "storage:2_monitor_30000 metal-3" [ style = bold] ++"storage:2_start_0 metal-3" [ style=bold color="green" fontcolor="black"] ++} +diff --git a/pengine/test10/bundle-order-startup-clone-2.exp b/pengine/test10/bundle-order-startup-clone-2.exp +new file mode 100644 +index 0000000..d05eb96 +--- /dev/null ++++ b/pengine/test10/bundle-order-startup-clone-2.exp +@@ -0,0 +1,1697 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++