diff --git a/SOURCES/080-docker-location.patch b/SOURCES/080-docker-location.patch new file mode 100644 index 0000000..c98f1b0 --- /dev/null +++ b/SOURCES/080-docker-location.patch @@ -0,0 +1,30 @@ +From 84e1eb06f3434edc20d87d3d3b5d09c755eeee69 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Fri, 5 May 2017 18:51:49 +1000 +Subject: [PATCH] container: The location of the docker resource is the more + interesting one + +--- + lib/pengine/container.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/lib/pengine/container.c b/lib/pengine/container.c +index 1280227..043f129 100644 +--- a/lib/pengine/container.c ++++ b/lib/pengine/container.c +@@ -883,9 +883,9 @@ tuple_print(container_grouping_t * tuple, const char *pre_text, long options, vo + offset += snprintf(buffer + offset, LINE_MAX - offset, " (%s)", tuple->ipaddr); + } + +- if(tuple->remote && tuple->remote->running_on != NULL) { +- node = tuple->remote->running_on->data; +- } else if (tuple->remote == NULL && rsc->running_on != NULL) { ++ if(tuple->docker && tuple->docker->running_on != NULL) { ++ node = tuple->docker->running_on->data; ++ } else if (tuple->docker == NULL && rsc->running_on != NULL) { + node = rsc->running_on->data; + } + common_print(rsc, pre_text, buffer, node, options, print_data); +-- +1.8.3.1 + diff --git a/SOURCES/081-fence-logs.patch b/SOURCES/081-fence-logs.patch new file mode 100644 index 0000000..ddb2eed --- /dev/null +++ b/SOURCES/081-fence-logs.patch @@ -0,0 +1,333 @@ +From 083c3a49ad41bd17387c8ae661c23b44d4b845c6 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 30 May 2017 14:43:25 -0500 +Subject: [PATCH] Log: pengine,libpe_status: revisit fencing messages + +--- + lib/pengine/unpack.c | 72 ++++++++++++++++++++++++++++++++-------------------- + pengine/allocate.c | 65 ++++++++++++++++++++++++++--------------------- + 2 files changed, 81 insertions(+), 56 deletions(-) + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 377100c..21eca90 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -63,6 +63,13 @@ is_dangling_container_remote_node(node_t *node) + } + + ++/*! ++ * \brief Schedule a fence action for a node ++ * ++ * \param[in,out] data_set Current working set of cluster ++ * \param[in,out] node Node to fence ++ * \param[in] reason Text description of why fencing is needed ++ */ + void + pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + { +@@ -74,11 +81,13 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + + if (is_set(rsc->flags, pe_rsc_failed) == FALSE) { + if (!is_set(rsc->flags, pe_rsc_managed)) { +- crm_notice("Not fencing node %s due to '%s': container %s is" +- " unmanaged" +- "%s", node->details->uname, reason, rsc->id); ++ crm_notice("Not fencing guest node %s " ++ "(otherwise would because %s): " ++ "its guest resource %s is unmanaged", ++ node->details->uname, reason, rsc->id); + } else { +- crm_warn("Remote node %s will be fenced due to '%s' by recovering %s", ++ crm_warn("Guest node %s will be fenced " ++ "(by recovering its guest resource %s): %s", + node->details->uname, rsc->id, reason); + + /* We don't mark the node as unclean because that would prevent the +@@ -91,8 +100,9 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + } + + } else if (is_dangling_container_remote_node(node)) { +- crm_info("Cleaning up dangling connection resource for guest node %s due to '%s'" +- " (fencing is already done, guest resource no longer exists)", ++ crm_info("Cleaning up dangling connection for guest node %s: " ++ "fencing was already done because %s, " ++ "and guest resource no longer exists", + node->details->uname, reason); + set_bit(node->details->remote_rsc->flags, pe_rsc_failed); + +@@ -100,31 +110,29 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + resource_t *rsc = node->details->remote_rsc; + + if (rsc && (!is_set(rsc->flags, pe_rsc_managed))) { +- crm_notice("Not fencing node %s due to '%s': connection is unmanaged", ++ crm_notice("Not fencing remote node %s " ++ "(otherwise would because %s): connection is unmanaged", + node->details->uname, reason); + } else if(node->details->remote_requires_reset == FALSE) { + node->details->remote_requires_reset = TRUE; +- if (pe_can_fence(data_set, node)) { +- crm_warn("Remote node %s will be fenced due to %s", node->details->uname, reason); +- } else { +- crm_warn("Remote node %s is unclean due to %s", node->details->uname, reason); +- } ++ crm_warn("Remote node %s %s: %s", ++ node->details->uname, ++ pe_can_fence(data_set, node)? "will be fenced" : "is unclean", ++ reason); + } + node->details->unclean = TRUE; + + } else if (node->details->unclean) { +- if (pe_can_fence(data_set, node)) { +- crm_trace("Node %s would also be fenced due to '%s'", node->details->uname, reason); +- } else { +- crm_trace("Node %s is also unclean due to '%s'", node->details->uname, reason); +- } +- +- } else if (pe_can_fence(data_set, node)) { +- crm_warn("Node %s will be fenced due to %s", node->details->uname, reason); +- node->details->unclean = TRUE; ++ crm_trace("Cluster node %s %s because %s", ++ node->details->uname, ++ pe_can_fence(data_set, node)? "would also be fenced" : "also is unclean", ++ reason); + + } else { +- crm_warn("Node %s is unclean due to %s", node->details->uname, reason); ++ crm_warn("Cluster node %s %s: %s", ++ node->details->uname, ++ pe_can_fence(data_set, node)? "will be fenced" : "is unclean", ++ reason); + node->details->unclean = TRUE; + } + } +@@ -1878,6 +1886,8 @@ process_rsc_state(resource_t * rsc, node_t * node, + xmlNode * migrate_op, pe_working_set_t * data_set) + { + node_t *tmpnode = NULL; ++ char *reason = NULL; ++ + CRM_ASSERT(rsc); + pe_rsc_trace(rsc, "Resource %s is %s on %s: on_fail=%s", + rsc->id, role2text(rsc->role), node->details->uname, fail2text(on_fail)); +@@ -1907,7 +1917,6 @@ process_rsc_state(resource_t * rsc, node_t * node, + && node->details->maintenance == FALSE + && is_set(rsc->flags, pe_rsc_managed)) { + +- char *reason = NULL; + gboolean should_fence = FALSE; + + /* If this is a guest node, fence it (regardless of whether fencing is +@@ -1922,14 +1931,19 @@ process_rsc_state(resource_t * rsc, node_t * node, + should_fence = TRUE; + + } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) { +- if (is_baremetal_remote_node(node) && node->details->remote_rsc && is_not_set(node->details->remote_rsc->flags, pe_rsc_failed)) { ++ if (is_baremetal_remote_node(node) && node->details->remote_rsc ++ && is_not_set(node->details->remote_rsc->flags, pe_rsc_failed)) { ++ + /* setting unseen = true means that fencing of the remote node will + * only occur if the connection resource is not going to start somewhere. + * This allows connection resources on a failed cluster-node to move to + * another node without requiring the baremetal remote nodes to be fenced + * as well. */ + node->details->unseen = TRUE; +- reason = crm_strdup_printf("%s is active there. Fencing will be revoked if remote-node connection can be re-established on another cluster-node.", rsc->id); ++ reason = crm_strdup_printf("%s is active there (fencing will be" ++ " revoked if remote connection can " ++ "be re-established elsewhere)", ++ rsc->id); + } + should_fence = TRUE; + } +@@ -1959,7 +1973,9 @@ process_rsc_state(resource_t * rsc, node_t * node, + /* treat it as if it is still running + * but also mark the node as unclean + */ +- pe_fence_node(data_set, node, "resource failure(s)"); ++ reason = crm_strdup_printf("%s failed there", rsc->id); ++ pe_fence_node(data_set, node, reason); ++ free(reason); + break; + + case action_fail_standby: +@@ -2002,6 +2018,7 @@ process_rsc_state(resource_t * rsc, node_t * node, + stop_action(rsc, node, FALSE); + } + break; ++ + case action_fail_reset_remote: + set_bit(rsc->flags, pe_rsc_failed); + if (is_set(data_set->flags, pe_flag_stonith_enabled)) { +@@ -2015,7 +2032,8 @@ process_rsc_state(resource_t * rsc, node_t * node, + + /* connection resource to baremetal resource failed in a way that + * should result in fencing the remote-node. */ +- pe_fence_node(data_set, tmpnode, "of connection failure(s)"); ++ pe_fence_node(data_set, tmpnode, ++ "remote connection is unrecoverable"); + } + } + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 0020af6..f2987cc 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -467,7 +467,7 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki + set_bit(action_clear->flags, pe_action_runnable); + + crm_notice("Clearing failure of %s on %s " +- "action definition changed " CRM_XS " %s", ++ "because action definition changed " CRM_XS " %s", + rsc->id, node->details->uname, action_clear->uuid); + } + } +@@ -1789,7 +1789,6 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set) + + CRM_ASSERT(action->node); + CRM_ASSERT(is_remote_node(action->node)); +- CRM_ASSERT(action->node->details->remote_rsc); + + remote_rsc = action->node->details->remote_rsc; + CRM_ASSERT(remote_rsc); +@@ -1801,7 +1800,13 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set) + pe_fence_node(data_set, action->node, "container failed"); + } + +- crm_trace("%s %s %s %s %d", action->uuid, action->task, remote_rsc->id, container->id, is_set(container->flags, pe_rsc_failed)); ++ crm_trace("Order %s action %s relative to %s%s for %s%s", ++ action->task, action->uuid, ++ is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "", ++ remote_rsc->id, ++ is_set(container->flags, pe_rsc_failed)? "failed " : "", ++ container->id); ++ + switch (task) { + case start_rsc: + case action_promote: +@@ -1874,6 +1879,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + node_t *cluster_node = NULL; + enum action_tasks task = text2task(action->task); + enum remote_connection_state state = remote_state_unknown; ++ enum pe_ordering order_opts = pe_order_none; + + if (action->rsc == NULL) { + return; +@@ -1881,7 +1887,6 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + + CRM_ASSERT(action->node); + CRM_ASSERT(is_remote_node(action->node)); +- CRM_ASSERT(action->node->details->remote_rsc); + + remote_rsc = action->node->details->remote_rsc; + CRM_ASSERT(remote_rsc); +@@ -1895,7 +1900,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + * on that remote node until after it starts elsewhere. + */ + if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) { +- /* There is no-where left to run the connection resource ++ /* There is nowhere left to run the connection resource, + * and the resource is in a failed state (either directly + * or because it is located on a failed node). + * +@@ -1903,8 +1908,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + * or if there are resources in an unknown state (probe), we + * must assume the worst and fence it. + */ +- +- if(is_set(action->node->details->remote_rsc->flags, pe_rsc_failed)) { ++ if (is_set(remote_rsc->flags, pe_rsc_failed)) { + state = remote_state_failed; + } else if(cluster_node && cluster_node->details->unclean) { + state = remote_state_failed; +@@ -1934,22 +1938,31 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + state = remote_state_alive; + } + +- crm_trace("%s %s %s %d %d", action->uuid, action->task, action->node->details->uname, state, is_set(remote_rsc->flags, pe_rsc_failed)); ++ crm_trace("Order %s action %s relative to %s%s (state %d)", ++ action->task, action->uuid, ++ is_set(remote_rsc->flags, pe_rsc_failed)? "failed " : "", ++ remote_rsc->id, state); + switch (task) { + case start_rsc: + case action_promote: +- if(state == remote_state_failed) { +- /* Wait for the connection resource to be up and force recovery */ +- custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, +- action->rsc, NULL, action, +- pe_order_preserve | pe_order_implies_then | pe_order_runnable_left, data_set); +- } else { +- /* Ensure the connection resource is up and assume everything is as we left it */ +- custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, +- action->rsc, NULL, action, +- pe_order_preserve | pe_order_runnable_left, data_set); ++ /* This as an internally generated constraint exempt from ++ * user constraint prohibitions, and this action isn't runnable ++ * if the connection start isn't runnable. ++ */ ++ order_opts = pe_order_preserve | pe_order_runnable_left; ++ ++ if (state == remote_state_failed) { ++ /* Force recovery, by making this action required */ ++ order_opts |= pe_order_implies_then; + } ++ ++ /* Ensure connection is up before running this action */ ++ custom_action_order(remote_rsc, ++ generate_op_key(remote_rsc->id, RSC_START, 0), ++ NULL, action->rsc, NULL, action, order_opts, ++ data_set); + break; ++ + case stop_rsc: + /* Handle special case with remote node where stop actions need to be + * ordered after the connection resource starts somewhere else. +@@ -1975,22 +1988,19 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + pe_order_preserve | pe_order_implies_first, data_set); + } + break; +- case action_demote: + +- /* If the connection is being torn down, we don't want +- * to build a constraint between a resource's demotion and +- * the connection resource starting... because the connection +- * resource can not start. The connection might already be up, +- * but the "start" action would not be allowed, which in turn would +- * block the demotion of any resources living in the node. ++ case action_demote: ++ /* Only order this demote relative to the connection start if the ++ * connection isn't being torn down. Otherwise, the demote would be ++ * blocked because the connection start would not be allowed. + */ +- + if(state == remote_state_resting || state == remote_state_unknown) { + custom_action_order(remote_rsc, generate_op_key(remote_rsc->id, RSC_START, 0), NULL, + action->rsc, NULL, action, + pe_order_preserve, data_set); + } /* Otherwise we can rely on the stop ordering */ + break; ++ + default: + /* Wait for the connection resource to be up */ + if (is_recurring_action(action)) { +@@ -2261,15 +2271,12 @@ stage7(pe_working_set_t * data_set) + order_probes(data_set); + + crm_trace("Updating %d actions", g_list_length(data_set->actions)); +- + for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { + action_t *action = (action_t *) gIter->data; + + update_action(action); + } + +- crm_trace("Processing reloads"); +- + LogNodeActions(data_set, FALSE); + for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) { + resource_t *rsc = (resource_t *) gIter->data; +-- +1.8.3.1 + diff --git a/SOURCES/082-unrecoverable-remotes.patch b/SOURCES/082-unrecoverable-remotes.patch new file mode 100644 index 0000000..8fd52ac --- /dev/null +++ b/SOURCES/082-unrecoverable-remotes.patch @@ -0,0 +1,225 @@ +From c97ac6cda7f148dfdf09b8b4f4ce9762d9c59bcd Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 4 Jul 2017 14:16:52 +1000 +Subject: [PATCH] Fix: PE: Ensure unrecoverable remote nodes are fenced even if + no resources can run on them + +--- + pengine/allocate.c | 106 ++++++++++++++++++++++++++++++++--------------------- + 1 file changed, 65 insertions(+), 41 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 3a95be6..3a883ad 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -39,6 +39,16 @@ void migrate_reload_madness(pe_working_set_t * data_set); + extern void ReloadRsc(resource_t * rsc, node_t *node, pe_working_set_t * data_set); + extern gboolean DeleteRsc(resource_t * rsc, node_t * node, gboolean optional, pe_working_set_t * data_set); + static void apply_remote_node_ordering(pe_working_set_t *data_set); ++static enum remote_connection_state get_remote_node_state(pe_node_t *node); ++enum remote_connection_state ++{ ++ remote_state_unknown = 0, ++ remote_state_alive = 1, ++ remote_state_resting = 2, ++ remote_state_failed = 3, ++ remote_state_stopped = 4 ++}; ++ + + resource_alloc_functions_t resource_class_alloc_functions[] = { + { +@@ -886,21 +896,25 @@ probe_resources(pe_working_set_t * data_set) + { + action_t *probe_node_complete = NULL; + +- GListPtr gIter = NULL; +- GListPtr gIter2 = NULL; +- +- for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { ++ for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + const char *probed = g_hash_table_lookup(node->details->attrs, CRM_OP_PROBED); + +- if (node->details->online == FALSE) { ++ if (is_container_remote_node(node)) { ++ /* TODO enable guest node probes once ordered probing is implemented */ + continue; + +- } else if (node->details->unclean) { ++ } else if (node->details->online == FALSE && node->details->remote_rsc) { ++ enum remote_connection_state state = get_remote_node_state(node); ++ if(state == remote_state_failed) { ++ pe_fence_node(data_set, node, "the connection is unrecoverable"); ++ } + continue; + +- } else if (is_container_remote_node(node)) { +- /* TODO enable guest node probes once ordered probing is implemented */ ++ } else if(node->details->online == FALSE) { ++ continue; ++ ++ } else if (node->details->unclean) { + continue; + + } else if (node->details->rsc_discovery_enabled == FALSE) { +@@ -916,7 +930,7 @@ probe_resources(pe_working_set_t * data_set) + continue; + } + +- for (gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) { ++ for (GListPtr gIter2 = data_set->resources; gIter2 != NULL; gIter2 = gIter2->next) { + resource_t *rsc = (resource_t *) gIter2->data; + + rsc->cmds->create_probe(rsc, node, probe_node_complete, FALSE, data_set); +@@ -1749,15 +1763,6 @@ rsc_order_first(resource_t * lh_rsc, order_constraint_t * order, pe_working_set_ + extern gboolean update_action(action_t * action); + extern void update_colo_start_chain(action_t * action); + +-enum remote_connection_state +-{ +- remote_state_unknown = 0, +- remote_state_alive = 1, +- remote_state_resting = 2, +- remote_state_failed = 3, +- remote_state_stopped = 4 +-}; +- + static int + is_recurring_action(action_t *action) + { +@@ -1874,29 +1879,24 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set) + } + } + +-static void +-apply_remote_ordering(action_t *action, pe_working_set_t *data_set) ++static enum remote_connection_state ++get_remote_node_state(pe_node_t *node) + { + resource_t *remote_rsc = NULL; + node_t *cluster_node = NULL; +- enum action_tasks task = text2task(action->task); +- enum remote_connection_state state = remote_state_unknown; +- enum pe_ordering order_opts = pe_order_none; + +- if (action->rsc == NULL) { +- return; ++ if(node == NULL) { ++ return remote_state_unknown; + } + +- CRM_ASSERT(action->node); +- CRM_ASSERT(is_remote_node(action->node)); +- +- remote_rsc = action->node->details->remote_rsc; ++ remote_rsc = node->details->remote_rsc; + CRM_ASSERT(remote_rsc); + + if(remote_rsc->running_on) { + cluster_node = remote_rsc->running_on->data; + } + ++ + /* If the cluster node the remote connection resource resides on + * is unclean or went offline, we can't process any operations + * on that remote node until after it starts elsewhere. +@@ -1911,21 +1911,21 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + * must assume the worst and fence it. + */ + if (is_set(remote_rsc->flags, pe_rsc_failed)) { +- state = remote_state_failed; ++ return remote_state_failed; + } else if(cluster_node && cluster_node->details->unclean) { +- state = remote_state_failed; ++ return remote_state_failed; + } else { +- state = remote_state_stopped; ++ return remote_state_stopped; + } + + } else if (cluster_node == NULL) { + /* Connection is recoverable but not currently running anywhere, see if we can recover it first */ +- state = remote_state_unknown; ++ return remote_state_unknown; + + } else if(cluster_node->details->unclean == TRUE + || cluster_node->details->online == FALSE) { + /* Connection is running on a dead node, see if we can recover it first */ +- state = remote_state_resting; ++ return remote_state_resting; + + } else if (g_list_length(remote_rsc->running_on) > 1 + && remote_rsc->partial_migration_source +@@ -1934,10 +1934,34 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + * wait until after the resource migrates before performing + * any actions. + */ +- state = remote_state_resting; ++ return remote_state_resting; + +- } else { +- state = remote_state_alive; ++ } ++ return remote_state_alive; ++} ++ ++static void ++apply_remote_ordering(action_t *action, pe_working_set_t *data_set) ++{ ++ resource_t *remote_rsc = NULL; ++ node_t *cluster_node = NULL; ++ enum action_tasks task = text2task(action->task); ++ enum remote_connection_state state = get_remote_node_state(action->node); ++ ++ enum pe_ordering order_opts = pe_order_none; ++ ++ if (action->rsc == NULL) { ++ return; ++ } ++ ++ CRM_ASSERT(action->node); ++ CRM_ASSERT(is_remote_node(action->node)); ++ ++ remote_rsc = action->node->details->remote_rsc; ++ CRM_ASSERT(remote_rsc); ++ ++ if(remote_rsc->running_on) { ++ cluster_node = remote_rsc->running_on->data; + } + + crm_trace("Order %s action %s relative to %s%s (state %d)", +@@ -2049,13 +2073,11 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set) + static void + apply_remote_node_ordering(pe_working_set_t *data_set) + { +- GListPtr gIter = data_set->actions; +- + if (is_set(data_set->flags, pe_flag_have_remote_nodes) == FALSE) { + return; + } + +- for (; gIter != NULL; gIter = gIter->next) { ++ for (GListPtr gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { + action_t *action = (action_t *) gIter->data; + + if (action->rsc == NULL) { +@@ -2092,12 +2114,14 @@ apply_remote_node_ordering(pe_working_set_t *data_set) + is_remote_node(action->node) == FALSE || + action->node->details->remote_rsc == NULL || + is_set(action->flags, pe_action_pseudo)) { +- crm_trace("Nothing required for %s", action->uuid); ++ crm_trace("Nothing required for %s on %s", action->uuid, action->node?action->node->details->uname:"NA"); + + } else if(action->node->details->remote_rsc->container) { ++ crm_trace("Container ordering for %s", action->uuid); + apply_container_ordering(action, data_set); + + } else { ++ crm_trace("Remote ordering for %s", action->uuid); + apply_remote_ordering(action, data_set); + } + } +-- +1.8.3.1 + diff --git a/SOURCES/083-unrecoverable-remotes-test.patch b/SOURCES/083-unrecoverable-remotes-test.patch new file mode 100644 index 0000000..893e24a --- /dev/null +++ b/SOURCES/083-unrecoverable-remotes-test.patch @@ -0,0 +1,2458 @@ +From 67788d4c70237ffe1f44e315491a841c7434718c Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 4 Jul 2017 14:17:00 +1000 +Subject: [PATCH] Test: PE: Ensure unrecoverable remote nodes are fenced even + if no resources can run on them + +--- + pengine/regression.sh | 1 + + pengine/test10/remote-fence-unclean-3.dot | 27 + + pengine/test10/remote-fence-unclean-3.exp | 163 ++++ + pengine/test10/remote-fence-unclean-3.scores | 1298 +++++++++++++++++++++++++ + pengine/test10/remote-fence-unclean-3.summary | 92 ++ + pengine/test10/remote-fence-unclean-3.xml | 813 ++++++++++++++++ + 6 files changed, 2394 insertions(+) + create mode 100644 pengine/test10/remote-fence-unclean-3.dot + create mode 100644 pengine/test10/remote-fence-unclean-3.exp + create mode 100644 pengine/test10/remote-fence-unclean-3.scores + create mode 100644 pengine/test10/remote-fence-unclean-3.summary + create mode 100644 pengine/test10/remote-fence-unclean-3.xml + +diff --git a/pengine/regression.sh b/pengine/regression.sh +index 5755660..d89d93c 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -834,6 +834,7 @@ do_test remote-startup-probes "Baremetal remote-node startup probes" + do_test remote-startup "Startup a newly discovered remote-nodes with no status." + do_test remote-fence-unclean "Fence unclean baremetal remote-node" + do_test remote-fence-unclean2 "Fence baremetal remote-node after cluster node fails and connection can not be recovered" ++do_test remote-fence-unclean-3 "Probe failed remote nodes (triggers fencing)" + do_test remote-move "Move remote-node connection resource" + do_test remote-disable "Disable a baremetal remote-node" + do_test remote-probe-disable "Probe then stop a baremetal remote-node" +diff --git a/pengine/test10/remote-fence-unclean-3.dot b/pengine/test10/remote-fence-unclean-3.dot +new file mode 100644 +index 0000000..11d1208 +--- /dev/null ++++ b/pengine/test10/remote-fence-unclean-3.dot +@@ -0,0 +1,27 @@ ++digraph "g" { ++"all_stopped" -> "fence1_start_0 overcloud-controller-1" [ style = bold] ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"fence1_monitor_0 overcloud-controller-0" -> "fence1_start_0 overcloud-controller-1" [ style = bold] ++"fence1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"fence1_monitor_0 overcloud-controller-1" -> "fence1_start_0 overcloud-controller-1" [ style = bold] ++"fence1_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"fence1_monitor_0 overcloud-controller-2" -> "fence1_start_0 overcloud-controller-1" [ style = bold] ++"fence1_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"fence1_monitor_60000 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"fence1_start_0 overcloud-controller-1" -> "fence1_monitor_60000 overcloud-controller-1" [ style = bold] ++"fence1_start_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"foo_monitor_0 overcloud-controller-0" -> "foo_start_0 overcloud-controller-0" [ style = bold] ++"foo_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"foo_monitor_0 overcloud-controller-1" -> "foo_start_0 overcloud-controller-0" [ style = bold] ++"foo_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"foo_monitor_0 overcloud-controller-2" -> "foo_start_0 overcloud-controller-0" [ style = bold] ++"foo_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"foo_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"overcloud-novacompute-0_stop_0 overcloud-controller-0" -> "all_stopped" [ style = bold] ++"overcloud-novacompute-0_stop_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' overcloud-novacompute-0" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' overcloud-novacompute-0" [ style=bold color="green" fontcolor="black"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "foo_start_0 overcloud-controller-0" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] ++} +diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp +new file mode 100644 +index 0000000..14e1b0b +--- /dev/null ++++ b/pengine/test10/remote-fence-unclean-3.exp +@@ -0,0 +1,163 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/remote-fence-unclean-3.scores b/pengine/test10/remote-fence-unclean-3.scores +new file mode 100644 +index 0000000..0284218 +--- /dev/null ++++ b/pengine/test10/remote-fence-unclean-3.scores +@@ -0,0 +1,1298 @@ ++Allocation scores: ++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 ++clone_color: galera-bundle-master allocation score on galera-bundle-1: 0 ++clone_color: galera-bundle-master allocation score on galera-bundle-2: 0 ++clone_color: galera-bundle-master allocation score on overcloud-controller-0: -INFINITY ++clone_color: galera-bundle-master allocation score on overcloud-controller-1: -INFINITY ++clone_color: galera-bundle-master allocation score on overcloud-controller-2: -INFINITY ++clone_color: galera-bundle-master allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY ++clone_color: galera:0 allocation score on galera-bundle-1: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-2: -INFINITY ++clone_color: galera:0 allocation score on overcloud-controller-0: -INFINITY ++clone_color: galera:0 allocation score on overcloud-controller-1: -INFINITY ++clone_color: galera:0 allocation score on overcloud-controller-2: -INFINITY ++clone_color: galera:0 allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: galera:1 allocation score on galera-bundle-0: -INFINITY ++clone_color: galera:1 allocation score on galera-bundle-1: INFINITY ++clone_color: galera:1 allocation score on galera-bundle-2: -INFINITY ++clone_color: galera:1 allocation score on overcloud-controller-0: -INFINITY ++clone_color: galera:1 allocation score on overcloud-controller-1: -INFINITY ++clone_color: galera:1 allocation score on overcloud-controller-2: -INFINITY ++clone_color: galera:1 allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: galera:2 allocation score on galera-bundle-0: -INFINITY ++clone_color: galera:2 allocation score on galera-bundle-1: -INFINITY ++clone_color: galera:2 allocation score on galera-bundle-2: INFINITY ++clone_color: galera:2 allocation score on overcloud-controller-0: -INFINITY ++clone_color: galera:2 allocation score on overcloud-controller-1: -INFINITY ++clone_color: galera:2 allocation score on overcloud-controller-2: -INFINITY ++clone_color: galera:2 allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-controller-0: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-controller-1: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-controller-2: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0 ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0 ++clone_color: rabbitmq:0 allocation score on overcloud-controller-0: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-controller-1: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-controller-2: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-controller-0: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-controller-1: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-controller-2: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-controller-0: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-controller-1: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-controller-2: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY ++clone_color: redis-bundle-master allocation score on overcloud-controller-0: -INFINITY ++clone_color: redis-bundle-master allocation score on overcloud-controller-1: -INFINITY ++clone_color: redis-bundle-master allocation score on overcloud-controller-2: -INFINITY ++clone_color: redis-bundle-master allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0 ++clone_color: redis-bundle-master allocation score on redis-bundle-1: 0 ++clone_color: redis-bundle-master allocation score on redis-bundle-2: 0 ++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-1: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-2: -INFINITY ++clone_color: redis:0 allocation score on overcloud-controller-0: -INFINITY ++clone_color: redis:0 allocation score on overcloud-controller-1: -INFINITY ++clone_color: redis:0 allocation score on overcloud-controller-2: -INFINITY ++clone_color: redis:0 allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY ++clone_color: redis:0 allocation score on redis-bundle-1: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-2: -INFINITY ++clone_color: redis:1 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:1 allocation score on galera-bundle-1: -INFINITY ++clone_color: redis:1 allocation score on galera-bundle-2: -INFINITY ++clone_color: redis:1 allocation score on overcloud-controller-0: -INFINITY ++clone_color: redis:1 allocation score on overcloud-controller-1: -INFINITY ++clone_color: redis:1 allocation score on overcloud-controller-2: -INFINITY ++clone_color: redis:1 allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: redis:1 allocation score on redis-bundle-0: -INFINITY ++clone_color: redis:1 allocation score on redis-bundle-1: INFINITY ++clone_color: redis:1 allocation score on redis-bundle-2: -INFINITY ++clone_color: redis:2 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:2 allocation score on galera-bundle-1: -INFINITY ++clone_color: redis:2 allocation score on galera-bundle-2: -INFINITY ++clone_color: redis:2 allocation score on overcloud-controller-0: -INFINITY ++clone_color: redis:2 allocation score on overcloud-controller-1: -INFINITY ++clone_color: redis:2 allocation score on overcloud-controller-2: -INFINITY ++clone_color: redis:2 allocation score on overcloud-novacompute-0: -INFINITY ++clone_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: redis:2 allocation score on redis-bundle-0: -INFINITY ++clone_color: redis:2 allocation score on redis-bundle-1: -INFINITY ++clone_color: redis:2 allocation score on redis-bundle-2: INFINITY ++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle allocation score on overcloud-controller-0: 0 ++container_color: galera-bundle allocation score on overcloud-controller-1: 0 ++container_color: galera-bundle allocation score on overcloud-controller-2: 0 ++container_color: galera-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-0 allocation score on overcloud-controller-0: INFINITY ++container_color: galera-bundle-0 allocation score on overcloud-controller-1: 0 ++container_color: galera-bundle-0 allocation score on overcloud-controller-2: 0 ++container_color: galera-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-1 allocation score on overcloud-controller-0: 0 ++container_color: galera-bundle-1 allocation score on overcloud-controller-1: INFINITY ++container_color: galera-bundle-1 allocation score on overcloud-controller-2: 0 ++container_color: galera-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-2 allocation score on overcloud-controller-0: 0 ++container_color: galera-bundle-2 allocation score on overcloud-controller-1: 0 ++container_color: galera-bundle-2 allocation score on overcloud-controller-2: INFINITY ++container_color: galera-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: galera-bundle-docker-0 allocation score on overcloud-controller-1: 0 ++container_color: galera-bundle-docker-0 allocation score on overcloud-controller-2: 0 ++container_color: galera-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on overcloud-controller-0: 0 ++container_color: galera-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: galera-bundle-docker-1 allocation score on overcloud-controller-2: 0 ++container_color: galera-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on overcloud-controller-0: 0 ++container_color: galera-bundle-docker-2 allocation score on overcloud-controller-1: 0 ++container_color: galera-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: galera-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-master allocation score on overcloud-controller-0: 0 ++container_color: galera-bundle-master allocation score on overcloud-controller-1: 0 ++container_color: galera-bundle-master allocation score on overcloud-controller-2: 0 ++container_color: galera-bundle-master allocation score on overcloud-novacompute-0: 0 ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:0 allocation score on overcloud-controller-0: 0 ++container_color: galera:0 allocation score on overcloud-controller-1: 0 ++container_color: galera:0 allocation score on overcloud-controller-2: 0 ++container_color: galera:0 allocation score on overcloud-novacompute-0: 0 ++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera:1 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:1 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:1 allocation score on overcloud-controller-0: 0 ++container_color: galera:1 allocation score on overcloud-controller-1: 0 ++container_color: galera:1 allocation score on overcloud-controller-2: 0 ++container_color: galera:1 allocation score on overcloud-novacompute-0: 0 ++container_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera:2 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:2 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:2 allocation score on overcloud-controller-0: 0 ++container_color: galera:2 allocation score on overcloud-controller-1: 0 ++container_color: galera:2 allocation score on overcloud-controller-2: 0 ++container_color: galera:2 allocation score on overcloud-novacompute-0: 0 ++container_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-0: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-1: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 ++container_color: haproxy-bundle allocation score on overcloud-controller-2: 0 ++container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: 0 ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: 0 ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: 0 ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: 0 ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: 0 ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: 0 ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: openstack-cinder-backup allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-backup allocation score on galera-bundle-1: -INFINITY ++container_color: openstack-cinder-backup allocation score on galera-bundle-2: -INFINITY ++container_color: openstack-cinder-backup allocation score on overcloud-controller-0: 0 ++container_color: openstack-cinder-backup allocation score on overcloud-controller-1: 0 ++container_color: openstack-cinder-backup allocation score on overcloud-controller-2: 0 ++container_color: openstack-cinder-backup allocation score on overcloud-novacompute-0: -INFINITY ++container_color: openstack-cinder-backup allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-backup allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: openstack-cinder-backup allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: openstack-cinder-backup allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-backup allocation score on redis-bundle-1: -INFINITY ++container_color: openstack-cinder-backup allocation score on redis-bundle-2: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-0: 0 ++container_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-1: INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-2: 0 ++container_color: openstack-cinder-backup-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on galera-bundle-1: -INFINITY ++container_color: openstack-cinder-volume allocation score on galera-bundle-2: -INFINITY ++container_color: openstack-cinder-volume allocation score on overcloud-controller-0: 0 ++container_color: openstack-cinder-volume allocation score on overcloud-controller-1: 0 ++container_color: openstack-cinder-volume allocation score on overcloud-controller-2: 0 ++container_color: openstack-cinder-volume allocation score on overcloud-novacompute-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: openstack-cinder-volume allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume allocation score on redis-bundle-1: -INFINITY ++container_color: openstack-cinder-volume allocation score on redis-bundle-2: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-1: 0 ++container_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-2: 0 ++container_color: openstack-cinder-volume-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: rabbitmq-bundle allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on overcloud-controller-0: INFINITY ++container_color: rabbitmq-bundle-0 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle-0 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle-1 allocation score on overcloud-controller-1: INFINITY ++container_color: rabbitmq-bundle-1 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle-2 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle-2 allocation score on overcloud-controller-2: INFINITY ++container_color: rabbitmq-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-clone allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-novacompute-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0 ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq:0 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq:0 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq:0 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq:0 allocation score on overcloud-novacompute-0: 0 ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: 0 ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: 0 ++container_color: rabbitmq:1 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq:1 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq:1 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq:1 allocation score on overcloud-novacompute-0: 0 ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: 0 ++container_color: rabbitmq:2 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq:2 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq:2 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq:2 allocation score on overcloud-novacompute-0: 0 ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: 0 ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle allocation score on overcloud-controller-0: 0 ++container_color: redis-bundle allocation score on overcloud-controller-1: 0 ++container_color: redis-bundle allocation score on overcloud-controller-2: 0 ++container_color: redis-bundle allocation score on overcloud-novacompute-0: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-0 allocation score on overcloud-controller-0: INFINITY ++container_color: redis-bundle-0 allocation score on overcloud-controller-1: 0 ++container_color: redis-bundle-0 allocation score on overcloud-controller-2: 0 ++container_color: redis-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-1 allocation score on overcloud-controller-0: 0 ++container_color: redis-bundle-1 allocation score on overcloud-controller-1: INFINITY ++container_color: redis-bundle-1 allocation score on overcloud-controller-2: 0 ++container_color: redis-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-2 allocation score on overcloud-controller-0: 0 ++container_color: redis-bundle-2 allocation score on overcloud-controller-1: 0 ++container_color: redis-bundle-2 allocation score on overcloud-controller-2: INFINITY ++container_color: redis-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++container_color: redis-bundle-docker-0 allocation score on overcloud-controller-1: 0 ++container_color: redis-bundle-docker-0 allocation score on overcloud-controller-2: 0 ++container_color: redis-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on overcloud-controller-0: 0 ++container_color: redis-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++container_color: redis-bundle-docker-1 allocation score on overcloud-controller-2: 0 ++container_color: redis-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on overcloud-controller-0: 0 ++container_color: redis-bundle-docker-2 allocation score on overcloud-controller-1: 0 ++container_color: redis-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++container_color: redis-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-master allocation score on overcloud-controller-0: 0 ++container_color: redis-bundle-master allocation score on overcloud-controller-1: 0 ++container_color: redis-bundle-master allocation score on overcloud-controller-2: 0 ++container_color: redis-bundle-master allocation score on overcloud-novacompute-0: 0 ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-2: -INFINITY ++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:0 allocation score on galera-bundle-1: -INFINITY ++container_color: redis:0 allocation score on galera-bundle-2: -INFINITY ++container_color: redis:0 allocation score on overcloud-controller-0: 0 ++container_color: redis:0 allocation score on overcloud-controller-1: 0 ++container_color: redis:0 allocation score on overcloud-controller-2: 0 ++container_color: redis:0 allocation score on overcloud-novacompute-0: 0 ++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-2: -INFINITY ++container_color: redis:1 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:1 allocation score on galera-bundle-1: -INFINITY ++container_color: redis:1 allocation score on galera-bundle-2: -INFINITY ++container_color: redis:1 allocation score on overcloud-controller-0: 0 ++container_color: redis:1 allocation score on overcloud-controller-1: 0 ++container_color: redis:1 allocation score on overcloud-controller-2: 0 ++container_color: redis:1 allocation score on overcloud-novacompute-0: 0 ++container_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-2: -INFINITY ++container_color: redis:2 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:2 allocation score on galera-bundle-1: -INFINITY ++container_color: redis:2 allocation score on galera-bundle-2: -INFINITY ++container_color: redis:2 allocation score on overcloud-controller-0: 0 ++container_color: redis:2 allocation score on overcloud-controller-1: 0 ++container_color: redis:2 allocation score on overcloud-controller-2: 0 ++container_color: redis:2 allocation score on overcloud-novacompute-0: 0 ++container_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-2: -INFINITY ++galera:0 promotion score on galera-bundle-0: 100 ++galera:1 promotion score on galera-bundle-1: 100 ++galera:2 promotion score on galera-bundle-2: 100 ++native_color: fence1 allocation score on overcloud-controller-0: 0 ++native_color: fence1 allocation score on overcloud-controller-1: 0 ++native_color: fence1 allocation score on overcloud-controller-2: 0 ++native_color: fence1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: foo allocation score on overcloud-controller-0: 0 ++native_color: foo allocation score on overcloud-controller-1: 0 ++native_color: foo allocation score on overcloud-controller-2: 0 ++native_color: foo allocation score on overcloud-novacompute-0: 0 ++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-0 allocation score on overcloud-controller-0: INFINITY ++native_color: galera-bundle-0 allocation score on overcloud-controller-1: 0 ++native_color: galera-bundle-0 allocation score on overcloud-controller-2: 0 ++native_color: galera-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-1 allocation score on overcloud-controller-0: 0 ++native_color: galera-bundle-1 allocation score on overcloud-controller-1: INFINITY ++native_color: galera-bundle-1 allocation score on overcloud-controller-2: 0 ++native_color: galera-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-2 allocation score on overcloud-controller-0: 0 ++native_color: galera-bundle-2 allocation score on overcloud-controller-1: 0 ++native_color: galera-bundle-2 allocation score on overcloud-controller-2: INFINITY ++native_color: galera-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++native_color: galera-bundle-docker-0 allocation score on overcloud-controller-1: 0 ++native_color: galera-bundle-docker-0 allocation score on overcloud-controller-2: 0 ++native_color: galera-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-controller-2: 0 ++native_color: galera-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera:0 allocation score on galera-bundle-0: INFINITY ++native_color: galera:0 allocation score on galera-bundle-1: -INFINITY ++native_color: galera:0 allocation score on galera-bundle-2: -INFINITY ++native_color: galera:0 allocation score on overcloud-controller-0: -INFINITY ++native_color: galera:0 allocation score on overcloud-controller-1: -INFINITY ++native_color: galera:0 allocation score on overcloud-controller-2: -INFINITY ++native_color: galera:0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera:1 allocation score on galera-bundle-0: -INFINITY ++native_color: galera:1 allocation score on galera-bundle-1: INFINITY ++native_color: galera:1 allocation score on galera-bundle-2: -INFINITY ++native_color: galera:1 allocation score on overcloud-controller-0: -INFINITY ++native_color: galera:1 allocation score on overcloud-controller-1: -INFINITY ++native_color: galera:1 allocation score on overcloud-controller-2: -INFINITY ++native_color: galera:1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera:2 allocation score on galera-bundle-0: -INFINITY ++native_color: galera:2 allocation score on galera-bundle-1: -INFINITY ++native_color: galera:2 allocation score on galera-bundle-2: INFINITY ++native_color: galera:2 allocation score on overcloud-controller-0: -INFINITY ++native_color: galera:2 allocation score on overcloud-controller-1: -INFINITY ++native_color: galera:2 allocation score on overcloud-controller-2: -INFINITY ++native_color: galera:2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-1: INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on overcloud-controller-2: INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on overcloud-controller-2: INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-10.0.0.7 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-10.0.0.7 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-10.0.0.7 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-10.0.0.7 allocation score on overcloud-controller-0: 0 ++native_color: ip-10.0.0.7 allocation score on overcloud-controller-1: INFINITY ++native_color: ip-10.0.0.7 allocation score on overcloud-controller-2: 0 ++native_color: ip-10.0.0.7 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: ip-10.0.0.7 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-10.0.0.7 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-10.0.0.7 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-10.0.0.7 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-10.0.0.7 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-10.0.0.7 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-172.16.1.9 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-172.16.1.9 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-172.16.1.9 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-172.16.1.9 allocation score on overcloud-controller-0: 0 ++native_color: ip-172.16.1.9 allocation score on overcloud-controller-1: INFINITY ++native_color: ip-172.16.1.9 allocation score on overcloud-controller-2: 0 ++native_color: ip-172.16.1.9 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: ip-172.16.1.9 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-172.16.1.9 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-172.16.1.9 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-172.16.1.9 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-172.16.1.9 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-172.16.1.9 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-172.16.2.4 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-172.16.2.4 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-172.16.2.4 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-172.16.2.4 allocation score on overcloud-controller-0: 0 ++native_color: ip-172.16.2.4 allocation score on overcloud-controller-1: 0 ++native_color: ip-172.16.2.4 allocation score on overcloud-controller-2: INFINITY ++native_color: ip-172.16.2.4 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: ip-172.16.2.4 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-172.16.2.4 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-172.16.2.4 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-172.16.2.4 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-172.16.2.4 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-172.16.2.4 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-172.16.2.8 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-172.16.2.8 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-172.16.2.8 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-172.16.2.8 allocation score on overcloud-controller-0: INFINITY ++native_color: ip-172.16.2.8 allocation score on overcloud-controller-1: 0 ++native_color: ip-172.16.2.8 allocation score on overcloud-controller-2: 0 ++native_color: ip-172.16.2.8 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: ip-172.16.2.8 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-172.16.2.8 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-172.16.2.8 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-172.16.2.8 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-172.16.2.8 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-172.16.2.8 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-172.16.3.9 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-172.16.3.9 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-172.16.3.9 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-172.16.3.9 allocation score on overcloud-controller-0: 0 ++native_color: ip-172.16.3.9 allocation score on overcloud-controller-1: 0 ++native_color: ip-172.16.3.9 allocation score on overcloud-controller-2: INFINITY ++native_color: ip-172.16.3.9 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: ip-172.16.3.9 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-172.16.3.9 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-172.16.3.9 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-172.16.3.9 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-172.16.3.9 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-172.16.3.9 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-192.168.24.9 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.24.9 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-192.168.24.9 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-192.168.24.9 allocation score on overcloud-controller-0: INFINITY ++native_color: ip-192.168.24.9 allocation score on overcloud-controller-1: 0 ++native_color: ip-192.168.24.9 allocation score on overcloud-controller-2: 0 ++native_color: ip-192.168.24.9 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: ip-192.168.24.9 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.24.9 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-192.168.24.9 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-192.168.24.9 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.24.9 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-192.168.24.9 allocation score on redis-bundle-2: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-1: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on galera-bundle-2: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-0: 0 ++native_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-1: INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on overcloud-controller-2: 0 ++native_color: openstack-cinder-backup-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-1: -INFINITY ++native_color: openstack-cinder-backup-docker-0 allocation score on redis-bundle-2: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-1: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on galera-bundle-2: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-0: INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-1: 0 ++native_color: openstack-cinder-volume-docker-0 allocation score on overcloud-controller-2: 0 ++native_color: openstack-cinder-volume-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-1: -INFINITY ++native_color: openstack-cinder-volume-docker-0 allocation score on redis-bundle-2: -INFINITY ++native_color: overcloud-novacompute-0 allocation score on overcloud-controller-0: -INFINITY ++native_color: overcloud-novacompute-0 allocation score on overcloud-controller-1: -INFINITY ++native_color: overcloud-novacompute-0 allocation score on overcloud-controller-2: -INFINITY ++native_color: overcloud-novacompute-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-0: INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-1: 0 ++native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-2: 0 ++native_color: rabbitmq-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-controller-0: 0 ++native_color: rabbitmq-bundle-1 allocation score on overcloud-controller-1: INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-controller-2: 0 ++native_color: rabbitmq-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-controller-0: 0 ++native_color: rabbitmq-bundle-2 allocation score on overcloud-controller-1: 0 ++native_color: rabbitmq-bundle-2 allocation score on overcloud-controller-2: INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-1: 0 ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-2: 0 ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-2: 0 ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-0 allocation score on overcloud-controller-0: INFINITY ++native_color: redis-bundle-0 allocation score on overcloud-controller-1: 0 ++native_color: redis-bundle-0 allocation score on overcloud-controller-2: 0 ++native_color: redis-bundle-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-1 allocation score on overcloud-controller-0: 0 ++native_color: redis-bundle-1 allocation score on overcloud-controller-1: INFINITY ++native_color: redis-bundle-1 allocation score on overcloud-controller-2: 0 ++native_color: redis-bundle-1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-2 allocation score on overcloud-controller-0: 0 ++native_color: redis-bundle-2 allocation score on overcloud-controller-1: 0 ++native_color: redis-bundle-2 allocation score on overcloud-controller-2: INFINITY ++native_color: redis-bundle-2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on overcloud-controller-0: INFINITY ++native_color: redis-bundle-docker-0 allocation score on overcloud-controller-1: 0 ++native_color: redis-bundle-docker-0 allocation score on overcloud-controller-2: 0 ++native_color: redis-bundle-docker-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on overcloud-controller-1: INFINITY ++native_color: redis-bundle-docker-1 allocation score on overcloud-controller-2: 0 ++native_color: redis-bundle-docker-1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on overcloud-controller-2: INFINITY ++native_color: redis-bundle-docker-2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:0 allocation score on galera-bundle-1: -INFINITY ++native_color: redis:0 allocation score on galera-bundle-2: -INFINITY ++native_color: redis:0 allocation score on overcloud-controller-0: -INFINITY ++native_color: redis:0 allocation score on overcloud-controller-1: -INFINITY ++native_color: redis:0 allocation score on overcloud-controller-2: -INFINITY ++native_color: redis:0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-0: INFINITY ++native_color: redis:0 allocation score on redis-bundle-1: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-2: -INFINITY ++native_color: redis:1 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:1 allocation score on galera-bundle-1: -INFINITY ++native_color: redis:1 allocation score on galera-bundle-2: -INFINITY ++native_color: redis:1 allocation score on overcloud-controller-0: -INFINITY ++native_color: redis:1 allocation score on overcloud-controller-1: -INFINITY ++native_color: redis:1 allocation score on overcloud-controller-2: -INFINITY ++native_color: redis:1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis:1 allocation score on redis-bundle-0: -INFINITY ++native_color: redis:1 allocation score on redis-bundle-1: INFINITY ++native_color: redis:1 allocation score on redis-bundle-2: -INFINITY ++native_color: redis:2 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:2 allocation score on galera-bundle-1: -INFINITY ++native_color: redis:2 allocation score on galera-bundle-2: -INFINITY ++native_color: redis:2 allocation score on overcloud-controller-0: -INFINITY ++native_color: redis:2 allocation score on overcloud-controller-1: -INFINITY ++native_color: redis:2 allocation score on overcloud-controller-2: -INFINITY ++native_color: redis:2 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis:2 allocation score on redis-bundle-0: -INFINITY ++native_color: redis:2 allocation score on redis-bundle-1: -INFINITY ++native_color: redis:2 allocation score on redis-bundle-2: INFINITY ++redis:0 promotion score on redis-bundle-0: 1 ++redis:1 promotion score on redis-bundle-1: 1 ++redis:2 promotion score on redis-bundle-2: 1 +diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary +new file mode 100644 +index 0000000..cc60ebe +--- /dev/null ++++ b/pengine/test10/remote-fence-unclean-3.summary +@@ -0,0 +1,92 @@ ++ ++Current cluster status: ++Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] ++RemoteOFFLINE: [ overcloud-novacompute-0 ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] ++ ++ foo (ocf::heartbeat:Dummy): Stopped ++ fence1 (stonith:fence_xvm): Stopped ++ overcloud-novacompute-0 (ocf::pacemaker:remote): FAILED overcloud-controller-0 ++ Docker container set: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-0 ++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-1 ++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-2 ++ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Master overcloud-controller-0 ++ galera-bundle-1 (ocf::heartbeat:galera): Master overcloud-controller-1 ++ galera-bundle-2 (ocf::heartbeat:galera): Master overcloud-controller-2 ++ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Master overcloud-controller-0 ++ redis-bundle-1 (ocf::heartbeat:redis): Slave overcloud-controller-1 ++ redis-bundle-2 (ocf::heartbeat:redis): Slave overcloud-controller-2 ++ ip-192.168.24.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-0 ++ ip-10.0.0.7 (ocf::heartbeat:IPaddr2): Started overcloud-controller-1 ++ ip-172.16.2.4 (ocf::heartbeat:IPaddr2): Started overcloud-controller-2 ++ ip-172.16.2.8 (ocf::heartbeat:IPaddr2): Started overcloud-controller-0 ++ ip-172.16.1.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-1 ++ ip-172.16.3.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-2 ++ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-0 ++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started overcloud-controller-1 ++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started overcloud-controller-2 ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-0 ++ Docker container: openstack-cinder-backup [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-backup:latest] ++ openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1 ++ ++Transition Summary: ++ * Fence overcloud-novacompute-0 ++ * Start foo (overcloud-controller-0) ++ * Start fence1 (overcloud-controller-1) ++ * Stop overcloud-novacompute-0 (overcloud-controller-0) ++ ++Executing cluster transition: ++ * Resource action: foo monitor on overcloud-controller-2 ++ * Resource action: foo monitor on overcloud-controller-1 ++ * Resource action: foo monitor on overcloud-controller-0 ++ * Resource action: fence1 monitor on overcloud-controller-2 ++ * Resource action: fence1 monitor on overcloud-controller-1 ++ * Resource action: fence1 monitor on overcloud-controller-0 ++ * Resource action: overcloud-novacompute-0 stop on overcloud-controller-0 ++ * Fencing overcloud-novacompute-0 (reboot) ++ * Pseudo action: stonith_complete ++ * Pseudo action: all_stopped ++ * Resource action: foo start on overcloud-controller-0 ++ * Resource action: fence1 start on overcloud-controller-1 ++ * Resource action: fence1 monitor=60000 on overcloud-controller-1 ++ ++Revised cluster status: ++Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] ++RemoteOFFLINE: [ overcloud-novacompute-0 ] ++Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] ++ ++ foo (ocf::heartbeat:Dummy): Started overcloud-controller-0 ++ fence1 (stonith:fence_xvm): Started overcloud-controller-1 ++ overcloud-novacompute-0 (ocf::pacemaker:remote): Stopped ++ Docker container set: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-0 ++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-1 ++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-2 ++ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-0 (ocf::heartbeat:galera): Master overcloud-controller-0 ++ galera-bundle-1 (ocf::heartbeat:galera): Master overcloud-controller-1 ++ galera-bundle-2 (ocf::heartbeat:galera): Master overcloud-controller-2 ++ Docker container set: redis-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-redis:latest] ++ redis-bundle-0 (ocf::heartbeat:redis): Master overcloud-controller-0 ++ redis-bundle-1 (ocf::heartbeat:redis): Slave overcloud-controller-1 ++ redis-bundle-2 (ocf::heartbeat:redis): Slave overcloud-controller-2 ++ ip-192.168.24.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-0 ++ ip-10.0.0.7 (ocf::heartbeat:IPaddr2): Started overcloud-controller-1 ++ ip-172.16.2.4 (ocf::heartbeat:IPaddr2): Started overcloud-controller-2 ++ ip-172.16.2.8 (ocf::heartbeat:IPaddr2): Started overcloud-controller-0 ++ ip-172.16.1.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-1 ++ ip-172.16.3.9 (ocf::heartbeat:IPaddr2): Started overcloud-controller-2 ++ Docker container set: haproxy-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-haproxy:latest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-0 ++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started overcloud-controller-1 ++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started overcloud-controller-2 ++ Docker container: openstack-cinder-volume [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-volume:latest] ++ openstack-cinder-volume-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-0 ++ Docker container: openstack-cinder-backup [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-backup:latest] ++ openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1 ++ +diff --git a/pengine/test10/remote-fence-unclean-3.xml b/pengine/test10/remote-fence-unclean-3.xml +new file mode 100644 +index 0000000..8856fc0 +--- /dev/null ++++ b/pengine/test10/remote-fence-unclean-3.xml +@@ -0,0 +1,813 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +-- +1.8.3.1 + diff --git a/SOURCES/084-remote-fence-test.patch b/SOURCES/084-remote-fence-test.patch new file mode 100644 index 0000000..ecba100 --- /dev/null +++ b/SOURCES/084-remote-fence-test.patch @@ -0,0 +1,300 @@ +From 201d058a6df884ddab4d83d482203e0ef7044d21 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 5 Jul 2017 18:03:14 -0500 +Subject: [PATCH] Test: pengine: update remote-fence-unclean-3 regression test + +--- + pengine/test10/remote-fence-unclean-3.dot | 22 ++----- + pengine/test10/remote-fence-unclean-3.exp | 95 +++++++-------------------- + pengine/test10/remote-fence-unclean-3.scores | 4 -- + pengine/test10/remote-fence-unclean-3.summary | 15 ++--- + 4 files changed, 34 insertions(+), 102 deletions(-) + +diff --git a/pengine/test10/remote-fence-unclean-3.dot b/pengine/test10/remote-fence-unclean-3.dot +index 11d1208..b32b77e 100644 +--- a/pengine/test10/remote-fence-unclean-3.dot ++++ b/pengine/test10/remote-fence-unclean-3.dot +@@ -1,27 +1,19 @@ + digraph "g" { +-"all_stopped" -> "fence1_start_0 overcloud-controller-1" [ style = bold] ++"all_stopped" -> "fence1_start_0 overcloud-controller-0" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] +-"fence1_monitor_0 overcloud-controller-0" -> "fence1_start_0 overcloud-controller-1" [ style = bold] ++"fence1_monitor_0 overcloud-controller-0" -> "fence1_start_0 overcloud-controller-0" [ style = bold] + "fence1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] +-"fence1_monitor_0 overcloud-controller-1" -> "fence1_start_0 overcloud-controller-1" [ style = bold] ++"fence1_monitor_0 overcloud-controller-1" -> "fence1_start_0 overcloud-controller-0" [ style = bold] + "fence1_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] +-"fence1_monitor_0 overcloud-controller-2" -> "fence1_start_0 overcloud-controller-1" [ style = bold] ++"fence1_monitor_0 overcloud-controller-2" -> "fence1_start_0 overcloud-controller-0" [ style = bold] + "fence1_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] +-"fence1_monitor_60000 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] +-"fence1_start_0 overcloud-controller-1" -> "fence1_monitor_60000 overcloud-controller-1" [ style = bold] +-"fence1_start_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] +-"foo_monitor_0 overcloud-controller-0" -> "foo_start_0 overcloud-controller-0" [ style = bold] +-"foo_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] +-"foo_monitor_0 overcloud-controller-1" -> "foo_start_0 overcloud-controller-0" [ style = bold] +-"foo_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] +-"foo_monitor_0 overcloud-controller-2" -> "foo_start_0 overcloud-controller-0" [ style = bold] +-"foo_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] +-"foo_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"fence1_monitor_60000 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"fence1_start_0 overcloud-controller-0" -> "fence1_monitor_60000 overcloud-controller-0" [ style = bold] ++"fence1_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] + "overcloud-novacompute-0_stop_0 overcloud-controller-0" -> "all_stopped" [ style = bold] + "overcloud-novacompute-0_stop_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' overcloud-novacompute-0" -> "stonith_complete" [ style = bold] + "stonith 'reboot' overcloud-novacompute-0" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] +-"stonith_complete" -> "foo_start_0 overcloud-controller-0" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp +index 14e1b0b..2e341bd 100644 +--- a/pengine/test10/remote-fence-unclean-3.exp ++++ b/pengine/test10/remote-fence-unclean-3.exp +@@ -1,71 +1,22 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + +- ++ + + + + +- ++ + + + +- ++ + +- ++ + +- ++ + + + +@@ -73,44 +24,44 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +@@ -122,9 +73,9 @@ + + + +- ++ + +- ++ + + + +@@ -133,19 +84,19 @@ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + + + +@@ -156,7 +107,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/remote-fence-unclean-3.scores b/pengine/test10/remote-fence-unclean-3.scores +index 0284218..24bdcf9 100644 +--- a/pengine/test10/remote-fence-unclean-3.scores ++++ b/pengine/test10/remote-fence-unclean-3.scores +@@ -872,10 +872,6 @@ native_color: fence1 allocation score on overcloud-controller-0: 0 + native_color: fence1 allocation score on overcloud-controller-1: 0 + native_color: fence1 allocation score on overcloud-controller-2: 0 + native_color: fence1 allocation score on overcloud-novacompute-0: -INFINITY +-native_color: foo allocation score on overcloud-controller-0: 0 +-native_color: foo allocation score on overcloud-controller-1: 0 +-native_color: foo allocation score on overcloud-controller-2: 0 +-native_color: foo allocation score on overcloud-novacompute-0: 0 + native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY + native_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY + native_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY +diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary +index cc60ebe..ec24500 100644 +--- a/pengine/test10/remote-fence-unclean-3.summary ++++ b/pengine/test10/remote-fence-unclean-3.summary +@@ -4,7 +4,6 @@ Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] + RemoteOFFLINE: [ overcloud-novacompute-0 ] + Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] + +- foo (ocf::heartbeat:Dummy): Stopped + fence1 (stonith:fence_xvm): Stopped + overcloud-novacompute-0 (ocf::pacemaker:remote): FAILED overcloud-controller-0 + Docker container set: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] +@@ -36,14 +35,10 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund + + Transition Summary: + * Fence overcloud-novacompute-0 +- * Start foo (overcloud-controller-0) +- * Start fence1 (overcloud-controller-1) ++ * Start fence1 (overcloud-controller-0) + * Stop overcloud-novacompute-0 (overcloud-controller-0) + + Executing cluster transition: +- * Resource action: foo monitor on overcloud-controller-2 +- * Resource action: foo monitor on overcloud-controller-1 +- * Resource action: foo monitor on overcloud-controller-0 + * Resource action: fence1 monitor on overcloud-controller-2 + * Resource action: fence1 monitor on overcloud-controller-1 + * Resource action: fence1 monitor on overcloud-controller-0 +@@ -51,17 +46,15 @@ Executing cluster transition: + * Fencing overcloud-novacompute-0 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped +- * Resource action: foo start on overcloud-controller-0 +- * Resource action: fence1 start on overcloud-controller-1 +- * Resource action: fence1 monitor=60000 on overcloud-controller-1 ++ * Resource action: fence1 start on overcloud-controller-0 ++ * Resource action: fence1 monitor=60000 on overcloud-controller-0 + + Revised cluster status: + Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] + RemoteOFFLINE: [ overcloud-novacompute-0 ] + Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-0:redis-bundle-docker-0 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] + +- foo (ocf::heartbeat:Dummy): Started overcloud-controller-0 +- fence1 (stonith:fence_xvm): Started overcloud-controller-1 ++ fence1 (stonith:fence_xvm): Started overcloud-controller-0 + overcloud-novacompute-0 (ocf::pacemaker:remote): Stopped + Docker container set: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] + rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-controller-0 +-- +1.8.3.1 + diff --git a/SOURCES/085-remote-fence-test2.patch b/SOURCES/085-remote-fence-test2.patch new file mode 100644 index 0000000..03d2304 --- /dev/null +++ b/SOURCES/085-remote-fence-test2.patch @@ -0,0 +1,37 @@ +From 25647ff86dd24aff9e6de0f091656db309ed0c02 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 6 Jul 2017 17:04:02 -0500 +Subject: [PATCH] Test: pengine: update remote-fence-unclean-3 test for 1.1.17 + merge + +--- + pengine/test10/remote-fence-unclean-3.scores | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/pengine/test10/remote-fence-unclean-3.scores b/pengine/test10/remote-fence-unclean-3.scores +index 24bdcf9..77d71d6 100644 +--- a/pengine/test10/remote-fence-unclean-3.scores ++++ b/pengine/test10/remote-fence-unclean-3.scores +@@ -872,6 +872,9 @@ native_color: fence1 allocation score on overcloud-controller-0: 0 + native_color: fence1 allocation score on overcloud-controller-1: 0 + native_color: fence1 allocation score on overcloud-controller-2: 0 + native_color: fence1 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: fence1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: fence1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: fence1 allocation score on rabbitmq-bundle-2: -INFINITY + native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY + native_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY + native_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY +@@ -1109,6 +1112,9 @@ native_color: overcloud-novacompute-0 allocation score on overcloud-controller-0 + native_color: overcloud-novacompute-0 allocation score on overcloud-controller-1: -INFINITY + native_color: overcloud-novacompute-0 allocation score on overcloud-controller-2: -INFINITY + native_color: overcloud-novacompute-0 allocation score on overcloud-novacompute-0: -INFINITY ++native_color: overcloud-novacompute-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: overcloud-novacompute-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: overcloud-novacompute-0 allocation score on rabbitmq-bundle-2: -INFINITY + native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-0: INFINITY + native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-1: 0 + native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-2: 0 +-- +1.8.3.1 + diff --git a/SOURCES/086-nested-containers.patch b/SOURCES/086-nested-containers.patch new file mode 100644 index 0000000..70d3097 --- /dev/null +++ b/SOURCES/086-nested-containers.patch @@ -0,0 +1,40 @@ +From 786f44f9ab2731c8fc7d534689d5925cd6d3d98e Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 18 Jul 2017 14:31:45 +1000 +Subject: [PATCH] Fix: PE: Ensure nested container connections run on the same + host + +--- + pengine/container.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/pengine/container.c b/pengine/container.c +index f742660..280a61a 100644 +--- a/pengine/container.c ++++ b/pengine/container.c +@@ -100,11 +100,22 @@ container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + + for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { + container_grouping_t *tuple = (container_grouping_t *)gIter->data; ++ pe_node_t *docker_host = tuple->docker->allocated_to; + + CRM_ASSERT(tuple); + if(tuple->ip) { + tuple->ip->cmds->allocate(tuple->ip, prefer, data_set); + } ++ ++ if(tuple->remote && is_remote_node(docker_host)) { ++ /* We need 'nested' connection resources to be on the same ++ * host because pacemaker-remoted only supports a single ++ * active connection ++ */ ++ rsc_colocation_new("child-remote-with-docker-remote", NULL, ++ INFINITY, tuple->remote, docker_host->details->remote_rsc, NULL, NULL, data_set); ++ } ++ + if(tuple->remote) { + tuple->remote->cmds->allocate(tuple->remote, prefer, data_set); + } +-- +1.8.3.1 + diff --git a/SOURCES/087-nested-containers-test.patch b/SOURCES/087-nested-containers-test.patch new file mode 100644 index 0000000..211ce27 --- /dev/null +++ b/SOURCES/087-nested-containers-test.patch @@ -0,0 +1,1461 @@ +From 345bf572da5969da1a3d77940fb3b5f5b2cef771 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 18 Jul 2017 14:35:44 +1000 +Subject: [PATCH] Test: PE: Ensure nested container connections run on the same + host + +--- + pengine/regression.sh | 2 +- + pengine/test10/bundle-nested-colocation.dot | 72 ++++ + pengine/test10/bundle-nested-colocation.exp | 385 ++++++++++++++++++++ + pengine/test10/bundle-nested-colocation.scores | 401 +++++++++++++++++++++ + pengine/test10/bundle-nested-colocation.summary | 75 ++++ + pengine/test10/bundle-nested-colocation.xml | 455 ++++++++++++++++++++++++ + 6 files changed, 1389 insertions(+), 1 deletion(-) + create mode 100644 pengine/test10/bundle-nested-colocation.dot + create mode 100644 pengine/test10/bundle-nested-colocation.exp + create mode 100644 pengine/test10/bundle-nested-colocation.scores + create mode 100644 pengine/test10/bundle-nested-colocation.summary + create mode 100644 pengine/test10/bundle-nested-colocation.xml + +diff --git a/pengine/regression.sh b/pengine/regression.sh +index d89d93c..d1a8a3f 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -28,7 +28,6 @@ echo "" + + info Performing the following tests from $io_dir + create_mode="false" +- + echo "" + + do_test simple1 "Offline " +@@ -810,6 +809,7 @@ do_test bundle-order-partial-stop "Bundle startup ordering when some dependancie + do_test bundle-order-startup-clone "Prevent startup because bundle isn't promoted" + do_test bundle-order-startup-clone-2 "Bundle startup with clones" + do_test bundle-order-stop-clone "Stop bundle because clone is stopping" ++do_test bundle-nested-colocation "Colocation of nested connection resources" + + echo "" + do_test whitebox-fail1 "Fail whitebox container rsc." +diff --git a/pengine/test10/bundle-nested-colocation.dot b/pengine/test10/bundle-nested-colocation.dot +new file mode 100644 +index 0000000..a84f857 +--- /dev/null ++++ b/pengine/test10/bundle-nested-colocation.dot +@@ -0,0 +1,72 @@ ++digraph "g" { ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-0_monitor_60000 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq-bundle-0_monitor_60000 overcloud-controller-0" [ style = bold] ++"rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_60000 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq-bundle-1_monitor_60000 overcloud-controller-1" [ style = bold] ++"rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_60000 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq-bundle-2_monitor_60000 overcloud-controller-2" [ style = bold] ++"rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq:2_monitor_10000 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq:2_start_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq:2_start_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-docker-0_monitor_60000 overcloud-rabbit-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-0_start_0 overcloud-rabbit-0" -> "rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style = bold] ++"rabbitmq-bundle-docker-0_start_0 overcloud-rabbit-0" -> "rabbitmq-bundle-docker-0_monitor_60000 overcloud-rabbit-0" [ style = bold] ++"rabbitmq-bundle-docker-0_start_0 overcloud-rabbit-0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-docker-0_start_0 overcloud-rabbit-0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-docker-0_start_0 overcloud-rabbit-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-0_stop_0 overcloud-rabbit-0" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 overcloud-rabbit-0" -> "rabbitmq-bundle-docker-0_start_0 overcloud-rabbit-0" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 overcloud-rabbit-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-1_monitor_60000 overcloud-rabbit-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-1_start_0 overcloud-rabbit-1" -> "rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style = bold] ++"rabbitmq-bundle-docker-1_start_0 overcloud-rabbit-1" -> "rabbitmq-bundle-docker-1_monitor_60000 overcloud-rabbit-1" [ style = bold] ++"rabbitmq-bundle-docker-1_start_0 overcloud-rabbit-1" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-docker-1_start_0 overcloud-rabbit-1" -> "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-docker-1_start_0 overcloud-rabbit-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-1_stop_0 overcloud-rabbit-1" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-docker-1_stop_0 overcloud-rabbit-1" -> "rabbitmq-bundle-docker-1_start_0 overcloud-rabbit-1" [ style = bold] ++"rabbitmq-bundle-docker-1_stop_0 overcloud-rabbit-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-2_monitor_60000 overcloud-rabbit-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-2_start_0 overcloud-rabbit-2" -> "rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style = bold] ++"rabbitmq-bundle-docker-2_start_0 overcloud-rabbit-2" -> "rabbitmq-bundle-docker-2_monitor_60000 overcloud-rabbit-2" [ style = bold] ++"rabbitmq-bundle-docker-2_start_0 overcloud-rabbit-2" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-docker-2_start_0 overcloud-rabbit-2" -> "rabbitmq:2_start_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-docker-2_start_0 overcloud-rabbit-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-docker-2_stop_0 overcloud-rabbit-2" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-docker-2_stop_0 overcloud-rabbit-2" -> "rabbitmq-bundle-docker-2_start_0 overcloud-rabbit-2" [ style = bold] ++"rabbitmq-bundle-docker-2_stop_0 overcloud-rabbit-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-0_start_0 overcloud-rabbit-0" [ style = bold] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-1_start_0 overcloud-rabbit-1" [ style = bold] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-2_start_0 overcloud-rabbit-2" [ style = bold] ++"rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:1_start_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq:1_start_0 rabbitmq-bundle-1" -> "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq:1_start_0 rabbitmq-bundle-1" -> "rabbitmq:2_start_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq:1_start_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:2_monitor_10000 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:2_start_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq:2_start_0 rabbitmq-bundle-2" -> "rabbitmq:2_monitor_10000 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq:2_start_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] ++} +diff --git a/pengine/test10/bundle-nested-colocation.exp b/pengine/test10/bundle-nested-colocation.exp +new file mode 100644 +index 0000000..3daf7d6 +--- /dev/null ++++ b/pengine/test10/bundle-nested-colocation.exp +@@ -0,0 +1,385 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-nested-colocation.scores b/pengine/test10/bundle-nested-colocation.scores +new file mode 100644 +index 0000000..c79e0ff +--- /dev/null ++++ b/pengine/test10/bundle-nested-colocation.scores +@@ -0,0 +1,401 @@ ++Allocation scores: ++Using the original execution date of: 2017-07-14 08:50:25Z ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-controller-0: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-controller-1: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-controller-2: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-galera-0: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-galera-1: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-galera-2: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-rabbit-0: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-rabbit-1: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on overcloud-rabbit-2: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0 ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0 ++clone_color: rabbitmq:0 allocation score on overcloud-controller-0: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-controller-1: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-controller-2: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-galera-0: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-galera-1: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-galera-2: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-rabbit-0: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-rabbit-1: -INFINITY ++clone_color: rabbitmq:0 allocation score on overcloud-rabbit-2: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-controller-0: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-controller-1: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-controller-2: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-galera-0: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-galera-1: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-galera-2: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-rabbit-0: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-rabbit-1: -INFINITY ++clone_color: rabbitmq:1 allocation score on overcloud-rabbit-2: -INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-controller-0: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-controller-1: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-controller-2: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-galera-0: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-galera-1: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-galera-2: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-rabbit-0: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-rabbit-1: -INFINITY ++clone_color: rabbitmq:2 allocation score on overcloud-rabbit-2: -INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++container_color: galera-bundle allocation score on overcloud-controller-0: -INFINITY ++container_color: galera-bundle allocation score on overcloud-controller-1: -INFINITY ++container_color: galera-bundle allocation score on overcloud-controller-2: -INFINITY ++container_color: galera-bundle allocation score on overcloud-galera-0: 0 ++container_color: galera-bundle allocation score on overcloud-galera-1: 0 ++container_color: galera-bundle allocation score on overcloud-galera-2: 0 ++container_color: galera-bundle allocation score on overcloud-rabbit-0: -INFINITY ++container_color: galera-bundle allocation score on overcloud-rabbit-1: -INFINITY ++container_color: galera-bundle allocation score on overcloud-rabbit-2: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on overcloud-controller-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on overcloud-controller-1: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on overcloud-controller-2: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on overcloud-galera-0: INFINITY ++container_color: galera-bundle-docker-0 allocation score on overcloud-galera-1: 0 ++container_color: galera-bundle-docker-0 allocation score on overcloud-galera-2: 0 ++container_color: galera-bundle-docker-0 allocation score on overcloud-rabbit-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on overcloud-rabbit-1: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on overcloud-rabbit-2: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on overcloud-controller-1: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on overcloud-controller-2: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on overcloud-galera-0: 0 ++container_color: galera-bundle-docker-1 allocation score on overcloud-galera-1: INFINITY ++container_color: galera-bundle-docker-1 allocation score on overcloud-galera-2: 0 ++container_color: galera-bundle-docker-1 allocation score on overcloud-rabbit-0: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on overcloud-rabbit-1: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on overcloud-rabbit-2: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on overcloud-controller-2: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on overcloud-galera-0: 0 ++container_color: galera-bundle-docker-2 allocation score on overcloud-galera-1: 0 ++container_color: galera-bundle-docker-2 allocation score on overcloud-galera-2: INFINITY ++container_color: galera-bundle-docker-2 allocation score on overcloud-rabbit-0: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on overcloud-rabbit-1: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on overcloud-rabbit-2: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle allocation score on overcloud-controller-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on overcloud-controller-1: -INFINITY ++container_color: rabbitmq-bundle allocation score on overcloud-controller-2: -INFINITY ++container_color: rabbitmq-bundle allocation score on overcloud-galera-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on overcloud-galera-1: -INFINITY ++container_color: rabbitmq-bundle allocation score on overcloud-galera-2: -INFINITY ++container_color: rabbitmq-bundle allocation score on overcloud-rabbit-0: 1 ++container_color: rabbitmq-bundle allocation score on overcloud-rabbit-1: 1 ++container_color: rabbitmq-bundle allocation score on overcloud-rabbit-2: 1 ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle-0 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle-0 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle-0 allocation score on overcloud-galera-0: 0 ++container_color: rabbitmq-bundle-0 allocation score on overcloud-galera-1: 0 ++container_color: rabbitmq-bundle-0 allocation score on overcloud-galera-2: 0 ++container_color: rabbitmq-bundle-0 allocation score on overcloud-rabbit-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on overcloud-rabbit-1: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on overcloud-rabbit-2: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle-1 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle-1 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle-1 allocation score on overcloud-galera-0: 0 ++container_color: rabbitmq-bundle-1 allocation score on overcloud-galera-1: 0 ++container_color: rabbitmq-bundle-1 allocation score on overcloud-galera-2: 0 ++container_color: rabbitmq-bundle-1 allocation score on overcloud-rabbit-0: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on overcloud-rabbit-1: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on overcloud-rabbit-2: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle-2 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle-2 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle-2 allocation score on overcloud-galera-0: 0 ++container_color: rabbitmq-bundle-2 allocation score on overcloud-galera-1: 0 ++container_color: rabbitmq-bundle-2 allocation score on overcloud-galera-2: 0 ++container_color: rabbitmq-bundle-2 allocation score on overcloud-rabbit-0: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on overcloud-rabbit-1: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on overcloud-rabbit-2: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-clone allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-galera-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-galera-1: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-galera-2: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-rabbit-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-rabbit-1: 0 ++container_color: rabbitmq-bundle-clone allocation score on overcloud-rabbit-2: 0 ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-1: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-2: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-galera-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-galera-1: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-galera-2: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-rabbit-0: INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-rabbit-1: 1 ++container_color: rabbitmq-bundle-docker-0 allocation score on overcloud-rabbit-2: 1 ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-1: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-2: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-galera-0: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-galera-1: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-galera-2: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-rabbit-0: 1 ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-rabbit-1: INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on overcloud-rabbit-2: 1 ++container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-2: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-galera-0: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-galera-1: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-galera-2: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-rabbit-0: 1 ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-rabbit-1: 1 ++container_color: rabbitmq-bundle-docker-2 allocation score on overcloud-rabbit-2: INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq:0 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq:0 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq:0 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq:0 allocation score on overcloud-galera-0: 0 ++container_color: rabbitmq:0 allocation score on overcloud-galera-1: 0 ++container_color: rabbitmq:0 allocation score on overcloud-galera-2: 0 ++container_color: rabbitmq:0 allocation score on overcloud-rabbit-0: 0 ++container_color: rabbitmq:0 allocation score on overcloud-rabbit-1: 0 ++container_color: rabbitmq:0 allocation score on overcloud-rabbit-2: 0 ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq:1 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq:1 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq:1 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq:1 allocation score on overcloud-galera-0: 0 ++container_color: rabbitmq:1 allocation score on overcloud-galera-1: 0 ++container_color: rabbitmq:1 allocation score on overcloud-galera-2: 0 ++container_color: rabbitmq:1 allocation score on overcloud-rabbit-0: 0 ++container_color: rabbitmq:1 allocation score on overcloud-rabbit-1: 0 ++container_color: rabbitmq:1 allocation score on overcloud-rabbit-2: 0 ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq:2 allocation score on overcloud-controller-0: 0 ++container_color: rabbitmq:2 allocation score on overcloud-controller-1: 0 ++container_color: rabbitmq:2 allocation score on overcloud-controller-2: 0 ++container_color: rabbitmq:2 allocation score on overcloud-galera-0: 0 ++container_color: rabbitmq:2 allocation score on overcloud-galera-1: 0 ++container_color: rabbitmq:2 allocation score on overcloud-galera-2: 0 ++container_color: rabbitmq:2 allocation score on overcloud-rabbit-0: 0 ++container_color: rabbitmq:2 allocation score on overcloud-rabbit-1: 0 ++container_color: rabbitmq:2 allocation score on overcloud-rabbit-2: 0 ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on overcloud-controller-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on overcloud-controller-1: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on overcloud-controller-2: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on overcloud-galera-0: INFINITY ++native_color: galera-bundle-docker-0 allocation score on overcloud-galera-1: 0 ++native_color: galera-bundle-docker-0 allocation score on overcloud-galera-2: 0 ++native_color: galera-bundle-docker-0 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-controller-1: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-controller-2: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-galera-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-galera-1: INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-galera-2: 0 ++native_color: galera-bundle-docker-1 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-controller-2: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-galera-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-galera-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-galera-2: INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: overcloud-rabbit-0 allocation score on overcloud-controller-0: INFINITY ++native_color: overcloud-rabbit-0 allocation score on overcloud-controller-1: 0 ++native_color: overcloud-rabbit-0 allocation score on overcloud-controller-2: 0 ++native_color: overcloud-rabbit-0 allocation score on overcloud-galera-0: 0 ++native_color: overcloud-rabbit-0 allocation score on overcloud-galera-1: 0 ++native_color: overcloud-rabbit-0 allocation score on overcloud-galera-2: 0 ++native_color: overcloud-rabbit-0 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: overcloud-rabbit-0 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: overcloud-rabbit-0 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: overcloud-rabbit-1 allocation score on overcloud-controller-0: 0 ++native_color: overcloud-rabbit-1 allocation score on overcloud-controller-1: INFINITY ++native_color: overcloud-rabbit-1 allocation score on overcloud-controller-2: 0 ++native_color: overcloud-rabbit-1 allocation score on overcloud-galera-0: 0 ++native_color: overcloud-rabbit-1 allocation score on overcloud-galera-1: 0 ++native_color: overcloud-rabbit-1 allocation score on overcloud-galera-2: 0 ++native_color: overcloud-rabbit-1 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: overcloud-rabbit-1 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: overcloud-rabbit-1 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: overcloud-rabbit-2 allocation score on overcloud-controller-0: 0 ++native_color: overcloud-rabbit-2 allocation score on overcloud-controller-1: 0 ++native_color: overcloud-rabbit-2 allocation score on overcloud-controller-2: INFINITY ++native_color: overcloud-rabbit-2 allocation score on overcloud-galera-0: 0 ++native_color: overcloud-rabbit-2 allocation score on overcloud-galera-1: 0 ++native_color: overcloud-rabbit-2 allocation score on overcloud-galera-2: 0 ++native_color: overcloud-rabbit-2 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: overcloud-rabbit-2 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: overcloud-rabbit-2 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-0: 0 ++native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-galera-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-galera-1: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-galera-2: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-controller-1: 0 ++native_color: rabbitmq-bundle-1 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-galera-0: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-galera-1: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-galera-2: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-controller-2: 0 ++native_color: rabbitmq-bundle-2 allocation score on overcloud-galera-0: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-galera-1: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-galera-2: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-galera-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-galera-1: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-galera-2: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-rabbit-0: INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-rabbit-1: -9999 ++native_color: rabbitmq-bundle-docker-0 allocation score on overcloud-rabbit-2: -9999 ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-galera-0: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-galera-1: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-galera-2: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-rabbit-1: INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on overcloud-rabbit-2: -9999 ++native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-galera-0: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-galera-1: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-galera-2: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on overcloud-rabbit-2: INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-galera-0: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-galera-1: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-galera-2: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: rabbitmq:0 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-galera-0: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-galera-1: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-galera-2: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: rabbitmq:1 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-controller-0: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-controller-1: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-controller-2: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-galera-0: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-galera-1: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-galera-2: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-rabbit-0: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-rabbit-1: -INFINITY ++native_color: rabbitmq:2 allocation score on overcloud-rabbit-2: -INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY +diff --git a/pengine/test10/bundle-nested-colocation.summary b/pengine/test10/bundle-nested-colocation.summary +new file mode 100644 +index 0000000..9755e5f +--- /dev/null ++++ b/pengine/test10/bundle-nested-colocation.summary +@@ -0,0 +1,75 @@ ++Using the original execution date of: 2017-07-14 08:50:25Z ++ ++Current cluster status: ++Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 overcloud-galera-0 overcloud-galera-1 overcloud-galera-2 ] ++RemoteOnline: [ overcloud-rabbit-0 overcloud-rabbit-1 overcloud-rabbit-2 ] ++ ++ overcloud-rabbit-0 (ocf::pacemaker:remote): Started overcloud-controller-0 ++ overcloud-rabbit-1 (ocf::pacemaker:remote): Started overcloud-controller-1 ++ overcloud-rabbit-2 (ocf::pacemaker:remote): Started overcloud-controller-2 ++ Docker container set: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Stopped overcloud-rabbit-0 ++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Stopped overcloud-rabbit-1 ++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Stopped overcloud-rabbit-2 ++ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-docker-0 (ocf::heartbeat:docker): Started overcloud-galera-0 ++ galera-bundle-docker-1 (ocf::heartbeat:docker): Started overcloud-galera-1 ++ galera-bundle-docker-2 (ocf::heartbeat:docker): Started overcloud-galera-2 ++ ++Transition Summary: ++ * Restart rabbitmq-bundle-docker-0 (Started overcloud-rabbit-0) ++ * Start rabbitmq-bundle-0 (overcloud-controller-0) ++ * Start rabbitmq:0 (rabbitmq-bundle-0) ++ * Restart rabbitmq-bundle-docker-1 (Started overcloud-rabbit-1) ++ * Start rabbitmq-bundle-1 (overcloud-controller-1) ++ * Start rabbitmq:1 (rabbitmq-bundle-1) ++ * Restart rabbitmq-bundle-docker-2 (Started overcloud-rabbit-2) ++ * Start rabbitmq-bundle-2 (overcloud-controller-2) ++ * Start rabbitmq:2 (rabbitmq-bundle-2) ++ ++Executing cluster transition: ++ * Resource action: rabbitmq-bundle-docker-0 stop on overcloud-rabbit-0 ++ * Resource action: rabbitmq-bundle-docker-1 stop on overcloud-rabbit-1 ++ * Resource action: rabbitmq-bundle-docker-2 stop on overcloud-rabbit-2 ++ * Pseudo action: rabbitmq-bundle_start_0 ++ * Pseudo action: all_stopped ++ * Resource action: rabbitmq-bundle-docker-0 start on overcloud-rabbit-0 ++ * Resource action: rabbitmq-bundle-docker-0 monitor=60000 on overcloud-rabbit-0 ++ * Resource action: rabbitmq-bundle-0 start on overcloud-controller-0 ++ * Resource action: rabbitmq-bundle-docker-1 start on overcloud-rabbit-1 ++ * Resource action: rabbitmq-bundle-docker-1 monitor=60000 on overcloud-rabbit-1 ++ * Resource action: rabbitmq-bundle-1 start on overcloud-controller-1 ++ * Resource action: rabbitmq-bundle-docker-2 start on overcloud-rabbit-2 ++ * Resource action: rabbitmq-bundle-docker-2 monitor=60000 on overcloud-rabbit-2 ++ * Resource action: rabbitmq-bundle-2 start on overcloud-controller-2 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 ++ * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 ++ * Resource action: rabbitmq-bundle-0 monitor=60000 on overcloud-controller-0 ++ * Resource action: rabbitmq:1 start on rabbitmq-bundle-1 ++ * Resource action: rabbitmq-bundle-1 monitor=60000 on overcloud-controller-1 ++ * Resource action: rabbitmq:2 start on rabbitmq-bundle-2 ++ * Resource action: rabbitmq-bundle-2 monitor=60000 on overcloud-controller-2 ++ * Pseudo action: rabbitmq-bundle-clone_running_0 ++ * Pseudo action: rabbitmq-bundle_running_0 ++ * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 ++ * Resource action: rabbitmq:1 monitor=10000 on rabbitmq-bundle-1 ++ * Resource action: rabbitmq:2 monitor=10000 on rabbitmq-bundle-2 ++Using the original execution date of: 2017-07-14 08:50:25Z ++ ++Revised cluster status: ++Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 overcloud-galera-0 overcloud-galera-1 overcloud-galera-2 ] ++RemoteOnline: [ overcloud-rabbit-0 overcloud-rabbit-1 overcloud-rabbit-2 ] ++Containers: [ rabbitmq-bundle-0:rabbitmq-bundle-docker-0 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 ] ++ ++ overcloud-rabbit-0 (ocf::pacemaker:remote): Started overcloud-controller-0 ++ overcloud-rabbit-1 (ocf::pacemaker:remote): Started overcloud-controller-1 ++ overcloud-rabbit-2 (ocf::pacemaker:remote): Started overcloud-controller-2 ++ Docker container set: rabbitmq-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-rabbitmq:latest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-rabbit-0 ++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-rabbit-1 ++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started overcloud-rabbit-2 ++ Docker container set: galera-bundle [192.168.24.1:8787/tripleoupstream/centos-binary-mariadb:latest] ++ galera-bundle-docker-0 (ocf::heartbeat:docker): Started overcloud-galera-0 ++ galera-bundle-docker-1 (ocf::heartbeat:docker): Started overcloud-galera-1 ++ galera-bundle-docker-2 (ocf::heartbeat:docker): Started overcloud-galera-2 ++ +diff --git a/pengine/test10/bundle-nested-colocation.xml b/pengine/test10/bundle-nested-colocation.xml +new file mode 100644 +index 0000000..d8f6139 +--- /dev/null ++++ b/pengine/test10/bundle-nested-colocation.xml +@@ -0,0 +1,455 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +-- +1.8.3.1 + diff --git a/SOURCES/088-str-table.patch b/SOURCES/088-str-table.patch new file mode 100644 index 0000000..084bb22 --- /dev/null +++ b/SOURCES/088-str-table.patch @@ -0,0 +1,128 @@ +From 19760e5b97136de3474743db8f2ae86380658f82 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 12 Jul 2017 15:53:20 -0500 +Subject: [PATCH] Refactor: libcrmcommon,etc.: convenience functions for string + tables + +A simple hash table of dynamically allocated strings is the most common type in +the code base. This adds convenience functions for regular and case-insensitive +table creation, which makes the code less ugly, hopefully without sacrificing +readability. + +As a side effect, this gets rid of a lot of usage of g_hash_destroy_str() when +we only need free(). +--- + include/crm/common/util.h | 32 ++++++++++++++++++++++++++++++++ + include/crm/crm.h | 5 ----- + lib/common/strings.c | 20 ++++++++++++++++++++ + 3 files changed, 52 insertions(+), 5 deletions(-) + +diff --git a/include/crm/common/util.h b/include/crm/common/util.h +index 904b40c..04f7c31 100644 +--- a/include/crm/common/util.h ++++ b/include/crm/common/util.h +@@ -30,6 +30,7 @@ + # include + # include + # include ++# include + + # include + +@@ -60,8 +61,11 @@ int crm_parse_int(const char *text, const char *default_text); + char * crm_strip_trailing_newline(char *str); + gboolean crm_str_eq(const char *a, const char *b, gboolean use_case); + gboolean safe_str_neq(const char *a, const char *b); ++guint crm_strcase_hash(gconstpointer v); ++guint g_str_hash_traditional(gconstpointer v); + + # define safe_str_eq(a, b) crm_str_eq(a, b, FALSE) ++# define crm_str_hash g_str_hash_traditional + + /* used with hash tables where case does not matter */ + static inline gboolean +@@ -70,6 +74,34 @@ crm_strcase_equal(gconstpointer a, gconstpointer b) + return crm_str_eq((const char *) a, (const char *) b, FALSE); + } + ++/*! ++ * \brief Create hash table with dynamically allocated string keys/values ++ * ++ * \return Newly hash table ++ * \note It is the caller's responsibility to free the result, using ++ * g_hash_table_destroy(). ++ */ ++static inline GHashTable * ++crm_str_table_new() ++{ ++ return g_hash_table_new_full(crm_str_hash, g_str_equal, free, free); ++} ++ ++/*! ++ * \brief Create hash table with case-insensitive dynamically allocated string keys/values ++ * ++ * \return Newly hash table ++ * \note It is the caller's responsibility to free the result, using ++ * g_hash_table_destroy(). ++ */ ++static inline GHashTable * ++crm_strcase_table_new() ++{ ++ return g_hash_table_new_full(crm_strcase_hash, crm_strcase_equal, free, free); ++} ++ ++GHashTable *crm_str_table_dup(GHashTable *old_table); ++ + # define crm_atoi(text, default_text) crm_parse_int(text, default_text) + + /* public I/O functions (from io.c) */ +diff --git a/include/crm/crm.h b/include/crm/crm.h +index d03bcf2..0da6bfe 100644 +--- a/include/crm/crm.h ++++ b/include/crm/crm.h +@@ -208,11 +208,6 @@ typedef GList *GListPtr; + # include + # include + +-# define crm_str_hash g_str_hash_traditional +- +-guint crm_strcase_hash(gconstpointer v); +-guint g_str_hash_traditional(gconstpointer v); +- + static inline const char *crm_action_str(const char *task, int interval) { + if(safe_str_eq(task, RSC_STATUS) && !interval) { + return "probe"; +diff --git a/lib/common/strings.c b/lib/common/strings.c +index 3df41f6..c624b4b 100644 +--- a/lib/common/strings.c ++++ b/lib/common/strings.c +@@ -288,6 +288,26 @@ crm_strcase_hash(gconstpointer v) + return h; + } + ++static void ++copy_str_table_entry(gpointer key, gpointer value, gpointer user_data) ++{ ++ if (key && value && user_data) { ++ g_hash_table_insert((GHashTable*)user_data, strdup(key), strdup(value)); ++ } ++} ++ ++GHashTable * ++crm_str_table_dup(GHashTable *old_table) ++{ ++ GHashTable *new_table = NULL; ++ ++ if (old_table) { ++ new_table = crm_str_table_new(); ++ g_hash_table_foreach(old_table, copy_str_table_entry, new_table); ++ } ++ return new_table; ++} ++ + char * + add_list_element(char *list, const char *value) + { +-- +1.8.3.1 + diff --git a/SOURCES/089-unfencing.patch b/SOURCES/089-unfencing.patch new file mode 100644 index 0000000..75a680e --- /dev/null +++ b/SOURCES/089-unfencing.patch @@ -0,0 +1,45472 @@ +From fc69905f116e41a724fdde160c83111d22ae8b2e Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 25 Jul 2017 09:24:57 +1000 +Subject: [PATCH 01/12] Log: PE: Ignore optional unfencing events and report + the fencing type + +--- + pengine/allocate.c | 10 +++++++--- + pengine/test10/594.summary | 2 +- + pengine/test10/829.summary | 2 +- + pengine/test10/bug-5186-partial-migrate.summary | 2 +- + pengine/test10/bug-cl-5247.summary | 2 +- + pengine/test10/bug-lf-2508.summary | 2 +- + pengine/test10/bug-lf-2551.summary | 2 +- + pengine/test10/bug-lf-2606.summary | 2 +- + pengine/test10/bug-rh-1097457.summary | 2 +- + pengine/test10/concurrent-fencing.summary | 6 +++--- + pengine/test10/guest-node-host-dies.summary | 6 +++--- + pengine/test10/interleave-pseudo-stop.summary | 2 +- + pengine/test10/master-7.summary | 2 +- + pengine/test10/master-8.summary | 2 +- + pengine/test10/migrate-fencing.summary | 2 +- + pengine/test10/rec-node-11.summary | 2 +- + pengine/test10/rec-node-12.summary | 2 +- + pengine/test10/rec-node-13.summary | 2 +- + pengine/test10/rec-node-14.summary | 6 +++--- + pengine/test10/rec-node-15.summary | 2 +- + pengine/test10/rec-node-2.summary | 2 +- + pengine/test10/rec-node-4.summary | 2 +- + pengine/test10/rec-node-6.summary | 2 +- + pengine/test10/rec-node-7.summary | 2 +- + pengine/test10/rec-rsc-5.summary | 2 +- + pengine/test10/remote-fence-before-reconnect.summary | 2 +- + pengine/test10/remote-fence-unclean-3.summary | 2 +- + pengine/test10/remote-fence-unclean.summary | 2 +- + pengine/test10/remote-fence-unclean2.summary | 2 +- + pengine/test10/remote-partial-migrate2.summary | 2 +- + pengine/test10/remote-recover-all.summary | 6 +++--- + pengine/test10/remote-recover-connection.summary | 2 +- + pengine/test10/remote-recover-fail.summary | 2 +- + pengine/test10/remote-recover-no-resources.summary | 4 ++-- + pengine/test10/remote-recover-unknown.summary | 6 +++--- + pengine/test10/remote-recovery.summary | 2 +- + pengine/test10/remote-unclean2.summary | 2 +- + pengine/test10/start-then-stop-with-unfence.summary | 3 +-- + pengine/test10/stonith-0.summary | 4 ++-- + pengine/test10/stonith-1.summary | 2 +- + pengine/test10/stonith-2.summary | 2 +- + pengine/test10/stonith-3.summary | 2 +- + pengine/test10/stonith-4.summary | 8 ++++---- + pengine/test10/stop-failure-no-quorum.summary | 2 +- + pengine/test10/stop-failure-with-fencing.summary | 2 +- + pengine/test10/systemhealth1.summary | 4 ++-- + pengine/test10/systemhealth2.summary | 2 +- + pengine/test10/systemhealth3.summary | 2 +- + pengine/test10/systemhealthm1.summary | 4 ++-- + pengine/test10/systemhealthm2.summary | 2 +- + pengine/test10/systemhealthm3.summary | 2 +- + pengine/test10/systemhealthn1.summary | 4 ++-- + pengine/test10/systemhealthn2.summary | 2 +- + pengine/test10/systemhealthn3.summary | 2 +- + pengine/test10/systemhealtho1.summary | 4 ++-- + pengine/test10/systemhealtho2.summary | 2 +- + pengine/test10/systemhealtho3.summary | 2 +- + pengine/test10/systemhealthp1.summary | 4 ++-- + pengine/test10/systemhealthp2.summary | 2 +- + pengine/test10/systemhealthp3.summary | 2 +- + pengine/test10/ticket-clone-21.summary | 4 ++-- + pengine/test10/ticket-clone-9.summary | 4 ++-- + pengine/test10/ticket-group-21.summary | 2 +- + pengine/test10/ticket-group-9.summary | 2 +- + pengine/test10/ticket-master-21.summary | 2 +- + pengine/test10/ticket-master-9.summary | 2 +- + pengine/test10/ticket-primitive-21.summary | 2 +- + pengine/test10/ticket-primitive-9.summary | 2 +- + pengine/test10/unfence-definition.summary | 8 +++----- + pengine/test10/unfence-parameters.summary | 9 ++++----- + pengine/test10/unfence-startup.summary | 7 ++----- + pengine/test10/whitebox-fail1.summary | 2 +- + pengine/test10/whitebox-fail2.summary | 2 +- + pengine/test10/whitebox-imply-stop-on-fence.summary | 6 +++--- + pengine/test10/whitebox-ms-ordering.summary | 4 ++-- + pengine/test10/whitebox-unexpectedly-running.summary | 2 +- + 76 files changed, 113 insertions(+), 116 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 3a883ad..8ea2ea1 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -2419,11 +2419,13 @@ LogNodeActions(pe_working_set_t * data_set, gboolean terminal) + + for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { + char *node_name = NULL; +- const char *task = NULL; ++ char *task = NULL; + action_t *action = (action_t *) gIter->data; + + if (action->rsc != NULL) { + continue; ++ } else if (is_set(action->flags, pe_action_optional)) { ++ continue; + } + + if (is_container_remote_node(action->node)) { +@@ -2433,9 +2435,10 @@ LogNodeActions(pe_working_set_t * data_set, gboolean terminal) + } + + if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) { +- task = "Shutdown"; ++ task = strdup("Shutdown"); + } else if (safe_str_eq(action->task, CRM_OP_FENCE)) { +- task = "Fence"; ++ const char *op = g_hash_table_lookup(action->meta, "stonith_action"); ++ task = crm_strdup_printf("Fence (%s)", op); + } + + if(task == NULL) { +@@ -2447,6 +2450,7 @@ LogNodeActions(pe_working_set_t * data_set, gboolean terminal) + } + + free(node_name); ++ free(task); + } + } + +diff --git a/pengine/test10/594.summary b/pengine/test10/594.summary +index d9fe8c1..ef2a02c 100644 +--- a/pengine/test10/594.summary ++++ b/pengine/test10/594.summary +@@ -13,7 +13,7 @@ Online: [ hadev1 hadev2 ] + child_DoFencing:2 (stonith:ssh): Started hadev1 + + Transition Summary: +- * Fence hadev3 ++ * Fence (reboot) hadev3 + * Shutdown hadev2 + * Move DcIPaddr (Started hadev2 -> hadev1) + * Move rsc_hadev2 (Started hadev2 -> hadev1) +diff --git a/pengine/test10/829.summary b/pengine/test10/829.summary +index a9d25e0..556699f 100644 +--- a/pengine/test10/829.summary ++++ b/pengine/test10/829.summary +@@ -15,7 +15,7 @@ Online: [ c001n01 c001n03 c001n08 ] + child_DoFencing:3 (stonith:ssh): Started c001n08 + + Transition Summary: +- * Fence c001n02 ++ * Fence (reboot) c001n02 + * Move rsc_c001n02 (Started c001n02 -> c001n01) + * Stop child_DoFencing:0 (c001n02) + +diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary +index 63ed2d7..a32e81d 100644 +--- a/pengine/test10/bug-5186-partial-migrate.summary ++++ b/pengine/test10/bug-5186-partial-migrate.summary +@@ -25,7 +25,7 @@ Online: [ bl460g1n6 bl460g1n8 ] + Started: [ bl460g1n6 bl460g1n8 ] + + Transition Summary: +- * Fence bl460g1n7 ++ * Fence (reboot) bl460g1n7 + * Move prmDummy (Started bl460g1n7 -> bl460g1n6) + * Move prmVM2 (Started bl460g1n7 -> bl460g1n8) + * Move prmStonith8-1 (Started bl460g1n7 -> bl460g1n6) +diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary +index 8ea3ff0..f70a9ea 100644 +--- a/pengine/test10/bug-cl-5247.summary ++++ b/pengine/test10/bug-cl-5247.summary +@@ -18,7 +18,7 @@ Containers: [ pgsr01:prmDB1 ] + Stopped: [ bl460g8n3 bl460g8n4 ] + + Transition Summary: +- * Fence pgsr02 (resource: prmDB2) ++ * Fence (off) pgsr02 (resource: prmDB2) + * Stop prmDB2 (bl460g8n4) + * Restart prmStonith1-2 (Started bl460g8n4) + * Restart prmStonith2-2 (Started bl460g8n3) +diff --git a/pengine/test10/bug-lf-2508.summary b/pengine/test10/bug-lf-2508.summary +index 7b436d4..7931d19 100644 +--- a/pengine/test10/bug-lf-2508.summary ++++ b/pengine/test10/bug-lf-2508.summary +@@ -35,7 +35,7 @@ Online: [ srv01 srv03 srv04 ] + Stopped: [ srv04 ] + + Transition Summary: +- * Fence srv02 ++ * Fence (reboot) srv02 + * Start Dummy01 (srv01) + * Move Dummy02 (Started srv02 -> srv04) + * Stop prmStonith1-1:1 (srv02) +diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary +index ffb7c6d..ef2e54a 100644 +--- a/pengine/test10/bug-lf-2551.summary ++++ b/pengine/test10/bug-lf-2551.summary +@@ -81,7 +81,7 @@ Online: [ hex-0 hex-7 hex-8 ] + vm-64 (ocf::heartbeat:Xen): Stopped + + Transition Summary: +- * Fence hex-9 ++ * Fence (reboot) hex-9 + * Move fencing-sbd (Started hex-9 -> hex-0) + * Move dummy1 (Started hex-9 -> hex-0) + * Stop dlm:3 (hex-9) +diff --git a/pengine/test10/bug-lf-2606.summary b/pengine/test10/bug-lf-2606.summary +index ab93bb3..f30a053 100644 +--- a/pengine/test10/bug-lf-2606.summary ++++ b/pengine/test10/bug-lf-2606.summary +@@ -12,7 +12,7 @@ Online: [ node1 ] + Slaves: [ node1 ] + + Transition Summary: +- * Fence node2 ++ * Fence (reboot) node2 + * Stop rsc1 (node2) + * Move rsc2 (Started node2 -> node1) + * Demote rsc3:1 (Master -> Stopped node2) +diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary +index 8fe474a..1c355c0 100644 +--- a/pengine/test10/bug-rh-1097457.summary ++++ b/pengine/test10/bug-rh-1097457.summary +@@ -32,7 +32,7 @@ Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ] + Started: [ lamaVM1 lamaVM2 lamaVM3 ] + + Transition Summary: +- * Fence lamaVM2 (resource: VM2) ++ * Fence (reboot) lamaVM2 (resource: VM2) + * Recover VM2 (Started lama3) + * Recover FSlun3 (Started lamaVM2 -> lama2) + * Restart FAKE4 (Started lamaVM2) +diff --git a/pengine/test10/concurrent-fencing.summary b/pengine/test10/concurrent-fencing.summary +index a274c3b..1e4a817 100644 +--- a/pengine/test10/concurrent-fencing.summary ++++ b/pengine/test10/concurrent-fencing.summary +@@ -8,9 +8,9 @@ Node node3 (uuid3): UNCLEAN (offline) + lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped + + Transition Summary: +- * Fence node3 +- * Fence node2 +- * Fence node1 ++ * Fence (reboot) node3 ++ * Fence (reboot) node2 ++ * Fence (reboot) node1 + + Executing cluster transition: + * Fencing node3 (reboot) +diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary +index 717c43d..9f85613 100644 +--- a/pengine/test10/guest-node-host-dies.summary ++++ b/pengine/test10/guest-node-host-dies.summary +@@ -11,9 +11,9 @@ Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + + Transition Summary: +- * Fence rhel7-1 +- * Fence lxc2 (resource: container2) +- * Fence lxc1 (resource: container1) ++ * Fence (reboot) rhel7-1 ++ * Fence (reboot) lxc2 (resource: container2) ++ * Fence (reboot) lxc1 (resource: container1) + * Restart Fencing (Started rhel7-4) + * Move rsc_rhel7-1 (Started rhel7-1 -> rhel7-5) + * Recover container1 (Started rhel7-1 -> rhel7-2) +diff --git a/pengine/test10/interleave-pseudo-stop.summary b/pengine/test10/interleave-pseudo-stop.summary +index cf30da0..12be956 100644 +--- a/pengine/test10/interleave-pseudo-stop.summary ++++ b/pengine/test10/interleave-pseudo-stop.summary +@@ -17,7 +17,7 @@ Online: [ node2 ] + Started: [ node2 ] + + Transition Summary: +- * Fence node1 ++ * Fence (reboot) node1 + * Stop stonithclone:1 (node1) + * Stop evmsclone:1 (node1) + * Stop imagestoreclone:1 (node1) +diff --git a/pengine/test10/master-7.summary b/pengine/test10/master-7.summary +index 348b4ee..2889efb 100644 +--- a/pengine/test10/master-7.summary ++++ b/pengine/test10/master-7.summary +@@ -29,7 +29,7 @@ Online: [ c001n02 c001n03 c001n08 ] + ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08 + + Transition Summary: +- * Fence c001n01 ++ * Fence (reboot) c001n01 + * Move DcIPaddr (Started c001n01 -> c001n03) + * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) + * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) +diff --git a/pengine/test10/master-8.summary b/pengine/test10/master-8.summary +index b77c884..5968a25 100644 +--- a/pengine/test10/master-8.summary ++++ b/pengine/test10/master-8.summary +@@ -29,7 +29,7 @@ Online: [ c001n02 c001n03 c001n08 ] + ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08 + + Transition Summary: +- * Fence c001n01 ++ * Fence (reboot) c001n01 + * Move DcIPaddr (Started c001n01 -> c001n03) + * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) + * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) +diff --git a/pengine/test10/migrate-fencing.summary b/pengine/test10/migrate-fencing.summary +index 831e49a..842c65d 100644 +--- a/pengine/test10/migrate-fencing.summary ++++ b/pengine/test10/migrate-fencing.summary +@@ -22,7 +22,7 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Slaves: [ pcmk-1 pcmk-2 pcmk-3 ] + + Transition Summary: +- * Fence pcmk-4 ++ * Fence (reboot) pcmk-4 + * Stop FencingChild:0 (pcmk-4) + * Move r192.168.101.181 (Started pcmk-4 -> pcmk-1) + * Move r192.168.101.182 (Started pcmk-4 -> pcmk-1) +diff --git a/pengine/test10/rec-node-11.summary b/pengine/test10/rec-node-11.summary +index eb967dd..22f5af7 100644 +--- a/pengine/test10/rec-node-11.summary ++++ b/pengine/test10/rec-node-11.summary +@@ -10,7 +10,7 @@ Online: [ node2 ] + rsc3 (heartbeat:apache): Started node2 + + Transition Summary: +- * Fence node1 ++ * Fence (reboot) node1 + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +diff --git a/pengine/test10/rec-node-12.summary b/pengine/test10/rec-node-12.summary +index 26f9b13..6316fdb 100644 +--- a/pengine/test10/rec-node-12.summary ++++ b/pengine/test10/rec-node-12.summary +@@ -15,7 +15,7 @@ Online: [ c001n01 c001n03 c001n08 ] + child_DoFencing:3 (stonith:ssh): Stopped + + Transition Summary: +- * Fence c001n02 ++ * Fence (reboot) c001n02 + * Start DcIPaddr (c001n08) + * Start rsc_c001n08 (c001n08) + * Start rsc_c001n02 (c001n01) +diff --git a/pengine/test10/rec-node-13.summary b/pengine/test10/rec-node-13.summary +index e273a79..c222ad8 100644 +--- a/pengine/test10/rec-node-13.summary ++++ b/pengine/test10/rec-node-13.summary +@@ -34,7 +34,7 @@ OFFLINE: [ c001n03 c001n05 ] + ocf_msdummy:11 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n07 + + Transition Summary: +- * Fence c001n04 ++ * Fence (reboot) c001n04 + * Stop ocf_msdummy:6 (c001n04) + + Executing cluster transition: +diff --git a/pengine/test10/rec-node-14.summary b/pengine/test10/rec-node-14.summary +index 532f3d9..f503a92 100644 +--- a/pengine/test10/rec-node-14.summary ++++ b/pengine/test10/rec-node-14.summary +@@ -8,9 +8,9 @@ Node node3 (uuid3): UNCLEAN (offline) + lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped + + Transition Summary: +- * Fence node3 +- * Fence node2 +- * Fence node1 ++ * Fence (reboot) node3 ++ * Fence (reboot) node2 ++ * Fence (reboot) node1 + + Executing cluster transition: + * Fencing node1 (reboot) +diff --git a/pengine/test10/rec-node-15.summary b/pengine/test10/rec-node-15.summary +index 8d88680..2f706d2 100644 +--- a/pengine/test10/rec-node-15.summary ++++ b/pengine/test10/rec-node-15.summary +@@ -21,7 +21,7 @@ Online: [ sapcl01 ] + oralsnr_25 (ocf::heartbeat:oralsnr): Stopped + + Transition Summary: +- * Fence sapcl03 ++ * Fence (reboot) sapcl03 + * Start stonith-1 (sapcl01) + * Move IPaddr_192_168_1_102 (Started sapcl02 -> sapcl01) + * Move LVM_12 (Started sapcl02 -> sapcl01) +diff --git a/pengine/test10/rec-node-2.summary b/pengine/test10/rec-node-2.summary +index 5c8db02..0e0183f 100644 +--- a/pengine/test10/rec-node-2.summary ++++ b/pengine/test10/rec-node-2.summary +@@ -14,7 +14,7 @@ Online: [ node2 ] + rsc6 (heartbeat:apache): Stopped + + Transition Summary: +- * Fence node1 ++ * Fence (reboot) node1 + * Start stonith-1 (node2) + * Start rsc1 (node2) + * Start rsc2 (node2) +diff --git a/pengine/test10/rec-node-4.summary b/pengine/test10/rec-node-4.summary +index 761573f..0a2606e 100644 +--- a/pengine/test10/rec-node-4.summary ++++ b/pengine/test10/rec-node-4.summary +@@ -8,7 +8,7 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + + Transition Summary: +- * Fence node1 ++ * Fence (reboot) node1 + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +diff --git a/pengine/test10/rec-node-6.summary b/pengine/test10/rec-node-6.summary +index fb294fb..530c7d2 100644 +--- a/pengine/test10/rec-node-6.summary ++++ b/pengine/test10/rec-node-6.summary +@@ -8,7 +8,7 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 + + Transition Summary: +- * Fence node1 ++ * Fence (reboot) node1 + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +diff --git a/pengine/test10/rec-node-7.summary b/pengine/test10/rec-node-7.summary +index 761573f..0a2606e 100644 +--- a/pengine/test10/rec-node-7.summary ++++ b/pengine/test10/rec-node-7.summary +@@ -8,7 +8,7 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + + Transition Summary: +- * Fence node1 ++ * Fence (reboot) node1 + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +diff --git a/pengine/test10/rec-rsc-5.summary b/pengine/test10/rec-rsc-5.summary +index 7bcb1a3..c3e658a 100644 +--- a/pengine/test10/rec-rsc-5.summary ++++ b/pengine/test10/rec-rsc-5.summary +@@ -8,7 +8,7 @@ Online: [ node1 ] + rsc2 (heartbeat:apache): Started node2 + + Transition Summary: +- * Fence node2 ++ * Fence (reboot) node2 + * Start stonith-1 (node1) + * Recover rsc1 (Started node2 -> node1) + * Move rsc2 (Started node2 -> node1) +diff --git a/pengine/test10/remote-fence-before-reconnect.summary b/pengine/test10/remote-fence-before-reconnect.summary +index 520f5cf..2ce3315 100644 +--- a/pengine/test10/remote-fence-before-reconnect.summary ++++ b/pengine/test10/remote-fence-before-reconnect.summary +@@ -12,7 +12,7 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + fake5 (ocf::heartbeat:Dummy): Started c7auto3 + + Transition Summary: +- * Fence c7auto4 ++ * Fence (reboot) c7auto4 + * Stop c7auto4 (c7auto1) + * Move fake2 (Started c7auto4 -> c7auto1) + +diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary +index ec24500..9296add 100644 +--- a/pengine/test10/remote-fence-unclean-3.summary ++++ b/pengine/test10/remote-fence-unclean-3.summary +@@ -34,7 +34,7 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund + openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1 + + Transition Summary: +- * Fence overcloud-novacompute-0 ++ * Fence (reboot) overcloud-novacompute-0 + * Start fence1 (overcloud-controller-0) + * Stop overcloud-novacompute-0 (overcloud-controller-0) + +diff --git a/pengine/test10/remote-fence-unclean.summary b/pengine/test10/remote-fence-unclean.summary +index 06940fc..667549b 100644 +--- a/pengine/test10/remote-fence-unclean.summary ++++ b/pengine/test10/remote-fence-unclean.summary +@@ -11,7 +11,7 @@ Online: [ 18builder 18node1 18node2 ] + FAKE4 (ocf::heartbeat:Dummy): Started 18node1 + + Transition Summary: +- * Fence remote1 ++ * Fence (reboot) remote1 + * Recover remote1 (Started 18node1) + * Move FAKE2 (Started remote1 -> 18builder) + * Move FAKE3 (Started 18builder -> 18node1) +diff --git a/pengine/test10/remote-fence-unclean2.summary b/pengine/test10/remote-fence-unclean2.summary +index 96f5d69..0844c29 100644 +--- a/pengine/test10/remote-fence-unclean2.summary ++++ b/pengine/test10/remote-fence-unclean2.summary +@@ -10,7 +10,7 @@ OFFLINE: [ rhel7-alt3 ] + fake (ocf::heartbeat:Dummy): Started rhel7-alt4 (UNCLEAN) + + Transition Summary: +- * Fence rhel7-alt4 ++ * Fence (reboot) rhel7-alt4 + * Stop fake (rhel7-alt4) + + Executing cluster transition: +diff --git a/pengine/test10/remote-partial-migrate2.summary b/pengine/test10/remote-partial-migrate2.summary +index 197bd8c..b8b9b4c 100644 +--- a/pengine/test10/remote-partial-migrate2.summary ++++ b/pengine/test10/remote-partial-migrate2.summary +@@ -63,7 +63,7 @@ RemoteOFFLINE: [ pcmk_remote4 ] + FAKE50 (ocf::heartbeat:Dummy): Started pcmk_remote5 + + Transition Summary: +- * Fence pcmk4 ++ * Fence (reboot) pcmk4 + * Migrate pcmk_remote2 (Started pcmk3 -> pcmk1) + * Start pcmk_remote4 (pcmk2) + * Migrate pcmk_remote5 (Started pcmk1 -> pcmk2) +diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary +index 3f29f70..5e4f51d 100644 +--- a/pengine/test10/remote-recover-all.summary ++++ b/pengine/test10/remote-recover-all.summary +@@ -38,9 +38,9 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: +- * Fence messaging-1 +- * Fence galera-2 +- * Fence controller-1 ++ * Fence (reboot) messaging-1 ++ * Fence (reboot) galera-2 ++ * Fence (reboot) controller-1 + * Stop messaging-1 (controller-1) + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +diff --git a/pengine/test10/remote-recover-connection.summary b/pengine/test10/remote-recover-connection.summary +index 43507af..79ea7da 100644 +--- a/pengine/test10/remote-recover-connection.summary ++++ b/pengine/test10/remote-recover-connection.summary +@@ -38,7 +38,7 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: +- * Fence controller-1 ++ * Fence (reboot) controller-1 + * Move messaging-1 (Started controller-1 -> controller-2) + * Move galera-0 (Started controller-1 -> controller-2) + * Move galera-2 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-recover-fail.summary b/pengine/test10/remote-recover-fail.summary +index ec2d701..313c8ac 100644 +--- a/pengine/test10/remote-recover-fail.summary ++++ b/pengine/test10/remote-recover-fail.summary +@@ -14,7 +14,7 @@ OFFLINE: [ rhel7-auto1 ] + FAKE6 (ocf::heartbeat:Dummy): Started rhel7-auto4 (UNCLEAN) + + Transition Summary: +- * Fence rhel7-auto4 ++ * Fence (reboot) rhel7-auto4 + * Recover rhel7-auto4 (Started rhel7-auto2) + * Start FAKE1 (rhel7-auto2) + * Move FAKE2 (Started rhel7-auto4 -> rhel7-auto3) +diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary +index 48355a4..bd27773 100644 +--- a/pengine/test10/remote-recover-no-resources.summary ++++ b/pengine/test10/remote-recover-no-resources.summary +@@ -38,8 +38,8 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: +- * Fence messaging-1 +- * Fence controller-1 ++ * Fence (reboot) messaging-1 ++ * Fence (reboot) controller-1 + * Stop messaging-1 (controller-1) + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary +index 212af4e..8a3d560 100644 +--- a/pengine/test10/remote-recover-unknown.summary ++++ b/pengine/test10/remote-recover-unknown.summary +@@ -38,9 +38,9 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: +- * Fence messaging-1 +- * Fence galera-2 +- * Fence controller-1 ++ * Fence (reboot) messaging-1 ++ * Fence (reboot) galera-2 ++ * Fence (reboot) controller-1 + * Stop messaging-1 (controller-1) + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +diff --git a/pengine/test10/remote-recovery.summary b/pengine/test10/remote-recovery.summary +index 43507af..79ea7da 100644 +--- a/pengine/test10/remote-recovery.summary ++++ b/pengine/test10/remote-recovery.summary +@@ -38,7 +38,7 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + + Transition Summary: +- * Fence controller-1 ++ * Fence (reboot) controller-1 + * Move messaging-1 (Started controller-1 -> controller-2) + * Move galera-0 (Started controller-1 -> controller-2) + * Move galera-2 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-unclean2.summary b/pengine/test10/remote-unclean2.summary +index ad4af90..877eb4d 100644 +--- a/pengine/test10/remote-unclean2.summary ++++ b/pengine/test10/remote-unclean2.summary +@@ -7,7 +7,7 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ] + rhel7-auto4 (ocf::pacemaker:remote): FAILED rhel7-auto1 + + Transition Summary: +- * Fence rhel7-auto4 ++ * Fence (reboot) rhel7-auto4 + * Recover rhel7-auto4 (Started rhel7-auto1) + + Executing cluster transition: +diff --git a/pengine/test10/start-then-stop-with-unfence.summary b/pengine/test10/start-then-stop-with-unfence.summary +index 4430cc0..ae54afc 100644 +--- a/pengine/test10/start-then-stop-with-unfence.summary ++++ b/pengine/test10/start-then-stop-with-unfence.summary +@@ -11,8 +11,7 @@ Online: [ rhel7-node1.example.com rhel7-node2.example.com ] + Stopped: [ rhel7-node1.example.com ] + + Transition Summary: +- * Fence rhel7-node2.example.com +- * Fence rhel7-node1.example.com ++ * Fence (on) rhel7-node1.example.com + * Start mpath-node1 (rhel7-node1.example.com) + * Move ip1 (Started rhel7-node2.example.com -> rhel7-node1.example.com) + * Start jrummy:1 (rhel7-node1.example.com) +diff --git a/pengine/test10/stonith-0.summary b/pengine/test10/stonith-0.summary +index 24008a1..ee8ae15 100644 +--- a/pengine/test10/stonith-0.summary ++++ b/pengine/test10/stonith-0.summary +@@ -37,8 +37,8 @@ Online: [ c001n02 c001n04 c001n06 c001n07 c001n08 ] + ocf_msdummy:13 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n06 + + Transition Summary: +- * Fence c001n05 +- * Fence c001n03 ++ * Fence (reboot) c001n05 ++ * Fence (reboot) c001n03 + * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) + * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) + * Recover ocf_192.168.100.183 (Started c001n03 -> c001n02) +diff --git a/pengine/test10/stonith-1.summary b/pengine/test10/stonith-1.summary +index 8556021..b68aca7 100644 +--- a/pengine/test10/stonith-1.summary ++++ b/pengine/test10/stonith-1.summary +@@ -28,7 +28,7 @@ Online: [ sles-1 sles-2 sles-4 ] + ocf_msdummy:7 (ocf::heartbeat:Stateful): Stopped + + Transition Summary: +- * Fence sles-3 ++ * Fence (reboot) sles-3 + * Start r192.168.100.183 (sles-1) + * Move migrator (Started sles-3 -> sles-4) + * Move rsc_sles-3 (Started sles-3 -> sles-4) +diff --git a/pengine/test10/stonith-2.summary b/pengine/test10/stonith-2.summary +index 0f7cb99..eef35c4 100644 +--- a/pengine/test10/stonith-2.summary ++++ b/pengine/test10/stonith-2.summary +@@ -33,7 +33,7 @@ Online: [ sles-1 sles-2 sles-3 sles-4 sles-6 ] + ocf_msdummy:11 (ocf::heartbeat:Stateful): Slave sles-3 + + Transition Summary: +- * Fence sles-5 ++ * Fence (reboot) sles-5 + * Start rsc_sles-5 (sles-6) + + Executing cluster transition: +diff --git a/pengine/test10/stonith-3.summary b/pengine/test10/stonith-3.summary +index cb25cc3..398b6bc 100644 +--- a/pengine/test10/stonith-3.summary ++++ b/pengine/test10/stonith-3.summary +@@ -8,7 +8,7 @@ Online: [ rh5node2 ] + Stopped: [ rh5node1 rh5node2 ] + + Transition Summary: +- * Fence rh5node1 ++ * Fence (reboot) rh5node1 + * Start prmIpPostgreSQLDB (rh5node2) + * Start prmStonith:0 (rh5node2) + +diff --git a/pengine/test10/stonith-4.summary b/pengine/test10/stonith-4.summary +index dc7cd89..6e3e396 100644 +--- a/pengine/test10/stonith-4.summary ++++ b/pengine/test10/stonith-4.summary +@@ -14,10 +14,10 @@ OFFLINE: [ pcmk-4 pcmk-6 ] + Fencing (stonith:fence_xvm): Stopped + + Transition Summary: +- * Fence pcmk-8 +- * Fence pcmk-7 +- * Fence pcmk-5 +- * Fence pcmk-10 ++ * Fence (reboot) pcmk-8 ++ * Fence (reboot) pcmk-7 ++ * Fence (reboot) pcmk-5 ++ * Fence (reboot) pcmk-10 + * Start Fencing (pcmk-1 - blocked) + + Executing cluster transition: +diff --git a/pengine/test10/stop-failure-no-quorum.summary b/pengine/test10/stop-failure-no-quorum.summary +index d864f1a..0248126 100644 +--- a/pengine/test10/stop-failure-no-quorum.summary ++++ b/pengine/test10/stop-failure-no-quorum.summary +@@ -15,7 +15,7 @@ Online: [ pcmk-1 ] + Fencing (stonith:fence_xvm): Stopped + + Transition Summary: +- * Fence pcmk-2 ++ * Fence (reboot) pcmk-2 + * Start dlm:0 (pcmk-1 - blocked) + * Stop clvm:0 (pcmk-2) + * Start clvm:2 (pcmk-1 - blocked) +diff --git a/pengine/test10/stop-failure-with-fencing.summary b/pengine/test10/stop-failure-with-fencing.summary +index e01b6c4..e6c296b 100644 +--- a/pengine/test10/stop-failure-with-fencing.summary ++++ b/pengine/test10/stop-failure-with-fencing.summary +@@ -14,7 +14,7 @@ Online: [ pcmk-1 ] + Fencing (stonith:fence_xvm): Stopped + + Transition Summary: +- * Fence pcmk-2 ++ * Fence (reboot) pcmk-2 + * Start dlm:0 (pcmk-1 - blocked) + * Stop clvm:0 (pcmk-2) + * Start clvm:1 (pcmk-1 - blocked) +diff --git a/pengine/test10/systemhealth1.summary b/pengine/test10/systemhealth1.summary +index 37b0b49..7301c6c 100644 +--- a/pengine/test10/systemhealth1.summary ++++ b/pengine/test10/systemhealth1.summary +@@ -8,8 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d +- * Fence hs21c ++ * Fence (reboot) hs21d ++ * Fence (reboot) hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealth2.summary b/pengine/test10/systemhealth2.summary +index a37ce18..c83a7e2 100644 +--- a/pengine/test10/systemhealth2.summary ++++ b/pengine/test10/systemhealth2.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealth3.summary b/pengine/test10/systemhealth3.summary +index a37ce18..c83a7e2 100644 +--- a/pengine/test10/systemhealth3.summary ++++ b/pengine/test10/systemhealth3.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealthm1.summary b/pengine/test10/systemhealthm1.summary +index 37b0b49..7301c6c 100644 +--- a/pengine/test10/systemhealthm1.summary ++++ b/pengine/test10/systemhealthm1.summary +@@ -8,8 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d +- * Fence hs21c ++ * Fence (reboot) hs21d ++ * Fence (reboot) hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealthm2.summary b/pengine/test10/systemhealthm2.summary +index a37ce18..c83a7e2 100644 +--- a/pengine/test10/systemhealthm2.summary ++++ b/pengine/test10/systemhealthm2.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealthm3.summary b/pengine/test10/systemhealthm3.summary +index b10d3f2..fbcdec5 100644 +--- a/pengine/test10/systemhealthm3.summary ++++ b/pengine/test10/systemhealthm3.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/systemhealthn1.summary b/pengine/test10/systemhealthn1.summary +index 37b0b49..7301c6c 100644 +--- a/pengine/test10/systemhealthn1.summary ++++ b/pengine/test10/systemhealthn1.summary +@@ -8,8 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d +- * Fence hs21c ++ * Fence (reboot) hs21d ++ * Fence (reboot) hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealthn2.summary b/pengine/test10/systemhealthn2.summary +index a37ce18..c83a7e2 100644 +--- a/pengine/test10/systemhealthn2.summary ++++ b/pengine/test10/systemhealthn2.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealthn3.summary b/pengine/test10/systemhealthn3.summary +index a37ce18..c83a7e2 100644 +--- a/pengine/test10/systemhealthn3.summary ++++ b/pengine/test10/systemhealthn3.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealtho1.summary b/pengine/test10/systemhealtho1.summary +index 37b0b49..7301c6c 100644 +--- a/pengine/test10/systemhealtho1.summary ++++ b/pengine/test10/systemhealtho1.summary +@@ -8,8 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d +- * Fence hs21c ++ * Fence (reboot) hs21d ++ * Fence (reboot) hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealtho2.summary b/pengine/test10/systemhealtho2.summary +index b10d3f2..fbcdec5 100644 +--- a/pengine/test10/systemhealtho2.summary ++++ b/pengine/test10/systemhealtho2.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/systemhealtho3.summary b/pengine/test10/systemhealtho3.summary +index b10d3f2..fbcdec5 100644 +--- a/pengine/test10/systemhealtho3.summary ++++ b/pengine/test10/systemhealtho3.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/systemhealthp1.summary b/pengine/test10/systemhealthp1.summary +index 37b0b49..7301c6c 100644 +--- a/pengine/test10/systemhealthp1.summary ++++ b/pengine/test10/systemhealthp1.summary +@@ -8,8 +8,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d +- * Fence hs21c ++ * Fence (reboot) hs21d ++ * Fence (reboot) hs21c + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealthp2.summary b/pengine/test10/systemhealthp2.summary +index 6605e12..ea9465b 100644 +--- a/pengine/test10/systemhealthp2.summary ++++ b/pengine/test10/systemhealthp2.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) + +diff --git a/pengine/test10/systemhealthp3.summary b/pengine/test10/systemhealthp3.summary +index b10d3f2..fbcdec5 100644 +--- a/pengine/test10/systemhealthp3.summary ++++ b/pengine/test10/systemhealthp3.summary +@@ -8,7 +8,7 @@ Online: [ hs21c ] + nfs_1 (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Fence hs21d ++ * Fence (reboot) hs21d + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/ticket-clone-21.summary b/pengine/test10/ticket-clone-21.summary +index 50df628..573f8c1 100644 +--- a/pengine/test10/ticket-clone-21.summary ++++ b/pengine/test10/ticket-clone-21.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Fence node2 +- * Fence node1 ++ * Fence (reboot) node2 ++ * Fence (reboot) node1 + * Stop rsc_stonith (node1) + * Stop rsc1:0 (node1) + * Stop rsc1:1 (node2) +diff --git a/pengine/test10/ticket-clone-9.summary b/pengine/test10/ticket-clone-9.summary +index 50df628..573f8c1 100644 +--- a/pengine/test10/ticket-clone-9.summary ++++ b/pengine/test10/ticket-clone-9.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Fence node2 +- * Fence node1 ++ * Fence (reboot) node2 ++ * Fence (reboot) node1 + * Stop rsc_stonith (node1) + * Stop rsc1:0 (node1) + * Stop rsc1:1 (node2) +diff --git a/pengine/test10/ticket-group-21.summary b/pengine/test10/ticket-group-21.summary +index eab14a9..ff809ae 100644 +--- a/pengine/test10/ticket-group-21.summary ++++ b/pengine/test10/ticket-group-21.summary +@@ -8,7 +8,7 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Fence node2 ++ * Fence (reboot) node2 + * Stop rsc1 (node2) + * Stop rsc2 (node2) + +diff --git a/pengine/test10/ticket-group-9.summary b/pengine/test10/ticket-group-9.summary +index eab14a9..ff809ae 100644 +--- a/pengine/test10/ticket-group-9.summary ++++ b/pengine/test10/ticket-group-9.summary +@@ -8,7 +8,7 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Fence node2 ++ * Fence (reboot) node2 + * Stop rsc1 (node2) + * Stop rsc2 (node2) + +diff --git a/pengine/test10/ticket-master-21.summary b/pengine/test10/ticket-master-21.summary +index a107a38..c196c9d 100644 +--- a/pengine/test10/ticket-master-21.summary ++++ b/pengine/test10/ticket-master-21.summary +@@ -8,7 +8,7 @@ Online: [ node1 node2 ] + Slaves: [ node2 ] + + Transition Summary: +- * Fence node1 ++ * Fence (reboot) node1 + * Move rsc_stonith (Started node1 -> node2) + * Demote rsc1:0 (Master -> Stopped node1) + +diff --git a/pengine/test10/ticket-master-9.summary b/pengine/test10/ticket-master-9.summary +index a107a38..c196c9d 100644 +--- a/pengine/test10/ticket-master-9.summary ++++ b/pengine/test10/ticket-master-9.summary +@@ -8,7 +8,7 @@ Online: [ node1 node2 ] + Slaves: [ node2 ] + + Transition Summary: +- * Fence node1 ++ * Fence (reboot) node1 + * Move rsc_stonith (Started node1 -> node2) + * Demote rsc1:0 (Master -> Stopped node1) + +diff --git a/pengine/test10/ticket-primitive-21.summary b/pengine/test10/ticket-primitive-21.summary +index 918b743..08a4860 100644 +--- a/pengine/test10/ticket-primitive-21.summary ++++ b/pengine/test10/ticket-primitive-21.summary +@@ -6,7 +6,7 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Fence node2 ++ * Fence (reboot) node2 + * Stop rsc1 (node2) + + Executing cluster transition: +diff --git a/pengine/test10/ticket-primitive-9.summary b/pengine/test10/ticket-primitive-9.summary +index 918b743..08a4860 100644 +--- a/pengine/test10/ticket-primitive-9.summary ++++ b/pengine/test10/ticket-primitive-9.summary +@@ -6,7 +6,7 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Fence node2 ++ * Fence (reboot) node2 + * Stop rsc1 (node2) + + Executing cluster transition: +diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary +index 03506a3..9477a02 100644 +--- a/pengine/test10/unfence-definition.summary ++++ b/pengine/test10/unfence-definition.summary +@@ -12,11 +12,9 @@ Online: [ virt-1 virt-2 virt-3 ] + Stopped: [ virt-2 virt-3 virt-4 ] + + Transition Summary: +- * Fence virt-4 +- * Fence virt-4 +- * Fence virt-3 +- * Fence virt-2 +- * Fence virt-1 ++ * Fence (reboot) virt-4 ++ * Fence (on) virt-3 ++ * Fence (on) virt-1 + * Restart fencing (Started virt-1) + * Restart dlm:0 (Started virt-1) + * Start dlm:2 (virt-3) +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +index 5cc4a4b..a713cd5 100644 +--- a/pengine/test10/unfence-parameters.summary ++++ b/pengine/test10/unfence-parameters.summary +@@ -12,11 +12,10 @@ Online: [ virt-1 virt-2 virt-3 ] + Stopped: [ virt-2 virt-3 virt-4 ] + + Transition Summary: +- * Fence virt-4 +- * Fence virt-4 +- * Fence virt-3 +- * Fence virt-2 +- * Fence virt-1 ++ * Fence (reboot) virt-4 ++ * Fence (on) virt-3 ++ * Fence (on) virt-2 ++ * Fence (on) virt-1 + * Restart fencing (Started virt-1) + * Restart dlm:0 (Started virt-1) + * Restart dlm:1 (Started virt-2) +diff --git a/pengine/test10/unfence-startup.summary b/pengine/test10/unfence-startup.summary +index 6cd38ad..c3f1be4 100644 +--- a/pengine/test10/unfence-startup.summary ++++ b/pengine/test10/unfence-startup.summary +@@ -12,11 +12,8 @@ Online: [ virt-1 virt-2 virt-3 ] + Stopped: [ virt-2 virt-3 virt-4 ] + + Transition Summary: +- * Fence virt-4 +- * Fence virt-4 +- * Fence virt-3 +- * Fence virt-2 +- * Fence virt-1 ++ * Fence (reboot) virt-4 ++ * Fence (on) virt-3 + * Start dlm:2 (virt-3) + * Start clvmd:1 (virt-2) + * Start clvmd:2 (virt-3) +diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary +index 5f70a66..57d72ba 100644 +--- a/pengine/test10/whitebox-fail1.summary ++++ b/pengine/test10/whitebox-fail1.summary +@@ -14,7 +14,7 @@ Containers: [ lxc2:container2 ] + D (ocf::pacemaker:Dummy): Started 18node1 + + Transition Summary: +- * Fence lxc1 (resource: container1) ++ * Fence (reboot) lxc1 (resource: container1) + * Recover container1 (Started 18node2) + * Recover M:4 (Started lxc1) + * Recover B (Started lxc1) +diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary +index 2922f16..bf12683 100644 +--- a/pengine/test10/whitebox-fail2.summary ++++ b/pengine/test10/whitebox-fail2.summary +@@ -14,7 +14,7 @@ Containers: [ lxc2:container2 ] + D (ocf::pacemaker:Dummy): Started 18node1 + + Transition Summary: +- * Fence lxc1 (resource: container1) ++ * Fence (reboot) lxc1 (resource: container1) + * Recover container1 (Started 18node2) + * Recover M:4 (Started lxc1) + * Recover B (Started lxc1) +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary +index 31cc4a5..2fb20a6 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.summary ++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary +@@ -25,9 +25,9 @@ Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ] + vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01 + + Transition Summary: +- * Fence lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) +- * Fence lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) +- * Fence kiff-01 ++ * Fence (reboot) lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) ++ * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) ++ * Fence (reboot) kiff-01 + * Move fence-kiff-02 (Started kiff-01 -> kiff-02) + * Stop dlm:0 (kiff-01) + * Stop clvmd:0 (kiff-01) +diff --git a/pengine/test10/whitebox-ms-ordering.summary b/pengine/test10/whitebox-ms-ordering.summary +index fcdef66..2230d0f 100644 +--- a/pengine/test10/whitebox-ms-ordering.summary ++++ b/pengine/test10/whitebox-ms-ordering.summary +@@ -9,8 +9,8 @@ Online: [ 18node1 18node2 18node3 ] + Stopped: [ 18node1 18node2 18node3 ] + + Transition Summary: +- * Fence lxc2 (resource: container2) +- * Fence lxc1 (resource: container1) ++ * Fence (reboot) lxc2 (resource: container2) ++ * Fence (reboot) lxc1 (resource: container1) + * Start container1 (18node1) + * Start container2 (18node1) + * Recover lxc-ms:0 (Master lxc1) +diff --git a/pengine/test10/whitebox-unexpectedly-running.summary b/pengine/test10/whitebox-unexpectedly-running.summary +index ed0a5bd..eabeb4d 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.summary ++++ b/pengine/test10/whitebox-unexpectedly-running.summary +@@ -5,7 +5,7 @@ Online: [ 18builder ] + FAKE (ocf::pacemaker:Dummy): FAILED 18builder + + Transition Summary: +- * Fence remote1 (resource: FAKE) ++ * Fence (reboot) remote1 (resource: FAKE) + * Recover FAKE (Started 18builder) + * Start remote1 (18builder) + +-- +1.8.3.1 + + +From 7f8ba3076f180aae699df78609b0573e56a7c379 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 26 Jul 2017 10:34:48 +1000 +Subject: [PATCH 02/12] Support unfencing of remote nodes + +--- + crmd/te_callbacks.c | 21 +- + include/crm/msg_xml.h | 1 + + include/crm/pengine/internal.h | 2 +- + lib/pengine/utils.c | 291 +++++++++++++++------ + pengine/allocate.c | 4 +- + pengine/native.c | 39 ++- + pengine/test10/594.scores | 1 + + pengine/test10/594.summary | 1 + + pengine/test10/829.scores | 1 + + pengine/test10/829.summary | 1 + + pengine/test10/bug-5186-partial-migrate.scores | 1 + + pengine/test10/bug-5186-partial-migrate.summary | 1 + + pengine/test10/bug-cl-5247.scores | 1 + + pengine/test10/bug-cl-5247.summary | 1 + + pengine/test10/bug-lf-2508.scores | 1 + + pengine/test10/bug-lf-2508.summary | 1 + + pengine/test10/bug-lf-2551.scores | 1 + + pengine/test10/bug-lf-2551.summary | 1 + + pengine/test10/bug-lf-2606.scores | 1 + + pengine/test10/bug-lf-2606.summary | 1 + + pengine/test10/bug-rh-1097457.scores | 1 + + pengine/test10/bug-rh-1097457.summary | 1 + + pengine/test10/concurrent-fencing.scores | 3 + + pengine/test10/concurrent-fencing.summary | 3 + + pengine/test10/guest-node-host-dies.scores | 3 + + pengine/test10/guest-node-host-dies.summary | 3 + + pengine/test10/interleave-pseudo-stop.scores | 1 + + pengine/test10/interleave-pseudo-stop.summary | 1 + + pengine/test10/master-7.scores | 1 + + pengine/test10/master-7.summary | 1 + + pengine/test10/master-8.scores | 1 + + pengine/test10/master-8.summary | 1 + + pengine/test10/migrate-fencing.scores | 1 + + pengine/test10/migrate-fencing.summary | 1 + + pengine/test10/rec-node-11.scores | 1 + + pengine/test10/rec-node-11.summary | 1 + + pengine/test10/rec-node-12.scores | 1 + + pengine/test10/rec-node-12.summary | 1 + + pengine/test10/rec-node-13.scores | 1 + + pengine/test10/rec-node-13.summary | 1 + + pengine/test10/rec-node-14.scores | 3 + + pengine/test10/rec-node-14.summary | 3 + + pengine/test10/rec-node-15.scores | 1 + + pengine/test10/rec-node-15.summary | 1 + + pengine/test10/rec-node-2.scores | 1 + + pengine/test10/rec-node-2.summary | 1 + + pengine/test10/rec-node-4.scores | 1 + + pengine/test10/rec-node-4.summary | 1 + + pengine/test10/rec-node-6.scores | 1 + + pengine/test10/rec-node-6.summary | 1 + + pengine/test10/rec-node-7.scores | 1 + + pengine/test10/rec-node-7.summary | 1 + + pengine/test10/rec-rsc-5.scores | 1 + + pengine/test10/rec-rsc-5.summary | 1 + + .../test10/remote-fence-before-reconnect.scores | 1 + + .../test10/remote-fence-before-reconnect.summary | 1 + + pengine/test10/remote-fence-unclean-3.scores | 1 + + pengine/test10/remote-fence-unclean-3.summary | 1 + + pengine/test10/remote-fence-unclean.scores | 1 + + pengine/test10/remote-fence-unclean.summary | 1 + + pengine/test10/remote-fence-unclean2.scores | 1 + + pengine/test10/remote-fence-unclean2.summary | 1 + + pengine/test10/remote-partial-migrate2.scores | 1 + + pengine/test10/remote-partial-migrate2.summary | 1 + + pengine/test10/remote-recover-all.scores | 3 + + pengine/test10/remote-recover-all.summary | 3 + + pengine/test10/remote-recover-connection.scores | 1 + + pengine/test10/remote-recover-connection.summary | 1 + + pengine/test10/remote-recover-fail.scores | 1 + + pengine/test10/remote-recover-fail.summary | 1 + + pengine/test10/remote-recover-no-resources.scores | 2 + + pengine/test10/remote-recover-no-resources.summary | 2 + + pengine/test10/remote-recover-unknown.scores | 3 + + pengine/test10/remote-recover-unknown.summary | 3 + + pengine/test10/remote-recovery.scores | 1 + + pengine/test10/remote-recovery.summary | 1 + + pengine/test10/remote-unclean2.scores | 1 + + pengine/test10/remote-unclean2.summary | 1 + + pengine/test10/start-then-stop-with-unfence.dot | 2 - + pengine/test10/start-then-stop-with-unfence.exp | 16 +- + pengine/test10/start-then-stop-with-unfence.scores | 1 + + .../test10/start-then-stop-with-unfence.summary | 9 +- + pengine/test10/start-then-stop-with-unfence.xml | 1 + + pengine/test10/stonith-0.scores | 2 + + pengine/test10/stonith-0.summary | 2 + + pengine/test10/stonith-1.scores | 1 + + pengine/test10/stonith-1.summary | 1 + + pengine/test10/stonith-2.scores | 1 + + pengine/test10/stonith-2.summary | 1 + + pengine/test10/stonith-3.scores | 1 + + pengine/test10/stonith-3.summary | 1 + + pengine/test10/stonith-4.scores | 4 + + pengine/test10/stonith-4.summary | 4 + + pengine/test10/stop-failure-no-quorum.scores | 1 + + pengine/test10/stop-failure-no-quorum.summary | 1 + + pengine/test10/stop-failure-with-fencing.scores | 1 + + pengine/test10/stop-failure-with-fencing.summary | 1 + + pengine/test10/systemhealth1.scores | 2 + + pengine/test10/systemhealth1.summary | 2 + + pengine/test10/systemhealth2.scores | 1 + + pengine/test10/systemhealth2.summary | 1 + + pengine/test10/systemhealth3.scores | 1 + + pengine/test10/systemhealth3.summary | 1 + + pengine/test10/systemhealthm1.scores | 2 + + pengine/test10/systemhealthm1.summary | 2 + + pengine/test10/systemhealthm2.scores | 1 + + pengine/test10/systemhealthm2.summary | 1 + + pengine/test10/systemhealthm3.scores | 1 + + pengine/test10/systemhealthm3.summary | 1 + + pengine/test10/systemhealthn1.scores | 2 + + pengine/test10/systemhealthn1.summary | 2 + + pengine/test10/systemhealthn2.scores | 1 + + pengine/test10/systemhealthn2.summary | 1 + + pengine/test10/systemhealthn3.scores | 1 + + pengine/test10/systemhealthn3.summary | 1 + + pengine/test10/systemhealtho1.scores | 2 + + pengine/test10/systemhealtho1.summary | 2 + + pengine/test10/systemhealtho2.scores | 1 + + pengine/test10/systemhealtho2.summary | 1 + + pengine/test10/systemhealtho3.scores | 1 + + pengine/test10/systemhealtho3.summary | 1 + + pengine/test10/systemhealthp1.scores | 2 + + pengine/test10/systemhealthp1.summary | 2 + + pengine/test10/systemhealthp2.scores | 1 + + pengine/test10/systemhealthp2.summary | 1 + + pengine/test10/systemhealthp3.scores | 1 + + pengine/test10/systemhealthp3.summary | 1 + + pengine/test10/ticket-clone-21.scores | 2 + + pengine/test10/ticket-clone-21.summary | 2 + + pengine/test10/ticket-clone-9.scores | 2 + + pengine/test10/ticket-clone-9.summary | 2 + + pengine/test10/ticket-group-21.scores | 1 + + pengine/test10/ticket-group-21.summary | 1 + + pengine/test10/ticket-group-9.scores | 1 + + pengine/test10/ticket-group-9.summary | 1 + + pengine/test10/ticket-master-21.scores | 1 + + pengine/test10/ticket-master-21.summary | 1 + + pengine/test10/ticket-master-9.scores | 1 + + pengine/test10/ticket-master-9.summary | 1 + + pengine/test10/ticket-primitive-21.scores | 1 + + pengine/test10/ticket-primitive-21.summary | 1 + + pengine/test10/ticket-primitive-9.scores | 1 + + pengine/test10/ticket-primitive-9.summary | 1 + + pengine/test10/unfence-definition.dot | 5 +- + pengine/test10/unfence-definition.exp | 31 +-- + pengine/test10/unfence-definition.scores | 3 + + pengine/test10/unfence-definition.summary | 13 +- + pengine/test10/unfence-definition.xml | 10 + + pengine/test10/unfence-parameters.dot | 7 +- + pengine/test10/unfence-parameters.exp | 42 ++- + pengine/test10/unfence-parameters.scores | 4 + + pengine/test10/unfence-parameters.summary | 14 +- + pengine/test10/unfence-parameters.xml | 10 + + pengine/test10/unfence-startup.dot | 5 +- + pengine/test10/unfence-startup.exp | 29 +- + pengine/test10/unfence-startup.scores | 2 + + pengine/test10/unfence-startup.summary | 10 +- + pengine/test10/unfence-startup.xml | 10 + + pengine/test10/whitebox-fail1.scores | 1 + + pengine/test10/whitebox-fail1.summary | 1 + + pengine/test10/whitebox-fail2.scores | 1 + + pengine/test10/whitebox-fail2.summary | 1 + + pengine/test10/whitebox-imply-stop-on-fence.scores | 3 + + .../test10/whitebox-imply-stop-on-fence.summary | 3 + + pengine/test10/whitebox-ms-ordering.scores | 2 + + pengine/test10/whitebox-ms-ordering.summary | 2 + + .../test10/whitebox-unexpectedly-running.scores | 1 + + .../test10/whitebox-unexpectedly-running.summary | 1 + + 168 files changed, 580 insertions(+), 194 deletions(-) + +diff --git a/crmd/te_callbacks.c b/crmd/te_callbacks.c +index 3c63aa6..b5c857d 100644 +--- a/crmd/te_callbacks.c ++++ b/crmd/te_callbacks.c +@@ -788,16 +788,31 @@ tengine_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data) + } + + stop_te_timer(action->timer); +- + if (rc == pcmk_ok) { + const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); + const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); + const char *op = crm_meta_value(action->params, "stonith_action"); + +- crm_debug("Stonith operation %d for %s passed", call_id, target); ++ crm_info("Stonith operation %d for %s passed", call_id, target); + if (action->confirmed == FALSE) { + te_action_confirmed(action); +- if (action->sent_update == FALSE && safe_str_neq("on", op)) { ++ if (safe_str_eq("on", op)) { ++ const char *key = NULL; ++ const char *value = NULL; ++ ++ key = XML_NODE_IS_UNFENCED; ++ value = crm_meta_value(action->params, key); ++ update_attrd(target, key, value, NULL, FALSE); ++ ++ key = "digests-all"; ++ value = crm_meta_value(action->params, key); ++ update_attrd(target, key, value, NULL, FALSE); ++ ++ key = "digests-secure"; ++ value = crm_meta_value(action->params, key); ++ update_attrd(target, key, value, NULL, FALSE); ++ ++ } else if (action->sent_update == FALSE) { + send_stonith_update(action, target, uuid); + action->sent_update = TRUE; + } +diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h +index a764e1d..8cf22f3 100644 +--- a/include/crm/msg_xml.h ++++ b/include/crm/msg_xml.h +@@ -262,6 +262,7 @@ + # define XML_NODE_IS_PEER "crmd" + # define XML_NODE_IS_REMOTE "remote_node" + # define XML_NODE_IS_FENCED "node_fenced" ++# define XML_NODE_IS_UNFENCED "node_unfenced" + # define XML_NODE_IS_MAINTENANCE "node_in_maintenance" + + # define XML_CIB_ATTR_SHUTDOWN "shutdown" +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index 1b6afd1..3e10a30 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -273,7 +273,7 @@ typedef struct op_digest_cache_s { + op_digest_cache_t *rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, + pe_working_set_t * data_set); + +-action_t *pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set); ++action_t *pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set); + void trigger_unfencing( + resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set); + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 7c29e7b..90da95a 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1916,7 +1733,7 @@ filter_parameters(xmlNode * param_set, const char *param_string, bool need_prese + return; + } + +- if (param_set) { ++ if (param_set && param_string) { + xmlAttrPtr xIter = param_set->properties; + + while (xIter) { +@@ -1982,104 +1814,162 @@ bool fix_remote_addr(resource_t * rsc) + return TRUE; + } + ++static op_digest_cache_t * ++rsc_action_digest(resource_t * rsc, const char *task, const char *key, ++ node_t * node, xmlNode * xml_op, pe_working_set_t * data_set) ++{ ++ op_digest_cache_t *data = NULL; ++ ++ data = g_hash_table_lookup(node->details->digest_cache, key); ++ if (data == NULL) { ++ GHashTable *local_rsc_params = crm_str_table_new(); ++ action_t *action = custom_action(rsc, strdup(key), task, node, TRUE, FALSE, data_set); ++ //xmlNode *local_versioned_params = create_xml_node(NULL, XML_TAG_RSC_VER_ATTRS); ++ ++ const char *op_version; ++ //const char *ra_version = NULL; ++ const char *restart_list = NULL; ++ const char *secure_list = " passwd password "; ++ ++ data = calloc(1, sizeof(op_digest_cache_t)); ++ CRM_ASSERT(data != NULL); ++ ++ get_rsc_attributes(local_rsc_params, rsc, node, data_set); ++ //pe_get_versioned_attributes(local_versioned_params, rsc, node, data_set); ++ ++ data->params_all = create_xml_node(NULL, XML_TAG_PARAMS); ++ if (fix_remote_addr(rsc)) { ++ // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside ++ crm_xml_add(data->params_all, "addr", node->details->uname); ++ crm_trace("Fixing addr for %s on %s", rsc->id, node->details->uname); ++ } ++ ++ g_hash_table_foreach(local_rsc_params, hash2field, data->params_all); ++ g_hash_table_foreach(action->extra, hash2field, data->params_all); ++ g_hash_table_foreach(rsc->parameters, hash2field, data->params_all); ++ g_hash_table_foreach(action->meta, hash2metafield, data->params_all); ++ ++ if(xml_op) { ++ secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE); ++ restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART); ++ ++ op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION); ++ //ra_version = crm_element_value(xml_op, XML_ATTR_RA_VERSION); ++ ++ } else { ++ op_version = CRM_FEATURE_SET; ++ } ++ ++ //append_versioned_params(local_versioned_params, ra_version, data->params_all); ++ //append_versioned_params(rsc->versioned_parameters, ra_version, data->params_all); ++ //append_versioned_params(action->versioned_parameters, ra_version, data->params_all); ++ ++ filter_action_parameters(data->params_all, op_version); ++ ++ g_hash_table_destroy(local_rsc_params); ++ pe_free_action(action); ++ ++ data->digest_all_calc = calculate_operation_digest(data->params_all, op_version); ++ ++ if (is_set(data_set->flags, pe_flag_sanitized)) { ++ data->params_secure = copy_xml(data->params_all); ++ if(secure_list) { ++ filter_parameters(data->params_secure, secure_list, FALSE); ++ } ++ data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version); ++ } ++ ++ if(crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) != NULL) { ++ data->params_restart = copy_xml(data->params_all); ++ if (restart_list) { ++ filter_parameters(data->params_restart, restart_list, TRUE); ++ } ++ data->digest_restart_calc = calculate_operation_digest(data->params_restart, op_version); ++ } ++ ++ g_hash_table_insert(node->details->digest_cache, strdup(key), data); ++ } ++ ++ return data; ++} ++ + op_digest_cache_t * + rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, + pe_working_set_t * data_set) + { + op_digest_cache_t *data = NULL; + +- GHashTable *local_rsc_params = NULL; +- +- action_t *action = NULL; + char *key = NULL; +- + int interval = 0; +- const char *op_id = ID(xml_op); ++ + const char *interval_s = crm_element_value(xml_op, XML_LRM_ATTR_INTERVAL); + const char *task = crm_element_value(xml_op, XML_LRM_ATTR_TASK); ++ + const char *digest_all; + const char *digest_restart; +- const char *secure_list; +- const char *restart_list; +- const char *op_version; + + CRM_ASSERT(node != NULL); + +- data = g_hash_table_lookup(node->details->digest_cache, op_id); +- if (data) { +- return data; +- } +- +- data = calloc(1, sizeof(op_digest_cache_t)); +- CRM_ASSERT(data != NULL); +- + digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST); + digest_restart = crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST); + +- secure_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_SECURE); +- restart_list = crm_element_value(xml_op, XML_LRM_ATTR_OP_RESTART); +- +- op_version = crm_element_value(xml_op, XML_ATTR_CRM_VERSION); +- +- /* key is freed in custom_action */ + interval = crm_parse_int(interval_s, "0"); + key = generate_op_key(rsc->id, task, interval); +- action = custom_action(rsc, key, task, node, TRUE, FALSE, data_set); +- key = NULL; +- +- local_rsc_params = g_hash_table_new_full(crm_str_hash, g_str_equal, +- g_hash_destroy_str, g_hash_destroy_str); +- get_rsc_attributes(local_rsc_params, rsc, node, data_set); +- data->params_all = create_xml_node(NULL, XML_TAG_PARAMS); ++ data = rsc_action_digest(rsc, task, key, node, xml_op, data_set); + +- if (fix_remote_addr(rsc)) { +- // REMOTE_CONTAINER_HACK: Allow remote nodes that start containers with pacemaker remote inside +- crm_xml_add(data->params_all, "addr", node->details->uname); +- crm_trace("Fixing addr for %s on %s", rsc->id, node->details->uname); +- } ++ data->rc = RSC_DIGEST_MATCH; ++ if (digest_restart && data->digest_restart_calc && strcmp(data->digest_restart_calc, digest_restart) != 0) { ++ data->rc = RSC_DIGEST_RESTART; + +- g_hash_table_foreach(local_rsc_params, hash2field, data->params_all); +- g_hash_table_foreach(action->extra, hash2field, data->params_all); +- g_hash_table_foreach(rsc->parameters, hash2field, data->params_all); +- g_hash_table_foreach(action->meta, hash2metafield, data->params_all); +- filter_action_parameters(data->params_all, op_version); ++ } else if (digest_all == NULL) { ++ /* it is unknown what the previous op digest was */ ++ data->rc = RSC_DIGEST_UNKNOWN; + +- data->digest_all_calc = calculate_operation_digest(data->params_all, op_version); ++ } else if (strcmp(digest_all, data->digest_all_calc) != 0) { ++ data->rc = RSC_DIGEST_ALL; ++ } + +- if (secure_list && is_set(data_set->flags, pe_flag_sanitized)) { +- data->params_secure = copy_xml(data->params_all); ++ free(key); ++ return data; ++} + +- if (secure_list) { +- filter_parameters(data->params_secure, secure_list, FALSE); +- } +- data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version); +- } ++#define STONITH_DIGEST_TASK "stonith-on" + +- if (digest_restart) { +- data->params_restart = copy_xml(data->params_all); ++static op_digest_cache_t * ++fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * data_set) ++{ ++ char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); ++ op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, node, NULL, data_set); + +- if (restart_list) { +- filter_parameters(data->params_restart, restart_list, TRUE); +- } +- data->digest_restart_calc = calculate_operation_digest(data->params_restart, op_version); +- } ++ const char *digest_all = g_hash_table_lookup(node->details->attrs, "digests-all"); ++ const char *digest_secure = g_hash_table_lookup(node->details->attrs, "digests-secure"); + +- data->rc = RSC_DIGEST_MATCH; +- if (digest_restart && strcmp(data->digest_restart_calc, digest_restart) != 0) { +- data->rc = RSC_DIGEST_RESTART; ++ /* No restarts for fencing device changes */ + +- } else if (digest_all == NULL) { ++ data->rc = RSC_DIGEST_ALL; ++ if (digest_all == NULL) { + /* it is unknown what the previous op digest was */ + data->rc = RSC_DIGEST_UNKNOWN; + +- } else if (strcmp(digest_all, data->digest_all_calc) != 0) { +- data->rc = RSC_DIGEST_ALL; ++ } else if (strcmp(digest_all, data->digest_all_calc) == 0) { ++ data->rc = RSC_DIGEST_MATCH; ++ ++ } else if(digest_secure && data->digest_secure_calc) { ++ char *search = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); ++ ++ if(strstr(digest_secure, search)) { ++ fprintf(stdout, "Only 'private' parameters to %s for unfencing %s changed\n", ++ rsc->id, node->details->uname); ++ data->rc = RSC_DIGEST_MATCH; ++ } + } + +- g_hash_table_insert(node->details->digest_cache, strdup(op_id), data); +- g_hash_table_destroy(local_rsc_params); +- pe_free_action(action); ++ if (data->rc == RSC_DIGEST_ALL && is_set(data_set->flags, pe_flag_sanitized) && data->digest_secure_calc) { ++ fprintf(stdout, "Parameters to %s for unfencing %s changed, try '%s:%s:%s'\n", ++ rsc->id, node->details->uname, rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); ++ } + ++ free(key); + return data; + } + +@@ -2117,35 +2007,113 @@ set_bit_recursive(resource_t * rsc, unsigned long long flag) + } + } + ++static GListPtr ++find_unfencing_devices(GListPtr candidates, GListPtr matches) ++{ ++ for (GListPtr gIter = candidates; gIter != NULL; gIter = gIter->next) { ++ resource_t *candidate = gIter->data; ++ const char *provides = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_PROVIDES); ++ const char *requires = g_hash_table_lookup(candidate->meta, XML_RSC_ATTR_REQUIRES); ++ ++ if(candidate->children) { ++ matches = find_unfencing_devices(candidate->children, matches); ++ } else if (is_not_set(candidate->flags, pe_rsc_fence_device)) { ++ continue; ++ ++ } else if (crm_str_eq(provides, "unfencing", FALSE) || crm_str_eq(requires, "unfencing", FALSE)) { ++ matches = g_list_prepend(matches, candidate); ++ } ++ } ++ return matches; ++} ++ ++ ++#define STONITH_DIGEST_TASK "stonith-on" ++ + action_t * +-pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set) ++pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set) + { +- char *key = NULL; ++ char *op_key = NULL; + action_t *stonith_op = NULL; + + if(op == NULL) { + op = data_set->stonith_action; + } + +- key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op); ++ op_key = crm_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op); + + if(data_set->singletons) { +- stonith_op = g_hash_table_lookup(data_set->singletons, key); ++ stonith_op = g_hash_table_lookup(data_set->singletons, op_key); + } + + if(stonith_op == NULL) { +- stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, optional, TRUE, data_set); ++ stonith_op = custom_action(NULL, op_key, CRM_OP_FENCE, node, TRUE, TRUE, data_set); + + add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname); + add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id); + add_hash_param(stonith_op->meta, "stonith_action", op); ++ ++ if(is_remote_node(node) && is_set(data_set->flags, pe_flag_enable_unfencing)) { ++ /* Extra work to detect device changes on remotes ++ * ++ * We may do this for all nodes in the future, but for now ++ * the check_action_definition() based stuff works fine. ++ * ++ * Use "stonith-on" to avoid creating cache entries for ++ * operations check_action_definition() would look for. ++ */ ++ long max = 1024; ++ long digests_all_offset = 0; ++ long digests_secure_offset = 0; ++ ++ char *digests_all = malloc(max); ++ char *digests_secure = malloc(max); ++ GListPtr matches = find_unfencing_devices(data_set->resources, NULL); ++ ++ for (GListPtr gIter = matches; gIter != NULL; gIter = gIter->next) { ++ resource_t *match = gIter->data; ++ op_digest_cache_t *data = fencing_action_digest_cmp(match, node, data_set); ++ ++ if(data->rc == RSC_DIGEST_ALL) { ++ optional = FALSE; ++ crm_notice("Unfencing %s (remote): because the definition of %s changed", node->details->uname, match->id); ++ if (is_set(data_set->flags, pe_flag_sanitized)) { ++ /* Extra detail for those running from the commandline */ ++ fprintf(stdout, " notice: Unfencing %s (remote): because the definition of %s changed\n", node->details->uname, match->id); ++ } ++ ++ } ++ ++ digests_all_offset += snprintf( ++ digests_all+digests_all_offset, max-digests_all_offset, ++ "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_all_calc); ++ ++ digests_secure_offset += snprintf( ++ digests_secure+digests_secure_offset, max-digests_secure_offset, ++ "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_secure_calc); ++ } ++ add_hash_param(stonith_op->meta, strdup("digests-all"), digests_all); ++ add_hash_param(stonith_op->meta, strdup("digests-secure"), digests_secure); ++ } ++ + } else { +- free(key); ++ free(op_key); + } + +- if(optional == FALSE) { +- crm_trace("%s is no longer optional", stonith_op->uuid); ++ if(optional == FALSE && is_set(stonith_op->flags, pe_action_optional)) { ++ const char *kind = "Fencing "; ++ + pe_clear_action_bit(stonith_op, pe_action_optional); ++ ++ if(safe_str_eq(op, "on")) { ++ kind = "Unfencing"; ++ } ++ ++ crm_notice("%s %s: %s", kind, node->details->uname, reason); ++ if (is_set(data_set->flags, pe_flag_sanitized)) { ++ /* Extra detail for those running from the commandline */ ++ fprintf(stdout, " notice: %s %s: %s\n", kind, node->details->uname, reason); ++ } + } + + return stonith_op; +@@ -2167,9 +2135,8 @@ trigger_unfencing( + && node->details->online + && node->details->unclean == FALSE + && node->details->shutdown == FALSE) { +- action_t *unfence = pe_fence_op(node, "on", FALSE, data_set); ++ action_t *unfence = pe_fence_op(node, "on", FALSE, reason, data_set); + +- crm_notice("Unfencing %s: %s", node->details->uname, reason); + if(dependency) { + order_actions(unfence, dependency, pe_order_optional); + } +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 8ea2ea1..fd9c4f9 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1406,7 +1406,7 @@ fence_guest(pe_node_t *node, pe_action_t *done, pe_working_set_t *data_set) + /* Create a fence pseudo-event, so we have an event to order actions + * against, and crmd can always detect it. + */ +- stonith_op = pe_fence_op(node, fence_action, FALSE, data_set); ++ stonith_op = pe_fence_op(node, fence_action, FALSE, "guest is unclean", data_set); + update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable, + __FUNCTION__, __LINE__); + +@@ -1488,7 +1488,7 @@ stage6(pe_working_set_t * data_set) + + pe_warn("Scheduling Node %s for STONITH", node->details->uname); + +- stonith_op = pe_fence_op(node, NULL, FALSE, data_set); ++ stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", data_set); + + stonith_constraints(node, stonith_op, data_set); + +diff --git a/pengine/native.c b/pengine/native.c +index dd5ff18..bc59405 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -1355,7 +1355,7 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + + g_hash_table_iter_init(&iter, rsc->allowed_nodes); + while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { +- action_t *unfence = pe_fence_op(node, "on", TRUE, data_set); ++ action_t *unfence = pe_fence_op(node, "on", TRUE, __FUNCTION__, data_set); + + crm_debug("Ordering any stops of %s before %s, and any starts after", + rsc->id, unfence->uuid); +@@ -2455,6 +2455,16 @@ StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * d + if (is_set(data_set->flags, pe_flag_remove_after_stop)) { + DeleteRsc(rsc, current, optional, data_set); + } ++ ++ if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { ++ action_t *unfence = pe_fence_op(current, "on", TRUE, __FUNCTION__, data_set); ++ const char *unfenced = g_hash_table_lookup(current->details->attrs, XML_NODE_IS_UNFENCED); ++ ++ order_actions(stop, unfence, pe_order_implies_first); ++ if (unfenced == NULL || safe_str_eq("0", unfenced)) { ++ pe_proc_err("Stopping %s until %s can be unfenced", rsc->id, current->details->uname); ++ } ++ } + } + + return TRUE; +@@ -2468,9 +2478,25 @@ StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * + CRM_ASSERT(rsc); + pe_rsc_trace(rsc, "%s on %s %d", rsc->id, next ? next->details->uname : "N/A", optional); + start = start_action(rsc, next, TRUE); ++ ++ if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { ++ action_t *unfence = pe_fence_op(next, "on", TRUE, __FUNCTION__, data_set); ++ const char *unfenced = g_hash_table_lookup(next->details->attrs, XML_NODE_IS_UNFENCED); ++ ++ order_actions(unfence, start, pe_order_implies_then); ++ ++ if (unfenced == NULL || safe_str_eq("0", unfenced)) { ++ char *reason = crm_strdup_printf("Required by %s", rsc->id); ++ trigger_unfencing(NULL, next, reason, NULL, data_set); ++ free(reason); ++ } ++ } ++ + if (is_set(start->flags, pe_action_runnable) && optional == FALSE) { + update_action_flags(start, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__); + } ++ ++ + return TRUE; + } + +@@ -2820,14 +2846,9 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + * probed, we know the state of all resources that require + * unfencing and that unfencing occurred. + */ +- if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) { +- trigger_unfencing(NULL, node, "node discovery", probe, data_set); +- probe->priority = INFINITY; /* Ensure this runs if unfencing succeeds */ +- +- } else if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { +- action_t *unfence = pe_fence_op(node, "on", TRUE, data_set); +- +- order_actions(probe, unfence, pe_order_optional); ++ if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { ++ action_t *unfence = pe_fence_op(node, "on", TRUE, __FUNCTION__, data_set); ++ order_actions(unfence, probe, pe_order_optional); + } + + /* +diff --git a/pengine/test10/594.scores b/pengine/test10/594.scores +index 972b89b..430550c 100644 +--- a/pengine/test10/594.scores ++++ b/pengine/test10/594.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hadev3: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on hadev1: 0 + clone_color: DoFencing allocation score on hadev2: 0 +diff --git a/pengine/test10/594.summary b/pengine/test10/594.summary +index ef2a02c..6cd78d3 100644 +--- a/pengine/test10/594.summary ++++ b/pengine/test10/594.summary +@@ -12,6 +12,7 @@ Online: [ hadev1 hadev2 ] + child_DoFencing:1 (stonith:ssh): Started hadev1 + child_DoFencing:2 (stonith:ssh): Started hadev1 + ++ notice: Fencing hadev3: node is unclean + Transition Summary: + * Fence (reboot) hadev3 + * Shutdown hadev2 +diff --git a/pengine/test10/829.scores b/pengine/test10/829.scores +index f0652ff..71a372b 100644 +--- a/pengine/test10/829.scores ++++ b/pengine/test10/829.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing c001n02: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n01: 0 + clone_color: DoFencing allocation score on c001n02: 0 +diff --git a/pengine/test10/829.summary b/pengine/test10/829.summary +index 556699f..8649ed8 100644 +--- a/pengine/test10/829.summary ++++ b/pengine/test10/829.summary +@@ -14,6 +14,7 @@ Online: [ c001n01 c001n03 c001n08 ] + child_DoFencing:2 (stonith:ssh): Started c001n01 + child_DoFencing:3 (stonith:ssh): Started c001n08 + ++ notice: Fencing c001n02: node is unclean + Transition Summary: + * Fence (reboot) c001n02 + * Move rsc_c001n02 (Started c001n02 -> c001n01) +diff --git a/pengine/test10/bug-5186-partial-migrate.scores b/pengine/test10/bug-5186-partial-migrate.scores +index 8b29529..dd59f34 100644 +--- a/pengine/test10/bug-5186-partial-migrate.scores ++++ b/pengine/test10/bug-5186-partial-migrate.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing bl460g1n7: node is unclean + Allocation scores: + clone_color: clnDiskd1 allocation score on bl460g1n6: 200 + clone_color: clnDiskd1 allocation score on bl460g1n7: 0 +diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary +index a32e81d..5c8231d 100644 +--- a/pengine/test10/bug-5186-partial-migrate.summary ++++ b/pengine/test10/bug-5186-partial-migrate.summary +@@ -24,6 +24,7 @@ Online: [ bl460g1n6 bl460g1n8 ] + prmPing (ocf::pacemaker:ping): Started bl460g1n7 (UNCLEAN) + Started: [ bl460g1n6 bl460g1n8 ] + ++ notice: Fencing bl460g1n7: node is unclean + Transition Summary: + * Fence (reboot) bl460g1n7 + * Move prmDummy (Started bl460g1n7 -> bl460g1n6) +diff --git a/pengine/test10/bug-cl-5247.scores b/pengine/test10/bug-cl-5247.scores +index e9e4709..f1ded69 100644 +--- a/pengine/test10/bug-cl-5247.scores ++++ b/pengine/test10/bug-cl-5247.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing pgsr02: guest is unclean + Allocation scores: + Using the original execution date of: 2015-08-12 02:53:40Z + clone_color: msPostgresql allocation score on bl460g8n3: -INFINITY +diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary +index f70a9ea..755e647 100644 +--- a/pengine/test10/bug-cl-5247.summary ++++ b/pengine/test10/bug-cl-5247.summary +@@ -17,6 +17,7 @@ Containers: [ pgsr01:prmDB1 ] + Masters: [ pgsr01 ] + Stopped: [ bl460g8n3 bl460g8n4 ] + ++ notice: Fencing pgsr02: guest is unclean + Transition Summary: + * Fence (off) pgsr02 (resource: prmDB2) + * Stop prmDB2 (bl460g8n4) +diff --git a/pengine/test10/bug-lf-2508.scores b/pengine/test10/bug-lf-2508.scores +index 3c3ce16..200d548 100644 +--- a/pengine/test10/bug-lf-2508.scores ++++ b/pengine/test10/bug-lf-2508.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing srv02: node is unclean + Allocation scores: + clone_color: clnStonith1 allocation score on srv01: -INFINITY + clone_color: clnStonith1 allocation score on srv02: 0 +diff --git a/pengine/test10/bug-lf-2508.summary b/pengine/test10/bug-lf-2508.summary +index 7931d19..18080cb 100644 +--- a/pengine/test10/bug-lf-2508.summary ++++ b/pengine/test10/bug-lf-2508.summary +@@ -34,6 +34,7 @@ Online: [ srv01 srv03 srv04 ] + Started: [ srv01 srv03 ] + Stopped: [ srv04 ] + ++ notice: Fencing srv02: node is unclean + Transition Summary: + * Fence (reboot) srv02 + * Start Dummy01 (srv01) +diff --git a/pengine/test10/bug-lf-2551.scores b/pengine/test10/bug-lf-2551.scores +index 2b0aa12..64cbca0 100644 +--- a/pengine/test10/bug-lf-2551.scores ++++ b/pengine/test10/bug-lf-2551.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hex-9: node is unclean + Allocation scores: + clone_color: base-clone allocation score on hex-0: 14 + clone_color: base-clone allocation score on hex-7: 16 +diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary +index ef2e54a..4fa553c 100644 +--- a/pengine/test10/bug-lf-2551.summary ++++ b/pengine/test10/bug-lf-2551.summary +@@ -80,6 +80,7 @@ Online: [ hex-0 hex-7 hex-8 ] + vm-63 (ocf::heartbeat:Xen): Stopped + vm-64 (ocf::heartbeat:Xen): Stopped + ++ notice: Fencing hex-9: node is unclean + Transition Summary: + * Fence (reboot) hex-9 + * Move fencing-sbd (Started hex-9 -> hex-0) +diff --git a/pengine/test10/bug-lf-2606.scores b/pengine/test10/bug-lf-2606.scores +index 6a9d522..b1e6a02 100644 +--- a/pengine/test10/bug-lf-2606.scores ++++ b/pengine/test10/bug-lf-2606.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node2: node is unclean + Allocation scores: + clone_color: ms3 allocation score on node1: 0 + clone_color: ms3 allocation score on node2: 0 +diff --git a/pengine/test10/bug-lf-2606.summary b/pengine/test10/bug-lf-2606.summary +index f30a053..e6cfa89 100644 +--- a/pengine/test10/bug-lf-2606.summary ++++ b/pengine/test10/bug-lf-2606.summary +@@ -11,6 +11,7 @@ Online: [ node1 ] + Masters: [ node2 ] + Slaves: [ node1 ] + ++ notice: Fencing node2: node is unclean + Transition Summary: + * Fence (reboot) node2 + * Stop rsc1 (node2) +diff --git a/pengine/test10/bug-rh-1097457.scores b/pengine/test10/bug-rh-1097457.scores +index 7729421..2fe2859 100644 +--- a/pengine/test10/bug-rh-1097457.scores ++++ b/pengine/test10/bug-rh-1097457.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing lamaVM2: guest is unclean + Allocation scores: + clone_color: FAKE6-clone allocation score on lama2: -INFINITY + clone_color: FAKE6-clone allocation score on lama3: -INFINITY +diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary +index 1c355c0..19aa39d 100644 +--- a/pengine/test10/bug-rh-1097457.summary ++++ b/pengine/test10/bug-rh-1097457.summary +@@ -31,6 +31,7 @@ Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ] + Clone Set: FAKE6-clone [FAKE6] + Started: [ lamaVM1 lamaVM2 lamaVM3 ] + ++ notice: Fencing lamaVM2: guest is unclean + Transition Summary: + * Fence (reboot) lamaVM2 (resource: VM2) + * Recover VM2 (Started lama3) +diff --git a/pengine/test10/concurrent-fencing.scores b/pengine/test10/concurrent-fencing.scores +index 0322231..1be723e 100644 +--- a/pengine/test10/concurrent-fencing.scores ++++ b/pengine/test10/concurrent-fencing.scores +@@ -1,3 +1,6 @@ ++ notice: Fencing node1: node is unclean ++ notice: Fencing node2: node is unclean ++ notice: Fencing node3: node is unclean + Allocation scores: + native_color: lsb_dummy allocation score on node1: 0 + native_color: lsb_dummy allocation score on node2: 0 +diff --git a/pengine/test10/concurrent-fencing.summary b/pengine/test10/concurrent-fencing.summary +index 1e4a817..3f54c2c 100644 +--- a/pengine/test10/concurrent-fencing.summary ++++ b/pengine/test10/concurrent-fencing.summary +@@ -7,6 +7,9 @@ Node node3 (uuid3): UNCLEAN (offline) + stonith-1 (stonith:dummy): Stopped + lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped + ++ notice: Fencing node1: node is unclean ++ notice: Fencing node2: node is unclean ++ notice: Fencing node3: node is unclean + Transition Summary: + * Fence (reboot) node3 + * Fence (reboot) node2 +diff --git a/pengine/test10/guest-node-host-dies.scores b/pengine/test10/guest-node-host-dies.scores +index 0d7ad3f..663d236 100644 +--- a/pengine/test10/guest-node-host-dies.scores ++++ b/pengine/test10/guest-node-host-dies.scores +@@ -1,3 +1,6 @@ ++ notice: Fencing lxc1: guest is unclean ++ notice: Fencing lxc2: guest is unclean ++ notice: Fencing rhel7-1: node is unclean + Allocation scores: + clone_color: lxc-ms-master allocation score on lxc1: INFINITY + clone_color: lxc-ms-master allocation score on lxc2: INFINITY +diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary +index 9f85613..679deb6 100644 +--- a/pengine/test10/guest-node-host-dies.summary ++++ b/pengine/test10/guest-node-host-dies.summary +@@ -10,6 +10,9 @@ Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + Master/Slave Set: lxc-ms-master [lxc-ms] + Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + ++ notice: Fencing lxc1: guest is unclean ++ notice: Fencing lxc2: guest is unclean ++ notice: Fencing rhel7-1: node is unclean + Transition Summary: + * Fence (reboot) rhel7-1 + * Fence (reboot) lxc2 (resource: container2) +diff --git a/pengine/test10/interleave-pseudo-stop.scores b/pengine/test10/interleave-pseudo-stop.scores +index 014f4de..625ba18 100644 +--- a/pengine/test10/interleave-pseudo-stop.scores ++++ b/pengine/test10/interleave-pseudo-stop.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node1: node is unclean + Allocation scores: + clone_color: configstoreclone:0 allocation score on node1: 0 + clone_color: configstoreclone:0 allocation score on node2: 1 +diff --git a/pengine/test10/interleave-pseudo-stop.summary b/pengine/test10/interleave-pseudo-stop.summary +index 12be956..8c22b2f 100644 +--- a/pengine/test10/interleave-pseudo-stop.summary ++++ b/pengine/test10/interleave-pseudo-stop.summary +@@ -16,6 +16,7 @@ Online: [ node2 ] + configstoreclone (ocf::heartbeat:Filesystem): Started node1 (UNCLEAN) + Started: [ node2 ] + ++ notice: Fencing node1: node is unclean + Transition Summary: + * Fence (reboot) node1 + * Stop stonithclone:1 (node1) +diff --git a/pengine/test10/master-7.scores b/pengine/test10/master-7.scores +index 37fc6cd..7569625 100644 +--- a/pengine/test10/master-7.scores ++++ b/pengine/test10/master-7.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing c001n01: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n01: 0 + clone_color: DoFencing allocation score on c001n02: 0 +diff --git a/pengine/test10/master-7.summary b/pengine/test10/master-7.summary +index 2889efb..d76051e 100644 +--- a/pengine/test10/master-7.summary ++++ b/pengine/test10/master-7.summary +@@ -28,6 +28,7 @@ Online: [ c001n02 c001n03 c001n08 ] + ocf_msdummy:6 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n02 + ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08 + ++ notice: Fencing c001n01: node is unclean + Transition Summary: + * Fence (reboot) c001n01 + * Move DcIPaddr (Started c001n01 -> c001n03) +diff --git a/pengine/test10/master-8.scores b/pengine/test10/master-8.scores +index c9d1cc9..880d7fb 100644 +--- a/pengine/test10/master-8.scores ++++ b/pengine/test10/master-8.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing c001n01: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n01: 0 + clone_color: DoFencing allocation score on c001n02: 0 +diff --git a/pengine/test10/master-8.summary b/pengine/test10/master-8.summary +index 5968a25..7d0b840 100644 +--- a/pengine/test10/master-8.summary ++++ b/pengine/test10/master-8.summary +@@ -28,6 +28,7 @@ Online: [ c001n02 c001n03 c001n08 ] + ocf_msdummy:6 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n02 + ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08 + ++ notice: Fencing c001n01: node is unclean + Transition Summary: + * Fence (reboot) c001n01 + * Move DcIPaddr (Started c001n01 -> c001n03) +diff --git a/pengine/test10/migrate-fencing.scores b/pengine/test10/migrate-fencing.scores +index 3febd69..37d42b4 100644 +--- a/pengine/test10/migrate-fencing.scores ++++ b/pengine/test10/migrate-fencing.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing pcmk-4: node is unclean + Allocation scores: + clone_color: Connectivity allocation score on pcmk-1: 0 + clone_color: Connectivity allocation score on pcmk-2: 0 +diff --git a/pengine/test10/migrate-fencing.summary b/pengine/test10/migrate-fencing.summary +index 842c65d..fdb905e 100644 +--- a/pengine/test10/migrate-fencing.summary ++++ b/pengine/test10/migrate-fencing.summary +@@ -21,6 +21,7 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Masters: [ pcmk-4 ] + Slaves: [ pcmk-1 pcmk-2 pcmk-3 ] + ++ notice: Fencing pcmk-4: node is unclean + Transition Summary: + * Fence (reboot) pcmk-4 + * Stop FencingChild:0 (pcmk-4) +diff --git a/pengine/test10/rec-node-11.scores b/pengine/test10/rec-node-11.scores +index 482deeb..86c2bf5 100644 +--- a/pengine/test10/rec-node-11.scores ++++ b/pengine/test10/rec-node-11.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node1: node is unclean + Allocation scores: + group_color: group1 allocation score on node1: 0 + group_color: group1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-11.summary b/pengine/test10/rec-node-11.summary +index 22f5af7..f45f0a8 100644 +--- a/pengine/test10/rec-node-11.summary ++++ b/pengine/test10/rec-node-11.summary +@@ -9,6 +9,7 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 + rsc3 (heartbeat:apache): Started node2 + ++ notice: Fencing node1: node is unclean + Transition Summary: + * Fence (reboot) node1 + * Start stonith-1 (node2) +diff --git a/pengine/test10/rec-node-12.scores b/pengine/test10/rec-node-12.scores +index 153c232..4fd9ff5 100644 +--- a/pengine/test10/rec-node-12.scores ++++ b/pengine/test10/rec-node-12.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing c001n02: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n01: 0 + clone_color: DoFencing allocation score on c001n02: 0 +diff --git a/pengine/test10/rec-node-12.summary b/pengine/test10/rec-node-12.summary +index 6316fdb..562cae9 100644 +--- a/pengine/test10/rec-node-12.summary ++++ b/pengine/test10/rec-node-12.summary +@@ -14,6 +14,7 @@ Online: [ c001n01 c001n03 c001n08 ] + child_DoFencing:2 (stonith:ssh): Stopped + child_DoFencing:3 (stonith:ssh): Stopped + ++ notice: Fencing c001n02: node is unclean + Transition Summary: + * Fence (reboot) c001n02 + * Start DcIPaddr (c001n08) +diff --git a/pengine/test10/rec-node-13.scores b/pengine/test10/rec-node-13.scores +index dda1134..bdb5004 100644 +--- a/pengine/test10/rec-node-13.scores ++++ b/pengine/test10/rec-node-13.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing c001n04: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n02: 0 + clone_color: DoFencing allocation score on c001n03: 0 +diff --git a/pengine/test10/rec-node-13.summary b/pengine/test10/rec-node-13.summary +index c222ad8..d3b4d65 100644 +--- a/pengine/test10/rec-node-13.summary ++++ b/pengine/test10/rec-node-13.summary +@@ -33,6 +33,7 @@ OFFLINE: [ c001n03 c001n05 ] + ocf_msdummy:10 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n06 + ocf_msdummy:11 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n07 + ++ notice: Fencing c001n04: node is unclean + Transition Summary: + * Fence (reboot) c001n04 + * Stop ocf_msdummy:6 (c001n04) +diff --git a/pengine/test10/rec-node-14.scores b/pengine/test10/rec-node-14.scores +index 0322231..1be723e 100644 +--- a/pengine/test10/rec-node-14.scores ++++ b/pengine/test10/rec-node-14.scores +@@ -1,3 +1,6 @@ ++ notice: Fencing node1: node is unclean ++ notice: Fencing node2: node is unclean ++ notice: Fencing node3: node is unclean + Allocation scores: + native_color: lsb_dummy allocation score on node1: 0 + native_color: lsb_dummy allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-14.summary b/pengine/test10/rec-node-14.summary +index f503a92..a707aee 100644 +--- a/pengine/test10/rec-node-14.summary ++++ b/pengine/test10/rec-node-14.summary +@@ -7,6 +7,9 @@ Node node3 (uuid3): UNCLEAN (offline) + stonith-1 (stonith:dummy): Stopped + lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped + ++ notice: Fencing node1: node is unclean ++ notice: Fencing node2: node is unclean ++ notice: Fencing node3: node is unclean + Transition Summary: + * Fence (reboot) node3 + * Fence (reboot) node2 +diff --git a/pengine/test10/rec-node-15.scores b/pengine/test10/rec-node-15.scores +index 03dc6e6..d8b9617 100644 +--- a/pengine/test10/rec-node-15.scores ++++ b/pengine/test10/rec-node-15.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing sapcl03: node is unclean + Allocation scores: + group_color: Filesystem_13 allocation score on sapcl01: 0 + group_color: Filesystem_13 allocation score on sapcl02: 0 +diff --git a/pengine/test10/rec-node-15.summary b/pengine/test10/rec-node-15.summary +index 2f706d2..863bfd4 100644 +--- a/pengine/test10/rec-node-15.summary ++++ b/pengine/test10/rec-node-15.summary +@@ -20,6 +20,7 @@ Online: [ sapcl01 ] + oracle_24 (ocf::heartbeat:oracle): Stopped + oralsnr_25 (ocf::heartbeat:oralsnr): Stopped + ++ notice: Fencing sapcl03: node is unclean + Transition Summary: + * Fence (reboot) sapcl03 + * Start stonith-1 (sapcl01) +diff --git a/pengine/test10/rec-node-2.scores b/pengine/test10/rec-node-2.scores +index 68e176f..6efa92d 100644 +--- a/pengine/test10/rec-node-2.scores ++++ b/pengine/test10/rec-node-2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node1: node is unclean + Allocation scores: + group_color: group1 allocation score on node1: 0 + group_color: group1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-2.summary b/pengine/test10/rec-node-2.summary +index 0e0183f..ecf6562 100644 +--- a/pengine/test10/rec-node-2.summary ++++ b/pengine/test10/rec-node-2.summary +@@ -13,6 +13,7 @@ Online: [ node2 ] + rsc5 (heartbeat:apache): Stopped + rsc6 (heartbeat:apache): Stopped + ++ notice: Fencing node1: node is unclean + Transition Summary: + * Fence (reboot) node1 + * Start stonith-1 (node2) +diff --git a/pengine/test10/rec-node-4.scores b/pengine/test10/rec-node-4.scores +index 1d008ec..e25a30f 100644 +--- a/pengine/test10/rec-node-4.scores ++++ b/pengine/test10/rec-node-4.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node1: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-4.summary b/pengine/test10/rec-node-4.summary +index 0a2606e..f46a3bb 100644 +--- a/pengine/test10/rec-node-4.summary ++++ b/pengine/test10/rec-node-4.summary +@@ -7,6 +7,7 @@ Online: [ node2 ] + rsc1 (heartbeat:apache): Started node1 (UNCLEAN) + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + ++ notice: Fencing node1: node is unclean + Transition Summary: + * Fence (reboot) node1 + * Start stonith-1 (node2) +diff --git a/pengine/test10/rec-node-6.scores b/pengine/test10/rec-node-6.scores +index 1d008ec..e25a30f 100644 +--- a/pengine/test10/rec-node-6.scores ++++ b/pengine/test10/rec-node-6.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node1: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-6.summary b/pengine/test10/rec-node-6.summary +index 530c7d2..37257fe 100644 +--- a/pengine/test10/rec-node-6.summary ++++ b/pengine/test10/rec-node-6.summary +@@ -7,6 +7,7 @@ Online: [ node2 ] + rsc1 (heartbeat:apache): Started node1 + rsc2 (heartbeat:apache): Started node1 + ++ notice: Fencing node1: node is unclean + Transition Summary: + * Fence (reboot) node1 + * Start stonith-1 (node2) +diff --git a/pengine/test10/rec-node-7.scores b/pengine/test10/rec-node-7.scores +index 1d008ec..e25a30f 100644 +--- a/pengine/test10/rec-node-7.scores ++++ b/pengine/test10/rec-node-7.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node1: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-7.summary b/pengine/test10/rec-node-7.summary +index 0a2606e..f46a3bb 100644 +--- a/pengine/test10/rec-node-7.summary ++++ b/pengine/test10/rec-node-7.summary +@@ -7,6 +7,7 @@ Online: [ node2 ] + rsc1 (heartbeat:apache): Started node1 (UNCLEAN) + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + ++ notice: Fencing node1: node is unclean + Transition Summary: + * Fence (reboot) node1 + * Start stonith-1 (node2) +diff --git a/pengine/test10/rec-rsc-5.scores b/pengine/test10/rec-rsc-5.scores +index 1c640ae..ebaedbd 100644 +--- a/pengine/test10/rec-rsc-5.scores ++++ b/pengine/test10/rec-rsc-5.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node2: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: 0 + native_color: rsc1 allocation score on node2: -INFINITY +diff --git a/pengine/test10/rec-rsc-5.summary b/pengine/test10/rec-rsc-5.summary +index c3e658a..6bc4010 100644 +--- a/pengine/test10/rec-rsc-5.summary ++++ b/pengine/test10/rec-rsc-5.summary +@@ -7,6 +7,7 @@ Online: [ node1 ] + rsc1 (heartbeat:apache): FAILED node2 + rsc2 (heartbeat:apache): Started node2 + ++ notice: Fencing node2: node is unclean + Transition Summary: + * Fence (reboot) node2 + * Start stonith-1 (node1) +diff --git a/pengine/test10/remote-fence-before-reconnect.scores b/pengine/test10/remote-fence-before-reconnect.scores +index fb46919..9423693 100644 +--- a/pengine/test10/remote-fence-before-reconnect.scores ++++ b/pengine/test10/remote-fence-before-reconnect.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing c7auto4: node is unclean + Allocation scores: + native_color: c7auto4 allocation score on c7auto1: -INFINITY + native_color: c7auto4 allocation score on c7auto2: -INFINITY +diff --git a/pengine/test10/remote-fence-before-reconnect.summary b/pengine/test10/remote-fence-before-reconnect.summary +index 2ce3315..f477884 100644 +--- a/pengine/test10/remote-fence-before-reconnect.summary ++++ b/pengine/test10/remote-fence-before-reconnect.summary +@@ -11,6 +11,7 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + fake4 (ocf::heartbeat:Dummy): Started c7auto2 + fake5 (ocf::heartbeat:Dummy): Started c7auto3 + ++ notice: Fencing c7auto4: node is unclean + Transition Summary: + * Fence (reboot) c7auto4 + * Stop c7auto4 (c7auto1) +diff --git a/pengine/test10/remote-fence-unclean-3.scores b/pengine/test10/remote-fence-unclean-3.scores +index 77d71d6..283a599 100644 +--- a/pengine/test10/remote-fence-unclean-3.scores ++++ b/pengine/test10/remote-fence-unclean-3.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing overcloud-novacompute-0: node is unclean + Allocation scores: + clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 + clone_color: galera-bundle-master allocation score on galera-bundle-1: 0 +diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary +index 9296add..ec5d304 100644 +--- a/pengine/test10/remote-fence-unclean-3.summary ++++ b/pengine/test10/remote-fence-unclean-3.summary +@@ -33,6 +33,7 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund + Docker container: openstack-cinder-backup [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-backup:latest] + openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1 + ++ notice: Fencing overcloud-novacompute-0: node is unclean + Transition Summary: + * Fence (reboot) overcloud-novacompute-0 + * Start fence1 (overcloud-controller-0) +diff --git a/pengine/test10/remote-fence-unclean.scores b/pengine/test10/remote-fence-unclean.scores +index 8d29662..038ca2e 100644 +--- a/pengine/test10/remote-fence-unclean.scores ++++ b/pengine/test10/remote-fence-unclean.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing remote1: node is unclean + Allocation scores: + native_color: FAKE1 allocation score on 18builder: 0 + native_color: FAKE1 allocation score on 18node1: 0 +diff --git a/pengine/test10/remote-fence-unclean.summary b/pengine/test10/remote-fence-unclean.summary +index 667549b..8f37909 100644 +--- a/pengine/test10/remote-fence-unclean.summary ++++ b/pengine/test10/remote-fence-unclean.summary +@@ -10,6 +10,7 @@ Online: [ 18builder 18node1 18node2 ] + FAKE3 (ocf::heartbeat:Dummy): Started 18builder + FAKE4 (ocf::heartbeat:Dummy): Started 18node1 + ++ notice: Fencing remote1: node is unclean + Transition Summary: + * Fence (reboot) remote1 + * Recover remote1 (Started 18node1) +diff --git a/pengine/test10/remote-fence-unclean2.scores b/pengine/test10/remote-fence-unclean2.scores +index 10fc7fd..937d0eb 100644 +--- a/pengine/test10/remote-fence-unclean2.scores ++++ b/pengine/test10/remote-fence-unclean2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing rhel7-alt4: node is unclean + Allocation scores: + native_color: fake allocation score on rhel7-alt1: 0 + native_color: fake allocation score on rhel7-alt2: 0 +diff --git a/pengine/test10/remote-fence-unclean2.summary b/pengine/test10/remote-fence-unclean2.summary +index 0844c29..060d55e 100644 +--- a/pengine/test10/remote-fence-unclean2.summary ++++ b/pengine/test10/remote-fence-unclean2.summary +@@ -9,6 +9,7 @@ OFFLINE: [ rhel7-alt3 ] + rhel7-alt4 (ocf::pacemaker:remote): Stopped + fake (ocf::heartbeat:Dummy): Started rhel7-alt4 (UNCLEAN) + ++ notice: Fencing rhel7-alt4: node is unclean + Transition Summary: + * Fence (reboot) rhel7-alt4 + * Stop fake (rhel7-alt4) +diff --git a/pengine/test10/remote-partial-migrate2.scores b/pengine/test10/remote-partial-migrate2.scores +index 6965507..ce095b1 100644 +--- a/pengine/test10/remote-partial-migrate2.scores ++++ b/pengine/test10/remote-partial-migrate2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing pcmk4: node is unclean + Allocation scores: + native_color: FAKE1 allocation score on pcmk1: -INFINITY + native_color: FAKE1 allocation score on pcmk2: 0 +diff --git a/pengine/test10/remote-partial-migrate2.summary b/pengine/test10/remote-partial-migrate2.summary +index b8b9b4c..c9f9592 100644 +--- a/pengine/test10/remote-partial-migrate2.summary ++++ b/pengine/test10/remote-partial-migrate2.summary +@@ -62,6 +62,7 @@ RemoteOFFLINE: [ pcmk_remote4 ] + FAKE49 (ocf::heartbeat:Dummy): Started pcmk_remote3 + FAKE50 (ocf::heartbeat:Dummy): Started pcmk_remote5 + ++ notice: Fencing pcmk4: node is unclean + Transition Summary: + * Fence (reboot) pcmk4 + * Migrate pcmk_remote2 (Started pcmk3 -> pcmk1) +diff --git a/pengine/test10/remote-recover-all.scores b/pengine/test10/remote-recover-all.scores +index 43cac62..947f228 100644 +--- a/pengine/test10/remote-recover-all.scores ++++ b/pengine/test10/remote-recover-all.scores +@@ -1,3 +1,6 @@ ++ notice: Fencing controller-1: node is unclean ++ notice: Fencing galera-2: node is unclean ++ notice: Fencing messaging-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary +index 5e4f51d..6868533 100644 +--- a/pengine/test10/remote-recover-all.summary ++++ b/pengine/test10/remote-recover-all.summary +@@ -37,6 +37,9 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + ++ notice: Fencing controller-1: node is unclean ++ notice: Fencing galera-2: node is unclean ++ notice: Fencing messaging-1: node is unclean + Transition Summary: + * Fence (reboot) messaging-1 + * Fence (reboot) galera-2 +diff --git a/pengine/test10/remote-recover-connection.scores b/pengine/test10/remote-recover-connection.scores +index 56a18eb..3d3beae 100644 +--- a/pengine/test10/remote-recover-connection.scores ++++ b/pengine/test10/remote-recover-connection.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing controller-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recover-connection.summary b/pengine/test10/remote-recover-connection.summary +index 79ea7da..f21b2b5 100644 +--- a/pengine/test10/remote-recover-connection.summary ++++ b/pengine/test10/remote-recover-connection.summary +@@ -37,6 +37,7 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + ++ notice: Fencing controller-1: node is unclean + Transition Summary: + * Fence (reboot) controller-1 + * Move messaging-1 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-recover-fail.scores b/pengine/test10/remote-recover-fail.scores +index 8fc7805..02a0643 100644 +--- a/pengine/test10/remote-recover-fail.scores ++++ b/pengine/test10/remote-recover-fail.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing rhel7-auto4: node is unclean + Allocation scores: + native_color: FAKE1 allocation score on rhel7-auto1: 0 + native_color: FAKE1 allocation score on rhel7-auto2: 0 +diff --git a/pengine/test10/remote-recover-fail.summary b/pengine/test10/remote-recover-fail.summary +index 313c8ac..fd518eb 100644 +--- a/pengine/test10/remote-recover-fail.summary ++++ b/pengine/test10/remote-recover-fail.summary +@@ -13,6 +13,7 @@ OFFLINE: [ rhel7-auto1 ] + FAKE5 (ocf::heartbeat:Dummy): Started rhel7-auto3 + FAKE6 (ocf::heartbeat:Dummy): Started rhel7-auto4 (UNCLEAN) + ++ notice: Fencing rhel7-auto4: node is unclean + Transition Summary: + * Fence (reboot) rhel7-auto4 + * Recover rhel7-auto4 (Started rhel7-auto2) +diff --git a/pengine/test10/remote-recover-no-resources.scores b/pengine/test10/remote-recover-no-resources.scores +index e918fc6..36a1ff4 100644 +--- a/pengine/test10/remote-recover-no-resources.scores ++++ b/pengine/test10/remote-recover-no-resources.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing controller-1: node is unclean ++ notice: Fencing messaging-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary +index bd27773..c3cd6a8 100644 +--- a/pengine/test10/remote-recover-no-resources.summary ++++ b/pengine/test10/remote-recover-no-resources.summary +@@ -37,6 +37,8 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + ++ notice: Fencing controller-1: node is unclean ++ notice: Fencing messaging-1: node is unclean + Transition Summary: + * Fence (reboot) messaging-1 + * Fence (reboot) controller-1 +diff --git a/pengine/test10/remote-recover-unknown.scores b/pengine/test10/remote-recover-unknown.scores +index e918fc6..d7d7713 100644 +--- a/pengine/test10/remote-recover-unknown.scores ++++ b/pengine/test10/remote-recover-unknown.scores +@@ -1,3 +1,6 @@ ++ notice: Fencing controller-1: node is unclean ++ notice: Fencing galera-2: node is unclean ++ notice: Fencing messaging-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary +index 8a3d560..5b03065 100644 +--- a/pengine/test10/remote-recover-unknown.summary ++++ b/pengine/test10/remote-recover-unknown.summary +@@ -37,6 +37,9 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + ++ notice: Fencing controller-1: node is unclean ++ notice: Fencing galera-2: node is unclean ++ notice: Fencing messaging-1: node is unclean + Transition Summary: + * Fence (reboot) messaging-1 + * Fence (reboot) galera-2 +diff --git a/pengine/test10/remote-recovery.scores b/pengine/test10/remote-recovery.scores +index 56a18eb..3d3beae 100644 +--- a/pengine/test10/remote-recovery.scores ++++ b/pengine/test10/remote-recovery.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing controller-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recovery.summary b/pengine/test10/remote-recovery.summary +index 79ea7da..f21b2b5 100644 +--- a/pengine/test10/remote-recovery.summary ++++ b/pengine/test10/remote-recovery.summary +@@ -37,6 +37,7 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + ++ notice: Fencing controller-1: node is unclean + Transition Summary: + * Fence (reboot) controller-1 + * Move messaging-1 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-unclean2.scores b/pengine/test10/remote-unclean2.scores +index 72a5953..2c2c9a5 100644 +--- a/pengine/test10/remote-unclean2.scores ++++ b/pengine/test10/remote-unclean2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing rhel7-auto4: node is unclean + Allocation scores: + native_color: rhel7-auto4 allocation score on rhel7-auto1: 0 + native_color: rhel7-auto4 allocation score on rhel7-auto2: 0 +diff --git a/pengine/test10/remote-unclean2.summary b/pengine/test10/remote-unclean2.summary +index 877eb4d..06047d1 100644 +--- a/pengine/test10/remote-unclean2.summary ++++ b/pengine/test10/remote-unclean2.summary +@@ -6,6 +6,7 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ] + shooter (stonith:fence_xvm): Started rhel7-auto2 + rhel7-auto4 (ocf::pacemaker:remote): FAILED rhel7-auto1 + ++ notice: Fencing rhel7-auto4: node is unclean + Transition Summary: + * Fence (reboot) rhel7-auto4 + * Recover rhel7-auto4 (Started rhel7-auto1) +diff --git a/pengine/test10/start-then-stop-with-unfence.dot b/pengine/test10/start-then-stop-with-unfence.dot +index d8f7a71..6e9569b 100644 +--- a/pengine/test10/start-then-stop-with-unfence.dot ++++ b/pengine/test10/start-then-stop-with-unfence.dot +@@ -23,7 +23,5 @@ digraph "g" { + "mpath-node2_monitor_0 rhel7-node1.example.com" [ style=bold color="green" fontcolor="black"] + "stonith 'on' rhel7-node1.example.com" -> "ip1_start_0 rhel7-node1.example.com" [ style = bold] + "stonith 'on' rhel7-node1.example.com" -> "jrummy_start_0 rhel7-node1.example.com" [ style = bold] +-"stonith 'on' rhel7-node1.example.com" -> "mpath-node1_monitor_0 rhel7-node1.example.com" [ style = bold] +-"stonith 'on' rhel7-node1.example.com" -> "mpath-node2_monitor_0 rhel7-node1.example.com" [ style = bold] + "stonith 'on' rhel7-node1.example.com" [ style=bold color="green" fontcolor="black"] + } +diff --git a/pengine/test10/start-then-stop-with-unfence.exp b/pengine/test10/start-then-stop-with-unfence.exp +index 359f25b..75cb356 100644 +--- a/pengine/test10/start-then-stop-with-unfence.exp ++++ b/pengine/test10/start-then-stop-with-unfence.exp +@@ -1,16 +1,12 @@ + +- ++ + + + + + + +- +- +- +- +- ++ + + + +@@ -38,18 +34,14 @@ + + + +- ++ + + + + + + +- +- +- +- +- ++ + + + +diff --git a/pengine/test10/start-then-stop-with-unfence.scores b/pengine/test10/start-then-stop-with-unfence.scores +index d353bef..a53e9f2 100644 +--- a/pengine/test10/start-then-stop-with-unfence.scores ++++ b/pengine/test10/start-then-stop-with-unfence.scores +@@ -1,3 +1,4 @@ ++ notice: Unfencing rhel7-node1.example.com: Required by ip1 + Allocation scores: + clone_color: jrummy-clone allocation score on rhel7-node1.example.com: 500 + clone_color: jrummy-clone allocation score on rhel7-node2.example.com: 500 +diff --git a/pengine/test10/start-then-stop-with-unfence.summary b/pengine/test10/start-then-stop-with-unfence.summary +index ae54afc..bb376f5 100644 +--- a/pengine/test10/start-then-stop-with-unfence.summary ++++ b/pengine/test10/start-then-stop-with-unfence.summary +@@ -10,6 +10,7 @@ Online: [ rhel7-node1.example.com rhel7-node2.example.com ] + Started: [ rhel7-node2.example.com ] + Stopped: [ rhel7-node1.example.com ] + ++ notice: Unfencing rhel7-node1.example.com: Required by ip1 + Transition Summary: + * Fence (on) rhel7-node1.example.com + * Start mpath-node1 (rhel7-node1.example.com) +@@ -17,17 +18,17 @@ Transition Summary: + * Start jrummy:1 (rhel7-node1.example.com) + + Executing cluster transition: +- * Pseudo action: jrummy-clone_start_0 +- * Fencing rhel7-node1.example.com (on) + * Resource action: mpath-node2 monitor on rhel7-node1.example.com + * Resource action: mpath-node1 monitor on rhel7-node1.example.com ++ * Pseudo action: jrummy-clone_start_0 ++ * Fencing rhel7-node1.example.com (on) ++ * Resource action: mpath-node1 start on rhel7-node1.example.com + * Resource action: jrummy start on rhel7-node1.example.com + * Pseudo action: jrummy-clone_running_0 +- * Resource action: mpath-node1 start on rhel7-node1.example.com ++ * Resource action: mpath-node1 monitor=60000 on rhel7-node1.example.com + * Resource action: ip1 stop on rhel7-node2.example.com + * Resource action: jrummy monitor=10000 on rhel7-node1.example.com + * Pseudo action: all_stopped +- * Resource action: mpath-node1 monitor=60000 on rhel7-node1.example.com + * Resource action: ip1 start on rhel7-node1.example.com + * Resource action: ip1 monitor=10000 on rhel7-node1.example.com + +diff --git a/pengine/test10/start-then-stop-with-unfence.xml b/pengine/test10/start-then-stop-with-unfence.xml +index 499022e..b856250 100644 +--- a/pengine/test10/start-then-stop-with-unfence.xml ++++ b/pengine/test10/start-then-stop-with-unfence.xml +@@ -95,6 +95,7 @@ + + + ++ + + + +diff --git a/pengine/test10/stonith-0.scores b/pengine/test10/stonith-0.scores +index 7aab8cd..db3ddb9 100644 +--- a/pengine/test10/stonith-0.scores ++++ b/pengine/test10/stonith-0.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing c001n03: node is unclean ++ notice: Fencing c001n05: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n02: 0 + clone_color: DoFencing allocation score on c001n03: 0 +diff --git a/pengine/test10/stonith-0.summary b/pengine/test10/stonith-0.summary +index ee8ae15..6983a90 100644 +--- a/pengine/test10/stonith-0.summary ++++ b/pengine/test10/stonith-0.summary +@@ -36,6 +36,8 @@ Online: [ c001n02 c001n04 c001n06 c001n07 c001n08 ] + ocf_msdummy:12 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n06 + ocf_msdummy:13 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n06 + ++ notice: Fencing c001n03: node is unclean ++ notice: Fencing c001n05: node is unclean + Transition Summary: + * Fence (reboot) c001n05 + * Fence (reboot) c001n03 +diff --git a/pengine/test10/stonith-1.scores b/pengine/test10/stonith-1.scores +index 7422355..196b9f5 100644 +--- a/pengine/test10/stonith-1.scores ++++ b/pengine/test10/stonith-1.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing sles-3: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on sles-1: 0 + clone_color: DoFencing allocation score on sles-2: 0 +diff --git a/pengine/test10/stonith-1.summary b/pengine/test10/stonith-1.summary +index b68aca7..111bcbe 100644 +--- a/pengine/test10/stonith-1.summary ++++ b/pengine/test10/stonith-1.summary +@@ -27,6 +27,7 @@ Online: [ sles-1 sles-2 sles-4 ] + ocf_msdummy:6 (ocf::heartbeat:Stateful): Stopped + ocf_msdummy:7 (ocf::heartbeat:Stateful): Stopped + ++ notice: Fencing sles-3: node is unclean + Transition Summary: + * Fence (reboot) sles-3 + * Start r192.168.100.183 (sles-1) +diff --git a/pengine/test10/stonith-2.scores b/pengine/test10/stonith-2.scores +index e1dd011..8c11561 100644 +--- a/pengine/test10/stonith-2.scores ++++ b/pengine/test10/stonith-2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing sles-5: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on sles-1: 0 + clone_color: DoFencing allocation score on sles-2: 0 +diff --git a/pengine/test10/stonith-2.summary b/pengine/test10/stonith-2.summary +index eef35c4..c8e1d70 100644 +--- a/pengine/test10/stonith-2.summary ++++ b/pengine/test10/stonith-2.summary +@@ -32,6 +32,7 @@ Online: [ sles-1 sles-2 sles-3 sles-4 sles-6 ] + ocf_msdummy:10 (ocf::heartbeat:Stateful): Slave sles-2 + ocf_msdummy:11 (ocf::heartbeat:Stateful): Slave sles-3 + ++ notice: Fencing sles-5: node is unclean + Transition Summary: + * Fence (reboot) sles-5 + * Start rsc_sles-5 (sles-6) +diff --git a/pengine/test10/stonith-3.scores b/pengine/test10/stonith-3.scores +index a1bd3d9..eef30fc 100644 +--- a/pengine/test10/stonith-3.scores ++++ b/pengine/test10/stonith-3.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing rh5node1: node is unclean + Allocation scores: + clone_color: clnStonith allocation score on rh5node1: 0 + clone_color: clnStonith allocation score on rh5node2: 0 +diff --git a/pengine/test10/stonith-3.summary b/pengine/test10/stonith-3.summary +index 398b6bc..015dd77 100644 +--- a/pengine/test10/stonith-3.summary ++++ b/pengine/test10/stonith-3.summary +@@ -7,6 +7,7 @@ Online: [ rh5node2 ] + Clone Set: clnStonith [grpStonith] + Stopped: [ rh5node1 rh5node2 ] + ++ notice: Fencing rh5node1: node is unclean + Transition Summary: + * Fence (reboot) rh5node1 + * Start prmIpPostgreSQLDB (rh5node2) +diff --git a/pengine/test10/stonith-4.scores b/pengine/test10/stonith-4.scores +index ac6fe0b..0bf1cf5 100644 +--- a/pengine/test10/stonith-4.scores ++++ b/pengine/test10/stonith-4.scores +@@ -1,3 +1,7 @@ ++ notice: Fencing pcmk-10: node is unclean ++ notice: Fencing pcmk-5: node is unclean ++ notice: Fencing pcmk-7: node is unclean ++ notice: Fencing pcmk-8: node is unclean + Allocation scores: + native_color: Fencing allocation score on pcmk-10: 0 + native_color: Fencing allocation score on pcmk-11: 0 +diff --git a/pengine/test10/stonith-4.summary b/pengine/test10/stonith-4.summary +index 6e3e396..0c9b5bd 100644 +--- a/pengine/test10/stonith-4.summary ++++ b/pengine/test10/stonith-4.summary +@@ -13,6 +13,10 @@ OFFLINE: [ pcmk-4 pcmk-6 ] + + Fencing (stonith:fence_xvm): Stopped + ++ notice: Fencing pcmk-10: node is unclean ++ notice: Fencing pcmk-5: node is unclean ++ notice: Fencing pcmk-7: node is unclean ++ notice: Fencing pcmk-8: node is unclean + Transition Summary: + * Fence (reboot) pcmk-8 + * Fence (reboot) pcmk-7 +diff --git a/pengine/test10/stop-failure-no-quorum.scores b/pengine/test10/stop-failure-no-quorum.scores +index df30423..c1893b6 100644 +--- a/pengine/test10/stop-failure-no-quorum.scores ++++ b/pengine/test10/stop-failure-no-quorum.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing pcmk-2: node is unclean + Allocation scores: + clone_color: clvm-clone allocation score on pcmk-1: 0 + clone_color: clvm-clone allocation score on pcmk-2: -INFINITY +diff --git a/pengine/test10/stop-failure-no-quorum.summary b/pengine/test10/stop-failure-no-quorum.summary +index 0248126..d91edc1 100644 +--- a/pengine/test10/stop-failure-no-quorum.summary ++++ b/pengine/test10/stop-failure-no-quorum.summary +@@ -14,6 +14,7 @@ Online: [ pcmk-1 ] + ClusterIP (ocf::heartbeat:IPaddr2): Stopped + Fencing (stonith:fence_xvm): Stopped + ++ notice: Fencing pcmk-2: node is unclean + Transition Summary: + * Fence (reboot) pcmk-2 + * Start dlm:0 (pcmk-1 - blocked) +diff --git a/pengine/test10/stop-failure-with-fencing.scores b/pengine/test10/stop-failure-with-fencing.scores +index cf62806..0dc0813 100644 +--- a/pengine/test10/stop-failure-with-fencing.scores ++++ b/pengine/test10/stop-failure-with-fencing.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing pcmk-2: node is unclean + Allocation scores: + clone_color: clvm-clone allocation score on pcmk-1: 0 + clone_color: clvm-clone allocation score on pcmk-2: -INFINITY +diff --git a/pengine/test10/stop-failure-with-fencing.summary b/pengine/test10/stop-failure-with-fencing.summary +index e6c296b..79e6105 100644 +--- a/pengine/test10/stop-failure-with-fencing.summary ++++ b/pengine/test10/stop-failure-with-fencing.summary +@@ -13,6 +13,7 @@ Online: [ pcmk-1 ] + ClusterIP (ocf::heartbeat:IPaddr2): Stopped + Fencing (stonith:fence_xvm): Stopped + ++ notice: Fencing pcmk-2: node is unclean + Transition Summary: + * Fence (reboot) pcmk-2 + * Start dlm:0 (pcmk-1 - blocked) +diff --git a/pengine/test10/systemhealth1.scores b/pengine/test10/systemhealth1.scores +index 8a59654..c55f6e2 100644 +--- a/pengine/test10/systemhealth1.scores ++++ b/pengine/test10/systemhealth1.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealth1.summary b/pengine/test10/systemhealth1.summary +index 7301c6c..335b94a 100644 +--- a/pengine/test10/systemhealth1.summary ++++ b/pengine/test10/systemhealth1.summary +@@ -7,6 +7,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Fence (reboot) hs21c +diff --git a/pengine/test10/systemhealth2.scores b/pengine/test10/systemhealth2.scores +index 8a59654..b20bd49 100644 +--- a/pengine/test10/systemhealth2.scores ++++ b/pengine/test10/systemhealth2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealth2.summary b/pengine/test10/systemhealth2.summary +index c83a7e2..b04d5a1 100644 +--- a/pengine/test10/systemhealth2.summary ++++ b/pengine/test10/systemhealth2.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Start stonith-1 (hs21c) +diff --git a/pengine/test10/systemhealth3.scores b/pengine/test10/systemhealth3.scores +index 8a59654..b20bd49 100644 +--- a/pengine/test10/systemhealth3.scores ++++ b/pengine/test10/systemhealth3.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealth3.summary b/pengine/test10/systemhealth3.summary +index c83a7e2..b04d5a1 100644 +--- a/pengine/test10/systemhealth3.summary ++++ b/pengine/test10/systemhealth3.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Start stonith-1 (hs21c) +diff --git a/pengine/test10/systemhealthm1.scores b/pengine/test10/systemhealthm1.scores +index 8a59654..c55f6e2 100644 +--- a/pengine/test10/systemhealthm1.scores ++++ b/pengine/test10/systemhealthm1.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthm1.summary b/pengine/test10/systemhealthm1.summary +index 7301c6c..335b94a 100644 +--- a/pengine/test10/systemhealthm1.summary ++++ b/pengine/test10/systemhealthm1.summary +@@ -7,6 +7,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Fence (reboot) hs21c +diff --git a/pengine/test10/systemhealthm2.scores b/pengine/test10/systemhealthm2.scores +index 8a59654..b20bd49 100644 +--- a/pengine/test10/systemhealthm2.scores ++++ b/pengine/test10/systemhealthm2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthm2.summary b/pengine/test10/systemhealthm2.summary +index c83a7e2..b04d5a1 100644 +--- a/pengine/test10/systemhealthm2.summary ++++ b/pengine/test10/systemhealthm2.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Start stonith-1 (hs21c) +diff --git a/pengine/test10/systemhealthm3.scores b/pengine/test10/systemhealthm3.scores +index 7e8804e..00b8d64 100644 +--- a/pengine/test10/systemhealthm3.scores ++++ b/pengine/test10/systemhealthm3.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: -INFINITY + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthm3.summary b/pengine/test10/systemhealthm3.summary +index fbcdec5..55d5cf7 100644 +--- a/pengine/test10/systemhealthm3.summary ++++ b/pengine/test10/systemhealthm3.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + +diff --git a/pengine/test10/systemhealthn1.scores b/pengine/test10/systemhealthn1.scores +index 8a59654..c55f6e2 100644 +--- a/pengine/test10/systemhealthn1.scores ++++ b/pengine/test10/systemhealthn1.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthn1.summary b/pengine/test10/systemhealthn1.summary +index 7301c6c..335b94a 100644 +--- a/pengine/test10/systemhealthn1.summary ++++ b/pengine/test10/systemhealthn1.summary +@@ -7,6 +7,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Fence (reboot) hs21c +diff --git a/pengine/test10/systemhealthn2.scores b/pengine/test10/systemhealthn2.scores +index 8a59654..b20bd49 100644 +--- a/pengine/test10/systemhealthn2.scores ++++ b/pengine/test10/systemhealthn2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthn2.summary b/pengine/test10/systemhealthn2.summary +index c83a7e2..b04d5a1 100644 +--- a/pengine/test10/systemhealthn2.summary ++++ b/pengine/test10/systemhealthn2.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Start stonith-1 (hs21c) +diff --git a/pengine/test10/systemhealthn3.scores b/pengine/test10/systemhealthn3.scores +index 8a59654..b20bd49 100644 +--- a/pengine/test10/systemhealthn3.scores ++++ b/pengine/test10/systemhealthn3.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthn3.summary b/pengine/test10/systemhealthn3.summary +index c83a7e2..b04d5a1 100644 +--- a/pengine/test10/systemhealthn3.summary ++++ b/pengine/test10/systemhealthn3.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Start stonith-1 (hs21c) +diff --git a/pengine/test10/systemhealtho1.scores b/pengine/test10/systemhealtho1.scores +index 8a59654..c55f6e2 100644 +--- a/pengine/test10/systemhealtho1.scores ++++ b/pengine/test10/systemhealtho1.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealtho1.summary b/pengine/test10/systemhealtho1.summary +index 7301c6c..335b94a 100644 +--- a/pengine/test10/systemhealtho1.summary ++++ b/pengine/test10/systemhealtho1.summary +@@ -7,6 +7,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Fence (reboot) hs21c +diff --git a/pengine/test10/systemhealtho2.scores b/pengine/test10/systemhealtho2.scores +index 7e8804e..00b8d64 100644 +--- a/pengine/test10/systemhealtho2.scores ++++ b/pengine/test10/systemhealtho2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: -INFINITY + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealtho2.summary b/pengine/test10/systemhealtho2.summary +index fbcdec5..55d5cf7 100644 +--- a/pengine/test10/systemhealtho2.summary ++++ b/pengine/test10/systemhealtho2.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + +diff --git a/pengine/test10/systemhealtho3.scores b/pengine/test10/systemhealtho3.scores +index 7e8804e..00b8d64 100644 +--- a/pengine/test10/systemhealtho3.scores ++++ b/pengine/test10/systemhealtho3.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: -INFINITY + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealtho3.summary b/pengine/test10/systemhealtho3.summary +index fbcdec5..55d5cf7 100644 +--- a/pengine/test10/systemhealtho3.summary ++++ b/pengine/test10/systemhealtho3.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + +diff --git a/pengine/test10/systemhealthp1.scores b/pengine/test10/systemhealthp1.scores +index 8a59654..c55f6e2 100644 +--- a/pengine/test10/systemhealthp1.scores ++++ b/pengine/test10/systemhealthp1.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthp1.summary b/pengine/test10/systemhealthp1.summary +index 7301c6c..335b94a 100644 +--- a/pengine/test10/systemhealthp1.summary ++++ b/pengine/test10/systemhealthp1.summary +@@ -7,6 +7,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21c: node is unclean ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Fence (reboot) hs21c +diff --git a/pengine/test10/systemhealthp2.scores b/pengine/test10/systemhealthp2.scores +index 20a08f2..7ca75bf 100644 +--- a/pengine/test10/systemhealthp2.scores ++++ b/pengine/test10/systemhealthp2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 0 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthp2.summary b/pengine/test10/systemhealthp2.summary +index ea9465b..6c44a74 100644 +--- a/pengine/test10/systemhealthp2.summary ++++ b/pengine/test10/systemhealthp2.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + * Start apache_1 (hs21c) +diff --git a/pengine/test10/systemhealthp3.scores b/pengine/test10/systemhealthp3.scores +index 7e8804e..00b8d64 100644 +--- a/pengine/test10/systemhealthp3.scores ++++ b/pengine/test10/systemhealthp3.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: -INFINITY + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthp3.summary b/pengine/test10/systemhealthp3.summary +index fbcdec5..55d5cf7 100644 +--- a/pengine/test10/systemhealthp3.summary ++++ b/pengine/test10/systemhealthp3.summary +@@ -7,6 +7,7 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + ++ notice: Fencing hs21d: node is unclean + Transition Summary: + * Fence (reboot) hs21d + +diff --git a/pengine/test10/ticket-clone-21.scores b/pengine/test10/ticket-clone-21.scores +index 89e2d0f..e0fc0d8 100644 +--- a/pengine/test10/ticket-clone-21.scores ++++ b/pengine/test10/ticket-clone-21.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing node1: node is unclean ++ notice: Fencing node2: node is unclean + Allocation scores: + clone_color: clone1 allocation score on node1: 0 + clone_color: clone1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-clone-21.summary b/pengine/test10/ticket-clone-21.summary +index 573f8c1..13d2887 100644 +--- a/pengine/test10/ticket-clone-21.summary ++++ b/pengine/test10/ticket-clone-21.summary +@@ -6,6 +6,8 @@ Online: [ node1 node2 ] + Clone Set: clone1 [rsc1] + Started: [ node1 node2 ] + ++ notice: Fencing node1: node is unclean ++ notice: Fencing node2: node is unclean + Transition Summary: + * Fence (reboot) node2 + * Fence (reboot) node1 +diff --git a/pengine/test10/ticket-clone-9.scores b/pengine/test10/ticket-clone-9.scores +index 89e2d0f..e0fc0d8 100644 +--- a/pengine/test10/ticket-clone-9.scores ++++ b/pengine/test10/ticket-clone-9.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing node1: node is unclean ++ notice: Fencing node2: node is unclean + Allocation scores: + clone_color: clone1 allocation score on node1: 0 + clone_color: clone1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-clone-9.summary b/pengine/test10/ticket-clone-9.summary +index 573f8c1..13d2887 100644 +--- a/pengine/test10/ticket-clone-9.summary ++++ b/pengine/test10/ticket-clone-9.summary +@@ -6,6 +6,8 @@ Online: [ node1 node2 ] + Clone Set: clone1 [rsc1] + Started: [ node1 node2 ] + ++ notice: Fencing node1: node is unclean ++ notice: Fencing node2: node is unclean + Transition Summary: + * Fence (reboot) node2 + * Fence (reboot) node1 +diff --git a/pengine/test10/ticket-group-21.scores b/pengine/test10/ticket-group-21.scores +index e5b8167..8f01379 100644 +--- a/pengine/test10/ticket-group-21.scores ++++ b/pengine/test10/ticket-group-21.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node2: node is unclean + Allocation scores: + group_color: group1 allocation score on node1: 0 + group_color: group1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-group-21.summary b/pengine/test10/ticket-group-21.summary +index ff809ae..80fb33f 100644 +--- a/pengine/test10/ticket-group-21.summary ++++ b/pengine/test10/ticket-group-21.summary +@@ -7,6 +7,7 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): Started node2 + rsc2 (ocf::pacemaker:Dummy): Started node2 + ++ notice: Fencing node2: node is unclean + Transition Summary: + * Fence (reboot) node2 + * Stop rsc1 (node2) +diff --git a/pengine/test10/ticket-group-9.scores b/pengine/test10/ticket-group-9.scores +index e5b8167..8f01379 100644 +--- a/pengine/test10/ticket-group-9.scores ++++ b/pengine/test10/ticket-group-9.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node2: node is unclean + Allocation scores: + group_color: group1 allocation score on node1: 0 + group_color: group1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-group-9.summary b/pengine/test10/ticket-group-9.summary +index ff809ae..80fb33f 100644 +--- a/pengine/test10/ticket-group-9.summary ++++ b/pengine/test10/ticket-group-9.summary +@@ -7,6 +7,7 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): Started node2 + rsc2 (ocf::pacemaker:Dummy): Started node2 + ++ notice: Fencing node2: node is unclean + Transition Summary: + * Fence (reboot) node2 + * Stop rsc1 (node2) +diff --git a/pengine/test10/ticket-master-21.scores b/pengine/test10/ticket-master-21.scores +index dc79d67..89eac5e 100644 +--- a/pengine/test10/ticket-master-21.scores ++++ b/pengine/test10/ticket-master-21.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node1: node is unclean + Allocation scores: + clone_color: ms1 allocation score on node1: 0 + clone_color: ms1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-master-21.summary b/pengine/test10/ticket-master-21.summary +index c196c9d..dc2f5db 100644 +--- a/pengine/test10/ticket-master-21.summary ++++ b/pengine/test10/ticket-master-21.summary +@@ -7,6 +7,7 @@ Online: [ node1 node2 ] + Masters: [ node1 ] + Slaves: [ node2 ] + ++ notice: Fencing node1: node is unclean + Transition Summary: + * Fence (reboot) node1 + * Move rsc_stonith (Started node1 -> node2) +diff --git a/pengine/test10/ticket-master-9.scores b/pengine/test10/ticket-master-9.scores +index dc79d67..89eac5e 100644 +--- a/pengine/test10/ticket-master-9.scores ++++ b/pengine/test10/ticket-master-9.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node1: node is unclean + Allocation scores: + clone_color: ms1 allocation score on node1: 0 + clone_color: ms1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-master-9.summary b/pengine/test10/ticket-master-9.summary +index c196c9d..dc2f5db 100644 +--- a/pengine/test10/ticket-master-9.summary ++++ b/pengine/test10/ticket-master-9.summary +@@ -7,6 +7,7 @@ Online: [ node1 node2 ] + Masters: [ node1 ] + Slaves: [ node2 ] + ++ notice: Fencing node1: node is unclean + Transition Summary: + * Fence (reboot) node1 + * Move rsc_stonith (Started node1 -> node2) +diff --git a/pengine/test10/ticket-primitive-21.scores b/pengine/test10/ticket-primitive-21.scores +index ed3f3cd..d1147ca 100644 +--- a/pengine/test10/ticket-primitive-21.scores ++++ b/pengine/test10/ticket-primitive-21.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node2: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: -INFINITY +diff --git a/pengine/test10/ticket-primitive-21.summary b/pengine/test10/ticket-primitive-21.summary +index 08a4860..d3d02fe 100644 +--- a/pengine/test10/ticket-primitive-21.summary ++++ b/pengine/test10/ticket-primitive-21.summary +@@ -5,6 +5,7 @@ Online: [ node1 node2 ] + rsc_stonith (stonith:null): Started node1 + rsc1 (ocf::pacemaker:Dummy): Started node2 + ++ notice: Fencing node2: node is unclean + Transition Summary: + * Fence (reboot) node2 + * Stop rsc1 (node2) +diff --git a/pengine/test10/ticket-primitive-9.scores b/pengine/test10/ticket-primitive-9.scores +index ed3f3cd..d1147ca 100644 +--- a/pengine/test10/ticket-primitive-9.scores ++++ b/pengine/test10/ticket-primitive-9.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing node2: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: -INFINITY +diff --git a/pengine/test10/ticket-primitive-9.summary b/pengine/test10/ticket-primitive-9.summary +index 08a4860..d3d02fe 100644 +--- a/pengine/test10/ticket-primitive-9.summary ++++ b/pengine/test10/ticket-primitive-9.summary +@@ -5,6 +5,7 @@ Online: [ node1 node2 ] + rsc_stonith (stonith:null): Started node1 + rsc1 (ocf::pacemaker:Dummy): Started node2 + ++ notice: Fencing node2: node is unclean + Transition Summary: + * Fence (reboot) node2 + * Stop rsc1 (node2) +diff --git a/pengine/test10/unfence-definition.dot b/pengine/test10/unfence-definition.dot +index a9e7e6b..c16bdb5 100644 +--- a/pengine/test10/unfence-definition.dot ++++ b/pengine/test10/unfence-definition.dot +@@ -19,7 +19,6 @@ digraph "g" { + "clvmd:1_start_0 virt-2" -> "clvmd:2_start_0 virt-3" [ style = bold] + "clvmd:1_start_0 virt-2" [ style=bold color="green" fontcolor="black"] + "clvmd:2_monitor_0 virt-3" -> "clvmd-clone_start_0" [ style = bold] +-"clvmd:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] + "clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold] + "clvmd:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] +@@ -44,7 +43,6 @@ digraph "g" { + "dlm-clone_stopped_0" -> "dlm-clone_start_0" [ style = bold] + "dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] + "dlm:2_monitor_0 virt-3" -> "dlm-clone_start_0" [ style = bold] +-"dlm:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] + "dlm:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] + "dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold] +@@ -70,9 +68,10 @@ digraph "g" { + "stonith 'on' virt-1" -> "clvmd_start_0 virt-1" [ style = bold] + "stonith 'on' virt-1" -> "dlm_start_0 virt-1" [ style = bold] + "stonith 'on' virt-1" [ style=bold color="green" fontcolor="black"] ++"stonith 'on' virt-3" -> "clvmd:2_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"stonith 'on' virt-3" -> "dlm:2_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" -> "dlm:2_start_0 virt-3" [ style = bold] +-"stonith 'on' virt-3" -> "fencing_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold] + "stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/unfence-definition.exp b/pengine/test10/unfence-definition.exp +index 46d5883..f9ad4c1 100644 +--- a/pengine/test10/unfence-definition.exp ++++ b/pengine/test10/unfence-definition.exp +@@ -21,18 +21,14 @@ + + + +- ++ + + + + + + +- +- +- +- +- ++ + + + +@@ -124,7 +120,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -284,7 +284,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -380,19 +384,12 @@ + + + +- +- +- +- +- +- +- +- ++ + + + + +- ++ + + + +diff --git a/pengine/test10/unfence-definition.scores b/pengine/test10/unfence-definition.scores +index d9fa9bb..3d5aa9f 100644 +--- a/pengine/test10/unfence-definition.scores ++++ b/pengine/test10/unfence-definition.scores +@@ -1,3 +1,6 @@ ++ notice: Fencing virt-4: node is unclean ++ notice: Unfencing virt-1: Device definition changed ++ notice: Unfencing virt-3: Required by dlm:2 + Allocation scores: + clone_color: clvmd-clone allocation score on virt-1: 0 + clone_color: clvmd-clone allocation score on virt-2: 0 +diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary +index 9477a02..a31cc20 100644 +--- a/pengine/test10/unfence-definition.summary ++++ b/pengine/test10/unfence-definition.summary +@@ -11,6 +11,9 @@ Online: [ virt-1 virt-2 virt-3 ] + Started: [ virt-1 ] + Stopped: [ virt-2 virt-3 virt-4 ] + ++ notice: Unfencing virt-1: Device definition changed ++ notice: Unfencing virt-3: Required by dlm:2 ++ notice: Fencing virt-4: node is unclean + Transition Summary: + * Fence (reboot) virt-4 + * Fence (on) virt-3 +@@ -23,18 +26,18 @@ Transition Summary: + * Start clvmd:2 (virt-3) + + Executing cluster transition: +- * Resource action: dlm monitor on virt-3 ++ * Resource action: fencing monitor on virt-3 ++ * Resource action: fencing stop on virt-1 + * Resource action: clvmd monitor on virt-2 +- * Resource action: clvmd monitor on virt-3 + * Pseudo action: clvmd-clone_stop_0 + * Fencing virt-4 (reboot) + * Pseudo action: stonith_complete + * Fencing virt-3 (on) +- * Resource action: fencing monitor on virt-3 +- * Resource action: fencing stop on virt-1 ++ * Resource action: fencing delete on virt-1 ++ * Resource action: dlm monitor on virt-3 + * Resource action: clvmd stop on virt-1 ++ * Resource action: clvmd monitor on virt-3 + * Pseudo action: clvmd-clone_stopped_0 +- * Resource action: fencing delete on virt-1 + * Pseudo action: dlm-clone_stop_0 + * Resource action: dlm stop on virt-1 + * Pseudo action: dlm-clone_stopped_0 +diff --git a/pengine/test10/unfence-definition.xml b/pengine/test10/unfence-definition.xml +index 5d77931..cb189cd 100644 +--- a/pengine/test10/unfence-definition.xml ++++ b/pengine/test10/unfence-definition.xml +@@ -41,6 +41,11 @@ + + + ++ ++ ++ ++ ++ + + + +@@ -56,6 +61,11 @@ + + + ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/unfence-parameters.dot b/pengine/test10/unfence-parameters.dot +index c96d314..dee850e 100644 +--- a/pengine/test10/unfence-parameters.dot ++++ b/pengine/test10/unfence-parameters.dot +@@ -14,13 +14,11 @@ digraph "g" { + "clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] + "clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] + "clvmd:1_monitor_0 virt-2" -> "clvmd-clone_start_0" [ style = bold] +-"clvmd:1_monitor_0 virt-2" -> "stonith 'on' virt-2" [ style = bold] + "clvmd:1_monitor_0 virt-2" [ style=bold color="green" fontcolor="black"] + "clvmd:1_start_0 virt-2" -> "clvmd-clone_running_0" [ style = bold] + "clvmd:1_start_0 virt-2" -> "clvmd:2_start_0 virt-3" [ style = bold] + "clvmd:1_start_0 virt-2" [ style=bold color="green" fontcolor="black"] + "clvmd:2_monitor_0 virt-3" -> "clvmd-clone_start_0" [ style = bold] +-"clvmd:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] + "clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold] + "clvmd:2_start_0 virt-3" -> "clvmd:3_start_0 " [ style = dashed] +@@ -50,7 +48,6 @@ digraph "g" { + "dlm-clone_stopped_0" -> "dlm-clone_start_0" [ style = bold] + "dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] + "dlm:2_monitor_0 virt-3" -> "dlm-clone_start_0" [ style = bold] +-"dlm:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] + "dlm:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] + "dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold] +@@ -85,12 +82,14 @@ digraph "g" { + "stonith 'on' virt-1" -> "clvmd_start_0 virt-1" [ style = bold] + "stonith 'on' virt-1" -> "dlm_start_0 virt-1" [ style = bold] + "stonith 'on' virt-1" [ style=bold color="green" fontcolor="black"] ++"stonith 'on' virt-2" -> "clvmd:1_monitor_0 virt-2" [ style = bold] + "stonith 'on' virt-2" -> "clvmd:1_start_0 virt-2" [ style = bold] + "stonith 'on' virt-2" -> "dlm_start_0 virt-2" [ style = bold] + "stonith 'on' virt-2" [ style=bold color="green" fontcolor="black"] ++"stonith 'on' virt-3" -> "clvmd:2_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"stonith 'on' virt-3" -> "dlm:2_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" -> "dlm:2_start_0 virt-3" [ style = bold] +-"stonith 'on' virt-3" -> "fencing_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold] + "stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/unfence-parameters.exp b/pengine/test10/unfence-parameters.exp +index ffc20c0..54c242a 100644 +--- a/pengine/test10/unfence-parameters.exp ++++ b/pengine/test10/unfence-parameters.exp +@@ -12,18 +12,14 @@ + + + +- ++ + + + + + + +- +- +- +- +- ++ + + + +@@ -152,7 +148,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -290,7 +290,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -324,7 +328,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -420,26 +428,16 @@ + + + +- +- +- +- +- +- +- +- ++ + + + + +- ++ + + + + +- +- +- + + + +@@ -447,7 +445,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unfence-parameters.scores b/pengine/test10/unfence-parameters.scores +index d9fa9bb..5089910 100644 +--- a/pengine/test10/unfence-parameters.scores ++++ b/pengine/test10/unfence-parameters.scores +@@ -1,3 +1,7 @@ ++ notice: Fencing virt-4: node is unclean ++ notice: Unfencing virt-1: Device parameters changed (reload) ++ notice: Unfencing virt-2: Device parameters changed (reload) ++ notice: Unfencing virt-3: Device parameters changed (reload) + Allocation scores: + clone_color: clvmd-clone allocation score on virt-1: 0 + clone_color: clvmd-clone allocation score on virt-2: 0 +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +index a713cd5..8effc63 100644 +--- a/pengine/test10/unfence-parameters.summary ++++ b/pengine/test10/unfence-parameters.summary +@@ -11,6 +11,10 @@ Online: [ virt-1 virt-2 virt-3 ] + Started: [ virt-1 ] + Stopped: [ virt-2 virt-3 virt-4 ] + ++ notice: Unfencing virt-1: Device parameters changed (reload) ++ notice: Unfencing virt-2: Device parameters changed (reload) ++ notice: Unfencing virt-3: Device parameters changed (reload) ++ notice: Fencing virt-4: node is unclean + Transition Summary: + * Fence (reboot) virt-4 + * Fence (on) virt-3 +@@ -25,23 +29,23 @@ Transition Summary: + * Start clvmd:2 (virt-3) + + Executing cluster transition: +- * Resource action: dlm monitor on virt-3 +- * Resource action: clvmd monitor on virt-2 +- * Resource action: clvmd monitor on virt-3 ++ * Resource action: fencing monitor on virt-3 + * Pseudo action: clvmd-clone_stop_0 + * Fencing virt-4 (reboot) + * Pseudo action: stonith_complete + * Fencing virt-3 (on) +- * Resource action: fencing monitor on virt-3 ++ * Resource action: fencing stop on virt-1 ++ * Resource action: dlm monitor on virt-3 + * Resource action: clvmd stop on virt-1 ++ * Resource action: clvmd monitor on virt-3 + * Pseudo action: clvmd-clone_stopped_0 +- * Resource action: fencing stop on virt-1 + * Pseudo action: dlm-clone_stop_0 + * Resource action: dlm stop on virt-2 + * Fencing virt-2 (on) + * Resource action: dlm stop on virt-1 + * Pseudo action: dlm-clone_stopped_0 + * Pseudo action: dlm-clone_start_0 ++ * Resource action: clvmd monitor on virt-2 + * Fencing virt-1 (on) + * Pseudo action: all_stopped + * Resource action: fencing start on virt-1 +diff --git a/pengine/test10/unfence-parameters.xml b/pengine/test10/unfence-parameters.xml +index f660100..26588bf 100644 +--- a/pengine/test10/unfence-parameters.xml ++++ b/pengine/test10/unfence-parameters.xml +@@ -41,6 +41,11 @@ + + + ++ ++ ++ ++ ++ + + + +@@ -56,6 +61,11 @@ + + + ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/unfence-startup.dot b/pengine/test10/unfence-startup.dot +index 20f1367..d496956 100644 +--- a/pengine/test10/unfence-startup.dot ++++ b/pengine/test10/unfence-startup.dot +@@ -11,7 +11,6 @@ digraph "g" { + "clvmd:1_start_0 virt-2" -> "clvmd:2_start_0 virt-3" [ style = bold] + "clvmd:1_start_0 virt-2" [ style=bold color="green" fontcolor="black"] + "clvmd:2_monitor_0 virt-3" -> "clvmd-clone_start_0" [ style = bold] +-"clvmd:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] + "clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold] + "clvmd:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] +@@ -21,15 +20,15 @@ digraph "g" { + "dlm-clone_start_0" -> "dlm:2_start_0 virt-3" [ style = bold] + "dlm-clone_start_0" [ style=bold color="green" fontcolor="orange"] + "dlm:2_monitor_0 virt-3" -> "dlm-clone_start_0" [ style = bold] +-"dlm:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] + "dlm:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] + "dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold] + "dlm:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] + "fencing_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"stonith 'on' virt-3" -> "clvmd:2_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"stonith 'on' virt-3" -> "dlm:2_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" -> "dlm:2_start_0 virt-3" [ style = bold] +-"stonith 'on' virt-3" -> "fencing_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold] + "stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/unfence-startup.exp b/pengine/test10/unfence-startup.exp +index a13ffca..70c1686 100644 +--- a/pengine/test10/unfence-startup.exp ++++ b/pengine/test10/unfence-startup.exp +@@ -1,16 +1,12 @@ + +- ++ + + + + + + +- +- +- +- +- ++ + + + +@@ -38,7 +34,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -124,7 +124,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -191,14 +195,7 @@ + + + +- +- +- +- +- +- +- +- ++ + + + +diff --git a/pengine/test10/unfence-startup.scores b/pengine/test10/unfence-startup.scores +index d9fa9bb..0172a9d 100644 +--- a/pengine/test10/unfence-startup.scores ++++ b/pengine/test10/unfence-startup.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing virt-4: node is unclean ++ notice: Unfencing virt-3: Required by dlm:2 + Allocation scores: + clone_color: clvmd-clone allocation score on virt-1: 0 + clone_color: clvmd-clone allocation score on virt-2: 0 +diff --git a/pengine/test10/unfence-startup.summary b/pengine/test10/unfence-startup.summary +index c3f1be4..6a49f42 100644 +--- a/pengine/test10/unfence-startup.summary ++++ b/pengine/test10/unfence-startup.summary +@@ -11,6 +11,8 @@ Online: [ virt-1 virt-2 virt-3 ] + Started: [ virt-1 ] + Stopped: [ virt-2 virt-3 virt-4 ] + ++ notice: Unfencing virt-3: Required by dlm:2 ++ notice: Fencing virt-4: node is unclean + Transition Summary: + * Fence (reboot) virt-4 + * Fence (on) virt-3 +@@ -19,15 +21,15 @@ Transition Summary: + * Start clvmd:2 (virt-3) + + Executing cluster transition: +- * Resource action: dlm monitor on virt-3 +- * Pseudo action: dlm-clone_start_0 ++ * Resource action: fencing monitor on virt-3 + * Resource action: clvmd monitor on virt-2 +- * Resource action: clvmd monitor on virt-3 + * Fencing virt-4 (reboot) + * Pseudo action: stonith_complete + * Fencing virt-3 (on) + * Pseudo action: all_stopped +- * Resource action: fencing monitor on virt-3 ++ * Resource action: dlm monitor on virt-3 ++ * Pseudo action: dlm-clone_start_0 ++ * Resource action: clvmd monitor on virt-3 + * Resource action: dlm start on virt-3 + * Pseudo action: dlm-clone_running_0 + * Pseudo action: clvmd-clone_start_0 +diff --git a/pengine/test10/unfence-startup.xml b/pengine/test10/unfence-startup.xml +index 83e9ffe..28637e3 100644 +--- a/pengine/test10/unfence-startup.xml ++++ b/pengine/test10/unfence-startup.xml +@@ -41,6 +41,11 @@ + + + ++ ++ ++ ++ ++ + + + +@@ -56,6 +61,11 @@ + + + ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/whitebox-fail1.scores b/pengine/test10/whitebox-fail1.scores +index 4fbbf29..02192a9 100644 +--- a/pengine/test10/whitebox-fail1.scores ++++ b/pengine/test10/whitebox-fail1.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing lxc1: guest is unclean + Allocation scores: + clone_color: M-clone allocation score on 18node1: 0 + clone_color: M-clone allocation score on 18node2: 0 +diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary +index 57d72ba..c7b5e19 100644 +--- a/pengine/test10/whitebox-fail1.summary ++++ b/pengine/test10/whitebox-fail1.summary +@@ -13,6 +13,7 @@ Containers: [ lxc2:container2 ] + C (ocf::pacemaker:Dummy): Started lxc2 + D (ocf::pacemaker:Dummy): Started 18node1 + ++ notice: Fencing lxc1: guest is unclean + Transition Summary: + * Fence (reboot) lxc1 (resource: container1) + * Recover container1 (Started 18node2) +diff --git a/pengine/test10/whitebox-fail2.scores b/pengine/test10/whitebox-fail2.scores +index 4fbbf29..02192a9 100644 +--- a/pengine/test10/whitebox-fail2.scores ++++ b/pengine/test10/whitebox-fail2.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing lxc1: guest is unclean + Allocation scores: + clone_color: M-clone allocation score on 18node1: 0 + clone_color: M-clone allocation score on 18node2: 0 +diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary +index bf12683..73e968e 100644 +--- a/pengine/test10/whitebox-fail2.summary ++++ b/pengine/test10/whitebox-fail2.summary +@@ -13,6 +13,7 @@ Containers: [ lxc2:container2 ] + C (ocf::pacemaker:Dummy): Started lxc2 + D (ocf::pacemaker:Dummy): Started 18node1 + ++ notice: Fencing lxc1: guest is unclean + Transition Summary: + * Fence (reboot) lxc1 (resource: container1) + * Recover container1 (Started 18node2) +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.scores b/pengine/test10/whitebox-imply-stop-on-fence.scores +index e50f077..5c3e325 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.scores ++++ b/pengine/test10/whitebox-imply-stop-on-fence.scores +@@ -1,3 +1,6 @@ ++ notice: Fencing kiff-01: node is unclean ++ notice: Fencing lxc-01_kiff-01: guest is unclean ++ notice: Fencing lxc-02_kiff-01: guest is unclean + Allocation scores: + clone_color: clvmd-clone allocation score on kiff-01: 0 + clone_color: clvmd-clone allocation score on kiff-02: 0 +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary +index 2fb20a6..a74bb72 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.summary ++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary +@@ -24,6 +24,9 @@ Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ] + R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01 + ++ notice: Fencing kiff-01: node is unclean ++ notice: Fencing lxc-01_kiff-01: guest is unclean ++ notice: Fencing lxc-02_kiff-01: guest is unclean + Transition Summary: + * Fence (reboot) lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) + * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) +diff --git a/pengine/test10/whitebox-ms-ordering.scores b/pengine/test10/whitebox-ms-ordering.scores +index f0e51f5..72e6d25 100644 +--- a/pengine/test10/whitebox-ms-ordering.scores ++++ b/pengine/test10/whitebox-ms-ordering.scores +@@ -1,3 +1,5 @@ ++ notice: Fencing lxc1: guest is unclean ++ notice: Fencing lxc2: guest is unclean + Allocation scores: + clone_color: lxc-ms-master allocation score on 18node1: 0 + clone_color: lxc-ms-master allocation score on 18node2: 0 +diff --git a/pengine/test10/whitebox-ms-ordering.summary b/pengine/test10/whitebox-ms-ordering.summary +index 2230d0f..41436f8 100644 +--- a/pengine/test10/whitebox-ms-ordering.summary ++++ b/pengine/test10/whitebox-ms-ordering.summary +@@ -8,6 +8,8 @@ Online: [ 18node1 18node2 18node3 ] + Master/Slave Set: lxc-ms-master [lxc-ms] + Stopped: [ 18node1 18node2 18node3 ] + ++ notice: Fencing lxc1: guest is unclean ++ notice: Fencing lxc2: guest is unclean + Transition Summary: + * Fence (reboot) lxc2 (resource: container2) + * Fence (reboot) lxc1 (resource: container1) +diff --git a/pengine/test10/whitebox-unexpectedly-running.scores b/pengine/test10/whitebox-unexpectedly-running.scores +index 7f8a1d9..eb8a6f9 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.scores ++++ b/pengine/test10/whitebox-unexpectedly-running.scores +@@ -1,3 +1,4 @@ ++ notice: Fencing remote1: guest is unclean + Allocation scores: + native_color: FAKE allocation score on 18builder: 0 + native_color: FAKE allocation score on remote1: -INFINITY +diff --git a/pengine/test10/whitebox-unexpectedly-running.summary b/pengine/test10/whitebox-unexpectedly-running.summary +index eabeb4d..3888aa2 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.summary ++++ b/pengine/test10/whitebox-unexpectedly-running.summary +@@ -4,6 +4,7 @@ Online: [ 18builder ] + + FAKE (ocf::pacemaker:Dummy): FAILED 18builder + ++ notice: Fencing remote1: guest is unclean + Transition Summary: + * Fence (reboot) remote1 (resource: FAKE) + * Recover FAKE (Started 18builder) +-- +1.8.3.1 + + +From 736725733cc23897b4f18d59db24747077092ed6 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 26 Jul 2017 13:51:15 +1000 +Subject: [PATCH 03/12] Fix: Correctly compare remote unfencing digests + +--- + lib/pengine/utils.c | 17 ++++++++++++----- + 1 file changed, 12 insertions(+), 5 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 90da95a..5d2a1b9 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -2112,20 +1944,24 @@ fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * da + const char *digest_all = g_hash_table_lookup(node->details->attrs, "digests-all"); + const char *digest_secure = g_hash_table_lookup(node->details->attrs, "digests-secure"); + +- /* No restarts for fencing device changes */ ++ /* No 'reloads' for fencing device changes ++ * ++ * We use the resource id + agent + digest so that we can detect ++ * changes to the agent and/or the parameters used ++ */ ++ char *search_all = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); ++ char *search_secure = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); + + data->rc = RSC_DIGEST_ALL; + if (digest_all == NULL) { + /* it is unknown what the previous op digest was */ + data->rc = RSC_DIGEST_UNKNOWN; + +- } else if (strcmp(digest_all, data->digest_all_calc) == 0) { ++ } else if (strstr(digest_all, search_all)) { + data->rc = RSC_DIGEST_MATCH; + + } else if(digest_secure && data->digest_secure_calc) { +- char *search = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); +- +- if(strstr(digest_secure, search)) { ++ if(strstr(digest_secure, search_secure)) { + fprintf(stdout, "Only 'private' parameters to %s for unfencing %s changed\n", + rsc->id, node->details->uname); + data->rc = RSC_DIGEST_MATCH; +@@ -2137,7 +1974,11 @@ fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * da + rsc->id, node->details->uname, rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); + } + ++ + free(key); ++ free(search_all); ++ free(search_secure); ++ + return data; + } + +-- +1.8.3.1 + + +From e0dbe6cf3ef982f6644cc4d4e81381bdf184b0d6 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 26 Jul 2017 13:47:58 +1000 +Subject: [PATCH 04/12] Log: Mechanism for recording why an action is required + +--- + include/crm/pengine/internal.h | 3 + + include/crm/pengine/status.h | 1 + + lib/pengine/unpack.c | 2 + + lib/pengine/utils.c | 30 +- + pengine/allocate.c | 8 +- + pengine/native.c | 8 +- + pengine/test10/594.exp | 113 ++++--- + pengine/test10/594.scores | 1 - + pengine/test10/594.summary | 5 +- + pengine/test10/797.exp | 134 ++++---- + pengine/test10/829.exp | 115 ++++--- + pengine/test10/829.scores | 1 - + pengine/test10/829.summary | 5 +- + pengine/test10/bug-5186-partial-migrate.exp | 178 +++++------ + pengine/test10/bug-5186-partial-migrate.scores | 1 - + pengine/test10/bug-5186-partial-migrate.summary | 5 +- + pengine/test10/bug-cl-5212.exp | 52 ++-- + pengine/test10/bug-cl-5247.scores | 1 - + pengine/test10/bug-cl-5247.summary | 3 +- + pengine/test10/bug-lf-2508.exp | 243 ++++++++------- + pengine/test10/bug-lf-2508.scores | 1 - + pengine/test10/bug-lf-2508.summary | 11 +- + pengine/test10/bug-lf-2551.exp | 325 ++++++++++---------- + pengine/test10/bug-lf-2551.scores | 1 - + pengine/test10/bug-lf-2551.summary | 7 +- + pengine/test10/bug-lf-2606.exp | 91 +++--- + pengine/test10/bug-lf-2606.scores | 1 - + pengine/test10/bug-lf-2606.summary | 5 +- + pengine/test10/bug-rh-1097457.scores | 1 - + pengine/test10/bug-rh-1097457.summary | 3 +- + pengine/test10/concurrent-fencing.exp | 72 ++--- + pengine/test10/concurrent-fencing.scores | 3 - + pengine/test10/concurrent-fencing.summary | 9 +- + pengine/test10/guest-node-host-dies.exp | 300 +++++++++--------- + pengine/test10/guest-node-host-dies.scores | 3 - + pengine/test10/guest-node-host-dies.summary | 9 +- + pengine/test10/interleave-pseudo-stop.exp | 191 ++++++------ + pengine/test10/interleave-pseudo-stop.scores | 1 - + pengine/test10/interleave-pseudo-stop.summary | 5 +- + pengine/test10/master-7.exp | 263 ++++++++-------- + pengine/test10/master-7.scores | 1 - + pengine/test10/master-7.summary | 9 +- + pengine/test10/master-8.exp | 295 +++++++++--------- + pengine/test10/master-8.scores | 1 - + pengine/test10/master-8.summary | 9 +- + pengine/test10/migrate-fencing.exp | 291 +++++++++--------- + pengine/test10/migrate-fencing.scores | 1 - + pengine/test10/migrate-fencing.summary | 15 +- + pengine/test10/rec-node-10.exp | 6 +- + pengine/test10/rec-node-11.exp | 127 ++++---- + pengine/test10/rec-node-11.scores | 1 - + pengine/test10/rec-node-11.summary | 5 +- + pengine/test10/rec-node-12.scores | 1 - + pengine/test10/rec-node-12.summary | 3 +- + pengine/test10/rec-node-13.exp | 51 ++-- + pengine/test10/rec-node-13.scores | 1 - + pengine/test10/rec-node-13.summary | 5 +- + pengine/test10/rec-node-14.exp | 61 ++-- + pengine/test10/rec-node-14.scores | 3 - + pengine/test10/rec-node-14.summary | 9 +- + pengine/test10/rec-node-15.exp | 207 +++++++------ + pengine/test10/rec-node-15.scores | 1 - + pengine/test10/rec-node-15.summary | 19 +- + pengine/test10/rec-node-2.scores | 1 - + pengine/test10/rec-node-2.summary | 3 +- + pengine/test10/rec-node-4.exp | 77 +++-- + pengine/test10/rec-node-4.scores | 1 - + pengine/test10/rec-node-4.summary | 5 +- + pengine/test10/rec-node-5.exp | 12 +- + pengine/test10/rec-node-6.exp | 77 +++-- + pengine/test10/rec-node-6.scores | 1 - + pengine/test10/rec-node-6.summary | 5 +- + pengine/test10/rec-node-7.exp | 77 +++-- + pengine/test10/rec-node-7.scores | 1 - + pengine/test10/rec-node-7.summary | 5 +- + pengine/test10/rec-node-8.exp | 8 +- + pengine/test10/rec-rsc-5.exp | 77 +++-- + pengine/test10/rec-rsc-5.scores | 1 - + pengine/test10/rec-rsc-5.summary | 5 +- + pengine/test10/remote-fence-before-reconnect.exp | 54 ++-- + .../test10/remote-fence-before-reconnect.scores | 1 - + .../test10/remote-fence-before-reconnect.summary | 5 +- + pengine/test10/remote-fence-unclean-3.exp | 34 +-- + pengine/test10/remote-fence-unclean-3.scores | 1 - + pengine/test10/remote-fence-unclean-3.summary | 3 +- + pengine/test10/remote-fence-unclean.exp | 90 +++--- + pengine/test10/remote-fence-unclean.scores | 1 - + pengine/test10/remote-fence-unclean.summary | 13 +- + pengine/test10/remote-fence-unclean2.exp | 38 +-- + pengine/test10/remote-fence-unclean2.scores | 1 - + pengine/test10/remote-fence-unclean2.summary | 5 +- + pengine/test10/remote-partial-migrate2.exp | 336 ++++++++++----------- + pengine/test10/remote-partial-migrate2.scores | 1 - + pengine/test10/remote-partial-migrate2.summary | 3 +- + pengine/test10/remote-recover-all.exp | 318 +++++++++---------- + pengine/test10/remote-recover-all.scores | 3 - + pengine/test10/remote-recover-all.summary | 9 +- + pengine/test10/remote-recover-connection.exp | 248 +++++++-------- + pengine/test10/remote-recover-connection.scores | 1 - + pengine/test10/remote-recover-connection.summary | 5 +- + pengine/test10/remote-recover-fail.exp | 88 +++--- + pengine/test10/remote-recover-fail.scores | 1 - + pengine/test10/remote-recover-fail.summary | 9 +- + pengine/test10/remote-recover-no-resources.exp | 276 ++++++++--------- + pengine/test10/remote-recover-no-resources.scores | 2 - + pengine/test10/remote-recover-no-resources.summary | 6 +- + pengine/test10/remote-recover-unknown.exp | 288 +++++++++--------- + pengine/test10/remote-recover-unknown.scores | 3 - + pengine/test10/remote-recover-unknown.summary | 27 +- + pengine/test10/remote-recover.exp | 16 +- + pengine/test10/remote-recovery.exp | 248 +++++++-------- + pengine/test10/remote-recovery.scores | 1 - + pengine/test10/remote-recovery.summary | 5 +- + pengine/test10/remote-unclean2.exp | 44 +-- + pengine/test10/remote-unclean2.scores | 1 - + pengine/test10/remote-unclean2.summary | 3 +- + pengine/test10/start-then-stop-with-unfence.scores | 1 - + .../test10/start-then-stop-with-unfence.summary | 3 +- + pengine/test10/stonith-0.exp | 222 +++++++------- + pengine/test10/stonith-0.scores | 2 - + pengine/test10/stonith-0.summary | 8 +- + pengine/test10/stonith-1.exp | 237 ++++++++------- + pengine/test10/stonith-1.scores | 1 - + pengine/test10/stonith-1.summary | 11 +- + pengine/test10/stonith-2.exp | 41 ++- + pengine/test10/stonith-2.scores | 1 - + pengine/test10/stonith-2.summary | 3 +- + pengine/test10/stonith-3.scores | 1 - + pengine/test10/stonith-3.summary | 3 +- + pengine/test10/stonith-4.exp | 68 ++--- + pengine/test10/stonith-4.scores | 4 - + pengine/test10/stonith-4.summary | 12 +- + pengine/test10/stop-failure-no-quorum.exp | 50 +-- + pengine/test10/stop-failure-no-quorum.scores | 1 - + pengine/test10/stop-failure-no-quorum.summary | 5 +- + pengine/test10/stop-failure-with-fencing.exp | 52 ++-- + pengine/test10/stop-failure-with-fencing.scores | 1 - + pengine/test10/stop-failure-with-fencing.summary | 5 +- + pengine/test10/systemhealth1.scores | 2 - + pengine/test10/systemhealth1.summary | 6 +- + pengine/test10/systemhealth2.scores | 1 - + pengine/test10/systemhealth2.summary | 3 +- + pengine/test10/systemhealth3.scores | 1 - + pengine/test10/systemhealth3.summary | 3 +- + pengine/test10/systemhealthm1.scores | 2 - + pengine/test10/systemhealthm1.summary | 6 +- + pengine/test10/systemhealthm2.scores | 1 - + pengine/test10/systemhealthm2.summary | 3 +- + pengine/test10/systemhealthm3.scores | 1 - + pengine/test10/systemhealthm3.summary | 3 +- + pengine/test10/systemhealthn1.scores | 2 - + pengine/test10/systemhealthn1.summary | 6 +- + pengine/test10/systemhealthn2.scores | 1 - + pengine/test10/systemhealthn2.summary | 3 +- + pengine/test10/systemhealthn3.scores | 1 - + pengine/test10/systemhealthn3.summary | 3 +- + pengine/test10/systemhealtho1.scores | 2 - + pengine/test10/systemhealtho1.summary | 6 +- + pengine/test10/systemhealtho2.scores | 1 - + pengine/test10/systemhealtho2.summary | 3 +- + pengine/test10/systemhealtho3.scores | 1 - + pengine/test10/systemhealtho3.summary | 3 +- + pengine/test10/systemhealthp1.scores | 2 - + pengine/test10/systemhealthp1.summary | 6 +- + pengine/test10/systemhealthp2.scores | 1 - + pengine/test10/systemhealthp2.summary | 3 +- + pengine/test10/systemhealthp3.scores | 1 - + pengine/test10/systemhealthp3.summary | 3 +- + pengine/test10/ticket-clone-21.exp | 97 +++--- + pengine/test10/ticket-clone-21.scores | 2 - + pengine/test10/ticket-clone-21.summary | 8 +- + pengine/test10/ticket-clone-9.exp | 97 +++--- + pengine/test10/ticket-clone-9.scores | 2 - + pengine/test10/ticket-clone-9.summary | 8 +- + pengine/test10/ticket-group-21.exp | 63 ++-- + pengine/test10/ticket-group-21.scores | 1 - + pengine/test10/ticket-group-21.summary | 5 +- + pengine/test10/ticket-group-9.exp | 63 ++-- + pengine/test10/ticket-group-9.scores | 1 - + pengine/test10/ticket-group-9.summary | 5 +- + pengine/test10/ticket-master-21.exp | 77 +++-- + pengine/test10/ticket-master-21.scores | 1 - + pengine/test10/ticket-master-21.summary | 5 +- + pengine/test10/ticket-master-9.exp | 77 +++-- + pengine/test10/ticket-master-9.scores | 1 - + pengine/test10/ticket-master-9.summary | 5 +- + pengine/test10/ticket-primitive-21.exp | 39 ++- + pengine/test10/ticket-primitive-21.scores | 1 - + pengine/test10/ticket-primitive-21.summary | 5 +- + pengine/test10/ticket-primitive-9.exp | 39 ++- + pengine/test10/ticket-primitive-9.scores | 1 - + pengine/test10/ticket-primitive-9.summary | 5 +- + pengine/test10/unfence-definition.scores | 3 - + pengine/test10/unfence-definition.summary | 9 +- + pengine/test10/unfence-parameters.scores | 4 - + pengine/test10/unfence-parameters.summary | 12 +- + pengine/test10/unfence-startup.scores | 2 - + pengine/test10/unfence-startup.summary | 6 +- + pengine/test10/unrunnable-1.exp | 28 +- + pengine/test10/whitebox-fail1.scores | 1 - + pengine/test10/whitebox-fail1.summary | 3 +- + pengine/test10/whitebox-fail2.scores | 1 - + pengine/test10/whitebox-fail2.summary | 3 +- + pengine/test10/whitebox-imply-stop-on-fence.exp | 212 ++++++------- + pengine/test10/whitebox-imply-stop-on-fence.scores | 3 - + .../test10/whitebox-imply-stop-on-fence.summary | 9 +- + pengine/test10/whitebox-ms-ordering.scores | 2 - + pengine/test10/whitebox-ms-ordering.summary | 6 +- + .../test10/whitebox-unexpectedly-running.scores | 1 - + .../test10/whitebox-unexpectedly-running.summary | 3 +- + 210 files changed, 3715 insertions(+), 3945 deletions(-) + +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index 3e10a30..99e2055 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -277,6 +277,9 @@ action_t *pe_fence_op(node_t * node, const char *op, bool optional, const char * + void trigger_unfencing( + resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set); + ++void pe_action_required_worker(pe_action_t *action, const char *reason, const char *function, long line); ++#define pe_action_required(action, reason) pe_action_required_worker(action, reason, __FUNCTION__, __LINE__) ++ + void set_bit_recursive(resource_t * rsc, unsigned long long flag); + void clear_bit_recursive(resource_t * rsc, unsigned long long flag); + +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index cfb7d97..d9e8354 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -323,6 +323,7 @@ struct pe_action_s { + + char *task; + char *uuid; ++ char *reason; + char *cancel_task; + + enum pe_action_flags flags; +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 629c557..299f9f6 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -121,6 +121,7 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + reason); + } + node->details->unclean = TRUE; ++ pe_fence_op(node, NULL, TRUE, reason, data_set); + + } else if (node->details->unclean) { + crm_trace("Cluster node %s %s because %s", +@@ -134,6 +135,7 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + pe_can_fence(data_set, node)? "will be fenced" : "is unclean", + reason); + node->details->unclean = TRUE; ++ pe_fence_op(node, NULL, TRUE, reason, data_set); + } + } + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 5d2a1b9..a53be6a 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1207,6 +1207,7 @@ pe_free_action(action_t * action) + g_hash_table_destroy(action->meta); + } + free(action->cancel_task); ++ free(action->reason); + free(action->task); + free(action->uuid); + free(action->node); +@@ -2088,7 +2089,6 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + /* Extra detail for those running from the commandline */ + fprintf(stdout, " notice: Unfencing %s (remote): because the definition of %s changed\n", node->details->uname, match->id); + } +- + } + + digests_all_offset += snprintf( +@@ -2107,20 +2107,10 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + free(op_key); + } + +- if(optional == FALSE && is_set(stonith_op->flags, pe_action_optional)) { +- const char *kind = "Fencing "; +- +- pe_clear_action_bit(stonith_op, pe_action_optional); +- +- if(safe_str_eq(op, "on")) { +- kind = "Unfencing"; +- } +- +- crm_notice("%s %s: %s", kind, node->details->uname, reason); +- if (is_set(data_set->flags, pe_flag_sanitized)) { +- /* Extra detail for those running from the commandline */ +- fprintf(stdout, " notice: %s %s: %s\n", kind, node->details->uname, reason); +- } ++ if(optional == FALSE && pe_can_fence(data_set, node)) { ++ pe_action_required(stonith_op, reason); ++ } else if(reason && stonith_op->reason == NULL) { ++ stonith_op->reason = strdup(reason); + } + + return stonith_op; +@@ -2196,3 +2186,13 @@ add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref) + + return TRUE; + } ++ ++void pe_action_required_worker(pe_action_t *action, const char *reason, const char *function, long line) ++{ ++ if(is_set(action->flags, pe_action_optional)) { ++ action->flags = crm_clear_bit(function, line, action->uuid, action->flags, pe_action_optional); ++ if(action->reason == NULL) { ++ action->reason = strdup(reason); ++ } ++ } ++} +diff --git a/pengine/allocate.c b/pengine/allocate.c +index fd9c4f9..ef26245 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1486,9 +1486,8 @@ stage6(pe_working_set_t * data_set) + if (node->details->unclean + && need_stonith && pe_can_fence(data_set, node)) { + +- pe_warn("Scheduling Node %s for STONITH", node->details->uname); +- + stonith_op = pe_fence_op(node, NULL, FALSE, "node is unclean", data_set); ++ pe_warn("Scheduling Node %s for STONITH", node->details->uname); + + stonith_constraints(node, stonith_op, data_set); + +@@ -2434,6 +2433,7 @@ LogNodeActions(pe_working_set_t * data_set, gboolean terminal) + node_name = crm_strdup_printf("%s", action->node->details->uname); + } + ++ + if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) { + task = strdup("Shutdown"); + } else if (safe_str_eq(action->task, CRM_OP_FENCE)) { +@@ -2443,8 +2443,12 @@ LogNodeActions(pe_working_set_t * data_set, gboolean terminal) + + if(task == NULL) { + /* Nothing to report */ ++ } else if(terminal && action->reason) { ++ printf(" * %s %s '%s'\n", task, node_name, action->reason); + } else if(terminal) { + printf(" * %s %s\n", task, node_name); ++ } else if(action->reason) { ++ crm_notice(" * %s %s '%s'\n", task, node_name, action->reason); + } else { + crm_notice(" * %s %s\n", task, node_name); + } +diff --git a/pengine/native.c b/pengine/native.c +index bc59405..16c9d5a 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -1355,7 +1355,7 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + + g_hash_table_iter_init(&iter, rsc->allowed_nodes); + while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { +- action_t *unfence = pe_fence_op(node, "on", TRUE, __FUNCTION__, data_set); ++ action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set); + + crm_debug("Ordering any stops of %s before %s, and any starts after", + rsc->id, unfence->uuid); +@@ -2457,7 +2457,7 @@ StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * d + } + + if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { +- action_t *unfence = pe_fence_op(current, "on", TRUE, __FUNCTION__, data_set); ++ action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, data_set); + const char *unfenced = g_hash_table_lookup(current->details->attrs, XML_NODE_IS_UNFENCED); + + order_actions(stop, unfence, pe_order_implies_first); +@@ -2480,7 +2480,7 @@ StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * + start = start_action(rsc, next, TRUE); + + if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { +- action_t *unfence = pe_fence_op(next, "on", TRUE, __FUNCTION__, data_set); ++ action_t *unfence = pe_fence_op(next, "on", TRUE, NULL, data_set); + const char *unfenced = g_hash_table_lookup(next->details->attrs, XML_NODE_IS_UNFENCED); + + order_actions(unfence, start, pe_order_implies_then); +@@ -2847,7 +2847,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + * unfencing and that unfencing occurred. + */ + if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { +- action_t *unfence = pe_fence_op(node, "on", TRUE, __FUNCTION__, data_set); ++ action_t *unfence = pe_fence_op(node, "on", TRUE, NULL, data_set); + order_actions(unfence, probe, pe_order_optional); + } + +diff --git a/pengine/test10/594.exp b/pengine/test10/594.exp +index 5d58ab4..400a5e7 100644 +--- a/pengine/test10/594.exp ++++ b/pengine/test10/594.exp +@@ -1,52 +1,52 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -64,52 +64,52 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -118,23 +118,23 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -143,20 +143,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -165,25 +165,25 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -191,18 +191,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -211,51 +200,61 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/594.scores b/pengine/test10/594.scores +index 430550c..972b89b 100644 +--- a/pengine/test10/594.scores ++++ b/pengine/test10/594.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hadev3: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on hadev1: 0 + clone_color: DoFencing allocation score on hadev2: 0 +diff --git a/pengine/test10/594.summary b/pengine/test10/594.summary +index 6cd78d3..44ed113 100644 +--- a/pengine/test10/594.summary ++++ b/pengine/test10/594.summary +@@ -12,10 +12,9 @@ Online: [ hadev1 hadev2 ] + child_DoFencing:1 (stonith:ssh): Started hadev1 + child_DoFencing:2 (stonith:ssh): Started hadev1 + +- notice: Fencing hadev3: node is unclean + Transition Summary: +- * Fence (reboot) hadev3 + * Shutdown hadev2 ++ * Fence (reboot) hadev3 'peer is no longer part of the cluster' + * Move DcIPaddr (Started hadev2 -> hadev1) + * Move rsc_hadev2 (Started hadev2 -> hadev1) + * Stop child_DoFencing:0 (hadev2) +@@ -29,13 +28,13 @@ Executing cluster transition: + * Resource action: child_DoFencing:2 monitor on hadev2 + * Pseudo action: DoFencing_stop_0 + * Fencing hadev3 (reboot) +- * Pseudo action: stonith_complete + * Resource action: DcIPaddr stop on hadev2 + * Resource action: rsc_hadev2 stop on hadev2 + * Resource action: child_DoFencing:0 stop on hadev2 + * Resource action: child_DoFencing:2 stop on hadev1 + * Pseudo action: DoFencing_stopped_0 + * Cluster action: do_shutdown on hadev2 ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + * Resource action: DcIPaddr start on hadev1 + * Resource action: rsc_hadev2 start on hadev1 +diff --git a/pengine/test10/797.exp b/pengine/test10/797.exp +index 51f64cb..4b83374 100644 +--- a/pengine/test10/797.exp ++++ b/pengine/test10/797.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -19,49 +19,49 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -70,7 +70,7 @@ + + + +- ++ + + + +@@ -79,23 +79,23 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -104,7 +104,7 @@ + + + +- ++ + + + +@@ -113,20 +113,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -135,7 +135,7 @@ + + + +- ++ + + + +@@ -144,20 +144,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -166,78 +166,78 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -246,7 +246,7 @@ + + + +- ++ + + + +@@ -255,7 +255,7 @@ + + + +- ++ + + + +@@ -264,7 +264,7 @@ + + + +- ++ + + + +@@ -273,28 +273,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -302,46 +302,46 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -350,46 +350,46 @@ + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/829.exp b/pengine/test10/829.exp +index 115a39f..d1f85fa 100644 +--- a/pengine/test10/829.exp ++++ b/pengine/test10/829.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -37,66 +37,66 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -105,7 +105,7 @@ + + + +- ++ + + + +@@ -114,7 +114,7 @@ + + + +- ++ + + + +@@ -123,7 +123,7 @@ + + + +- ++ + + + +@@ -132,7 +132,7 @@ + + + +- ++ + + + +@@ -141,7 +141,7 @@ + + + +- ++ + + + +@@ -150,7 +150,7 @@ + + + +- ++ + + + +@@ -159,25 +159,25 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -186,7 +186,7 @@ + + + +- ++ + + + +@@ -195,7 +195,7 @@ + + + +- ++ + + + +@@ -204,7 +204,7 @@ + + + +- ++ + + + +@@ -213,7 +213,7 @@ + + + +- ++ + + + +@@ -222,7 +222,7 @@ + + + +- ++ + + + +@@ -231,71 +231,70 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/829.scores b/pengine/test10/829.scores +index 71a372b..f0652ff 100644 +--- a/pengine/test10/829.scores ++++ b/pengine/test10/829.scores +@@ -1,4 +1,3 @@ +- notice: Fencing c001n02: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n01: 0 + clone_color: DoFencing allocation score on c001n02: 0 +diff --git a/pengine/test10/829.summary b/pengine/test10/829.summary +index 8649ed8..9e66733 100644 +--- a/pengine/test10/829.summary ++++ b/pengine/test10/829.summary +@@ -14,9 +14,8 @@ Online: [ c001n01 c001n03 c001n08 ] + child_DoFencing:2 (stonith:ssh): Started c001n01 + child_DoFencing:3 (stonith:ssh): Started c001n08 + +- notice: Fencing c001n02: node is unclean + Transition Summary: +- * Fence (reboot) c001n02 ++ * Fence (reboot) c001n02 'peer is no longer part of the cluster' + * Move rsc_c001n02 (Started c001n02 -> c001n01) + * Stop child_DoFencing:0 (c001n02) + +@@ -39,9 +38,9 @@ Executing cluster transition: + * Resource action: child_DoFencing:3 monitor on c001n03 + * Resource action: child_DoFencing:3 monitor on c001n01 + * Fencing c001n02 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: rsc_c001n02_stop_0 + * Pseudo action: DoFencing_stop_0 ++ * Pseudo action: stonith_complete + * Resource action: rsc_c001n02 start on c001n01 + * Pseudo action: child_DoFencing:0_stop_0 + * Pseudo action: DoFencing_stopped_0 +diff --git a/pengine/test10/bug-5186-partial-migrate.exp b/pengine/test10/bug-5186-partial-migrate.exp +index bba083e..d9e806f 100644 +--- a/pengine/test10/bug-5186-partial-migrate.exp ++++ b/pengine/test10/bug-5186-partial-migrate.exp +@@ -1,79 +1,79 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -82,25 +82,25 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -108,305 +108,305 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/bug-5186-partial-migrate.scores b/pengine/test10/bug-5186-partial-migrate.scores +index dd59f34..8b29529 100644 +--- a/pengine/test10/bug-5186-partial-migrate.scores ++++ b/pengine/test10/bug-5186-partial-migrate.scores +@@ -1,4 +1,3 @@ +- notice: Fencing bl460g1n7: node is unclean + Allocation scores: + clone_color: clnDiskd1 allocation score on bl460g1n6: 200 + clone_color: clnDiskd1 allocation score on bl460g1n7: 0 +diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary +index 5c8231d..c2685d8 100644 +--- a/pengine/test10/bug-5186-partial-migrate.summary ++++ b/pengine/test10/bug-5186-partial-migrate.summary +@@ -24,9 +24,8 @@ Online: [ bl460g1n6 bl460g1n8 ] + prmPing (ocf::pacemaker:ping): Started bl460g1n7 (UNCLEAN) + Started: [ bl460g1n6 bl460g1n8 ] + +- notice: Fencing bl460g1n7: node is unclean + Transition Summary: +- * Fence (reboot) bl460g1n7 ++ * Fence (reboot) bl460g1n7 'prmDummy is thought to be active there' + * Move prmDummy (Started bl460g1n7 -> bl460g1n6) + * Move prmVM2 (Started bl460g1n7 -> bl460g1n8) + * Move prmStonith8-1 (Started bl460g1n7 -> bl460g1n6) +@@ -40,13 +39,13 @@ Executing cluster transition: + * Pseudo action: grpStonith8_stop_0 + * Pseudo action: prmStonith8-2_stop_0 + * Fencing bl460g1n7 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: prmDummy_stop_0 + * Pseudo action: prmVM2_stop_0 + * Pseudo action: prmStonith8-1_stop_0 + * Pseudo action: clnDiskd1_stop_0 + * Pseudo action: clnDiskd2_stop_0 + * Pseudo action: clnPing_stop_0 ++ * Pseudo action: stonith_complete + * Resource action: prmDummy start on bl460g1n6 + * Resource action: prmVM2 start on bl460g1n8 + * Pseudo action: grpStonith8_stopped_0 +diff --git a/pengine/test10/bug-cl-5212.exp b/pengine/test10/bug-cl-5212.exp +index 6d0a2f4..344711b 100644 +--- a/pengine/test10/bug-cl-5212.exp ++++ b/pengine/test10/bug-cl-5212.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +@@ -17,7 +17,7 @@ + + + +- ++ + + + +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -33,7 +33,7 @@ + + + +- ++ + + + +@@ -41,78 +41,78 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -120,43 +120,43 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +diff --git a/pengine/test10/bug-cl-5247.scores b/pengine/test10/bug-cl-5247.scores +index f1ded69..e9e4709 100644 +--- a/pengine/test10/bug-cl-5247.scores ++++ b/pengine/test10/bug-cl-5247.scores +@@ -1,4 +1,3 @@ +- notice: Fencing pgsr02: guest is unclean + Allocation scores: + Using the original execution date of: 2015-08-12 02:53:40Z + clone_color: msPostgresql allocation score on bl460g8n3: -INFINITY +diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary +index 755e647..fc97e8d 100644 +--- a/pengine/test10/bug-cl-5247.summary ++++ b/pengine/test10/bug-cl-5247.summary +@@ -17,9 +17,8 @@ Containers: [ pgsr01:prmDB1 ] + Masters: [ pgsr01 ] + Stopped: [ bl460g8n3 bl460g8n4 ] + +- notice: Fencing pgsr02: guest is unclean + Transition Summary: +- * Fence (off) pgsr02 (resource: prmDB2) ++ * Fence (off) pgsr02 (resource: prmDB2) 'guest is unclean' + * Stop prmDB2 (bl460g8n4) + * Restart prmStonith1-2 (Started bl460g8n4) + * Restart prmStonith2-2 (Started bl460g8n3) +diff --git a/pengine/test10/bug-lf-2508.exp b/pengine/test10/bug-lf-2508.exp +index bf70378..925afbc 100644 +--- a/pengine/test10/bug-lf-2508.exp ++++ b/pengine/test10/bug-lf-2508.exp +@@ -1,22 +1,22 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -24,317 +24,317 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -343,232 +343,231 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/bug-lf-2508.scores b/pengine/test10/bug-lf-2508.scores +index 200d548..3c3ce16 100644 +--- a/pengine/test10/bug-lf-2508.scores ++++ b/pengine/test10/bug-lf-2508.scores +@@ -1,4 +1,3 @@ +- notice: Fencing srv02: node is unclean + Allocation scores: + clone_color: clnStonith1 allocation score on srv01: -INFINITY + clone_color: clnStonith1 allocation score on srv02: 0 +diff --git a/pengine/test10/bug-lf-2508.summary b/pengine/test10/bug-lf-2508.summary +index 18080cb..ddaf1c3 100644 +--- a/pengine/test10/bug-lf-2508.summary ++++ b/pengine/test10/bug-lf-2508.summary +@@ -34,9 +34,8 @@ Online: [ srv01 srv03 srv04 ] + Started: [ srv01 srv03 ] + Stopped: [ srv04 ] + +- notice: Fencing srv02: node is unclean + Transition Summary: +- * Fence (reboot) srv02 ++ * Fence (reboot) srv02 'peer is no longer part of the cluster' + * Start Dummy01 (srv01) + * Move Dummy02 (Started srv02 -> srv04) + * Stop prmStonith1-1:1 (srv02) +@@ -51,15 +50,13 @@ Executing cluster transition: + * Pseudo action: Group01_start_0 + * Resource action: prmStonith3-1:1 monitor=3600000 on srv01 + * Fencing srv02 (reboot) +- * Pseudo action: stonith_complete +- * Resource action: Dummy01 start on srv01 + * Pseudo action: Group02_stop_0 + * Pseudo action: Dummy02_stop_0 + * Pseudo action: clnStonith1_stop_0 + * Pseudo action: clnStonith3_stop_0 + * Pseudo action: clnStonith4_stop_0 +- * Pseudo action: Group01_running_0 +- * Resource action: Dummy01 monitor=10000 on srv01 ++ * Pseudo action: stonith_complete ++ * Resource action: Dummy01 start on srv01 + * Pseudo action: Group02_stopped_0 + * Pseudo action: Group02_start_0 + * Resource action: Dummy02 start on srv04 +@@ -69,6 +66,8 @@ Executing cluster transition: + * Pseudo action: prmStonith3-3:1_stop_0 + * Pseudo action: grpStonith4:1_stop_0 + * Pseudo action: prmStonith4-3:1_stop_0 ++ * Pseudo action: Group01_running_0 ++ * Resource action: Dummy01 monitor=10000 on srv01 + * Pseudo action: Group02_running_0 + * Resource action: Dummy02 monitor=10000 on srv04 + * Pseudo action: prmStonith1-1:1_stop_0 +diff --git a/pengine/test10/bug-lf-2551.exp b/pengine/test10/bug-lf-2551.exp +index f5b8236..a14ec5e 100644 +--- a/pengine/test10/bug-lf-2551.exp ++++ b/pengine/test10/bug-lf-2551.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -22,7 +22,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -40,646 +40,645 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/bug-lf-2551.scores b/pengine/test10/bug-lf-2551.scores +index 64cbca0..2b0aa12 100644 +--- a/pengine/test10/bug-lf-2551.scores ++++ b/pengine/test10/bug-lf-2551.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hex-9: node is unclean + Allocation scores: + clone_color: base-clone allocation score on hex-0: 14 + clone_color: base-clone allocation score on hex-7: 16 +diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary +index 4fa553c..1b57ea7 100644 +--- a/pengine/test10/bug-lf-2551.summary ++++ b/pengine/test10/bug-lf-2551.summary +@@ -80,9 +80,8 @@ Online: [ hex-0 hex-7 hex-8 ] + vm-63 (ocf::heartbeat:Xen): Stopped + vm-64 (ocf::heartbeat:Xen): Stopped + +- notice: Fencing hex-9: node is unclean + Transition Summary: +- * Fence (reboot) hex-9 ++ * Fence (reboot) hex-9 'peer is no longer part of the cluster' + * Move fencing-sbd (Started hex-9 -> hex-0) + * Move dummy1 (Started hex-9 -> hex-0) + * Stop dlm:3 (hex-9) +@@ -112,11 +111,10 @@ Executing cluster transition: + * Pseudo action: fencing-sbd_stop_0 + * Resource action: dummy1 monitor=300000 on hex-8 + * Resource action: dummy1 monitor=300000 on hex-7 +- * Fencing hex-9 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: load_stopped_hex-8 + * Pseudo action: load_stopped_hex-7 + * Pseudo action: load_stopped_hex-0 ++ * Fencing hex-9 (reboot) + * Resource action: fencing-sbd start on hex-0 + * Pseudo action: dummy1_stop_0 + * Pseudo action: vm-03_stop_0 +@@ -135,6 +133,7 @@ Executing cluster transition: + * Pseudo action: vm-53_stop_0 + * Pseudo action: vm-57_stop_0 + * Pseudo action: vm-61_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: load_stopped_hex-9 + * Resource action: dummy1 start on hex-0 + * Pseudo action: base-clone_stop_0 +diff --git a/pengine/test10/bug-lf-2606.exp b/pengine/test10/bug-lf-2606.exp +index 55f138b..b14069e 100644 +--- a/pengine/test10/bug-lf-2606.exp ++++ b/pengine/test10/bug-lf-2606.exp +@@ -1,108 +1,108 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -110,77 +110,76 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/bug-lf-2606.scores b/pengine/test10/bug-lf-2606.scores +index b1e6a02..6a9d522 100644 +--- a/pengine/test10/bug-lf-2606.scores ++++ b/pengine/test10/bug-lf-2606.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node2: node is unclean + Allocation scores: + clone_color: ms3 allocation score on node1: 0 + clone_color: ms3 allocation score on node2: 0 +diff --git a/pengine/test10/bug-lf-2606.summary b/pengine/test10/bug-lf-2606.summary +index e6cfa89..7e889c9 100644 +--- a/pengine/test10/bug-lf-2606.summary ++++ b/pengine/test10/bug-lf-2606.summary +@@ -11,9 +11,8 @@ Online: [ node1 ] + Masters: [ node2 ] + Slaves: [ node1 ] + +- notice: Fencing node2: node is unclean + Transition Summary: +- * Fence (reboot) node2 ++ * Fence (reboot) node2 'rsc1 failed there' + * Stop rsc1 (node2) + * Move rsc2 (Started node2 -> node1) + * Demote rsc3:1 (Master -> Stopped node2) +@@ -21,12 +20,12 @@ Transition Summary: + Executing cluster transition: + * Pseudo action: ms3_demote_0 + * Fencing node2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 + * Pseudo action: rsc3:1_demote_0 + * Pseudo action: ms3_demoted_0 + * Pseudo action: ms3_stop_0 ++ * Pseudo action: stonith_complete + * Resource action: rsc2 start on node1 + * Pseudo action: rsc3:1_stop_0 + * Pseudo action: ms3_stopped_0 +diff --git a/pengine/test10/bug-rh-1097457.scores b/pengine/test10/bug-rh-1097457.scores +index 2fe2859..7729421 100644 +--- a/pengine/test10/bug-rh-1097457.scores ++++ b/pengine/test10/bug-rh-1097457.scores +@@ -1,4 +1,3 @@ +- notice: Fencing lamaVM2: guest is unclean + Allocation scores: + clone_color: FAKE6-clone allocation score on lama2: -INFINITY + clone_color: FAKE6-clone allocation score on lama3: -INFINITY +diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary +index 19aa39d..29dd018 100644 +--- a/pengine/test10/bug-rh-1097457.summary ++++ b/pengine/test10/bug-rh-1097457.summary +@@ -31,9 +31,8 @@ Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ] + Clone Set: FAKE6-clone [FAKE6] + Started: [ lamaVM1 lamaVM2 lamaVM3 ] + +- notice: Fencing lamaVM2: guest is unclean + Transition Summary: +- * Fence (reboot) lamaVM2 (resource: VM2) ++ * Fence (reboot) lamaVM2 (resource: VM2) 'guest is unclean' + * Recover VM2 (Started lama3) + * Recover FSlun3 (Started lamaVM2 -> lama2) + * Restart FAKE4 (Started lamaVM2) +diff --git a/pengine/test10/concurrent-fencing.exp b/pengine/test10/concurrent-fencing.exp +index 354111a..c916c1b 100644 +--- a/pengine/test10/concurrent-fencing.exp ++++ b/pengine/test10/concurrent-fencing.exp +@@ -1,39 +1,40 @@ + + + +- +- +- +- +- +- ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- +- +- +- +- +- ++ ++ ++ + + + +- +- +- +- ++ + + + + + +- +- ++ ++ + +- ++ + + + +@@ -41,32 +42,31 @@ + + + +- +- +- ++ ++ ++ ++ ++ ++ + + + +- +- +- +- ++ + + +- ++ + + + + + +- +- +- ++ ++ ++ ++ ++ ++ + +- +- +- +- +- ++ + + +diff --git a/pengine/test10/concurrent-fencing.scores b/pengine/test10/concurrent-fencing.scores +index 1be723e..0322231 100644 +--- a/pengine/test10/concurrent-fencing.scores ++++ b/pengine/test10/concurrent-fencing.scores +@@ -1,6 +1,3 @@ +- notice: Fencing node1: node is unclean +- notice: Fencing node2: node is unclean +- notice: Fencing node3: node is unclean + Allocation scores: + native_color: lsb_dummy allocation score on node1: 0 + native_color: lsb_dummy allocation score on node2: 0 +diff --git a/pengine/test10/concurrent-fencing.summary b/pengine/test10/concurrent-fencing.summary +index 3f54c2c..46d3323 100644 +--- a/pengine/test10/concurrent-fencing.summary ++++ b/pengine/test10/concurrent-fencing.summary +@@ -7,13 +7,10 @@ Node node3 (uuid3): UNCLEAN (offline) + stonith-1 (stonith:dummy): Stopped + lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped + +- notice: Fencing node1: node is unclean +- notice: Fencing node2: node is unclean +- notice: Fencing node3: node is unclean + Transition Summary: +- * Fence (reboot) node3 +- * Fence (reboot) node2 +- * Fence (reboot) node1 ++ * Fence (reboot) node3 'peer is no longer part of the cluster' ++ * Fence (reboot) node2 'peer is no longer part of the cluster' ++ * Fence (reboot) node1 'peer is no longer part of the cluster' + + Executing cluster transition: + * Fencing node3 (reboot) +diff --git a/pengine/test10/guest-node-host-dies.exp b/pengine/test10/guest-node-host-dies.exp +index 73a0264..235813d 100644 +--- a/pengine/test10/guest-node-host-dies.exp ++++ b/pengine/test10/guest-node-host-dies.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -10,296 +10,296 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -307,168 +307,168 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -477,7 +477,7 @@ + + + +- ++ + + + +@@ -486,7 +486,7 @@ + + + +- ++ + + + +@@ -495,69 +495,69 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -566,7 +566,7 @@ + + + +- ++ + + + +@@ -575,7 +575,7 @@ + + + +- ++ + + + +@@ -584,20 +584,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -606,13 +593,13 @@ + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -621,62 +608,75 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/guest-node-host-dies.scores b/pengine/test10/guest-node-host-dies.scores +index 663d236..0d7ad3f 100644 +--- a/pengine/test10/guest-node-host-dies.scores ++++ b/pengine/test10/guest-node-host-dies.scores +@@ -1,6 +1,3 @@ +- notice: Fencing lxc1: guest is unclean +- notice: Fencing lxc2: guest is unclean +- notice: Fencing rhel7-1: node is unclean + Allocation scores: + clone_color: lxc-ms-master allocation score on lxc1: INFINITY + clone_color: lxc-ms-master allocation score on lxc2: INFINITY +diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary +index 679deb6..d22fe3d 100644 +--- a/pengine/test10/guest-node-host-dies.summary ++++ b/pengine/test10/guest-node-host-dies.summary +@@ -10,13 +10,10 @@ Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + Master/Slave Set: lxc-ms-master [lxc-ms] + Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + +- notice: Fencing lxc1: guest is unclean +- notice: Fencing lxc2: guest is unclean +- notice: Fencing rhel7-1: node is unclean + Transition Summary: +- * Fence (reboot) rhel7-1 +- * Fence (reboot) lxc2 (resource: container2) +- * Fence (reboot) lxc1 (resource: container1) ++ * Fence (reboot) lxc2 (resource: container2) 'guest is unclean' ++ * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' ++ * Fence (reboot) rhel7-1 'rsc_rhel7-1 is thought to be active there' + * Restart Fencing (Started rhel7-4) + * Move rsc_rhel7-1 (Started rhel7-1 -> rhel7-5) + * Recover container1 (Started rhel7-1 -> rhel7-2) +diff --git a/pengine/test10/interleave-pseudo-stop.exp b/pengine/test10/interleave-pseudo-stop.exp +index 7a6b7ed..63217e8 100644 +--- a/pengine/test10/interleave-pseudo-stop.exp ++++ b/pengine/test10/interleave-pseudo-stop.exp +@@ -1,43 +1,43 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -50,7 +50,7 @@ + + + +- ++ + + + +@@ -63,7 +63,7 @@ + + + +- ++ + + + +@@ -93,40 +93,40 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -135,28 +135,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -165,7 +165,7 @@ + + + +- ++ + + + +@@ -173,37 +173,37 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +@@ -216,7 +216,7 @@ + + + +- ++ + + + +@@ -229,7 +229,7 @@ + + + +- ++ + + + +@@ -259,34 +259,34 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -295,28 +295,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -325,7 +325,7 @@ + + + +- ++ + + + +@@ -333,31 +333,31 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -370,7 +370,7 @@ + + + +- ++ + + + +@@ -383,7 +383,7 @@ + + + +- ++ + + + +@@ -413,34 +413,34 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -449,28 +449,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -479,7 +479,7 @@ + + + +- ++ + + + +@@ -487,87 +487,76 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +@@ -580,5 +569,15 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/interleave-pseudo-stop.scores b/pengine/test10/interleave-pseudo-stop.scores +index 625ba18..014f4de 100644 +--- a/pengine/test10/interleave-pseudo-stop.scores ++++ b/pengine/test10/interleave-pseudo-stop.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node1: node is unclean + Allocation scores: + clone_color: configstoreclone:0 allocation score on node1: 0 + clone_color: configstoreclone:0 allocation score on node2: 1 +diff --git a/pengine/test10/interleave-pseudo-stop.summary b/pengine/test10/interleave-pseudo-stop.summary +index 8c22b2f..01ce0ec 100644 +--- a/pengine/test10/interleave-pseudo-stop.summary ++++ b/pengine/test10/interleave-pseudo-stop.summary +@@ -16,9 +16,8 @@ Online: [ node2 ] + configstoreclone (ocf::heartbeat:Filesystem): Started node1 (UNCLEAN) + Started: [ node2 ] + +- notice: Fencing node1: node is unclean + Transition Summary: +- * Fence (reboot) node1 ++ * Fence (reboot) node1 'peer is no longer part of the cluster' + * Stop stonithclone:1 (node1) + * Stop evmsclone:1 (node1) + * Stop imagestoreclone:1 (node1) +@@ -29,7 +28,6 @@ Executing cluster transition: + * Pseudo action: imagestorecloneset_pre_notify_stop_0 + * Pseudo action: configstorecloneset_pre_notify_stop_0 + * Fencing node1 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: stonithcloneset_stop_0 + * Resource action: evmsclone:1 notify on node2 + * Pseudo action: evmsclone:0_post_notify_stop_0 +@@ -42,6 +40,7 @@ Executing cluster transition: + * Pseudo action: configstoreclone:0_post_notify_stop_0 + * Pseudo action: configstorecloneset_confirmed-pre_notify_stop_0 + * Pseudo action: configstorecloneset_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: stonithclone:0_stop_0 + * Pseudo action: stonithcloneset_stopped_0 + * Pseudo action: imagestoreclone:0_stop_0 +diff --git a/pengine/test10/master-7.exp b/pengine/test10/master-7.exp +index be05853..971949d 100644 +--- a/pengine/test10/master-7.exp ++++ b/pengine/test10/master-7.exp +@@ -1,69 +1,69 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -71,88 +71,88 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -161,13 +161,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -176,22 +176,22 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -200,93 +200,93 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -295,63 +295,63 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -360,7 +360,7 @@ + + + +- ++ + + + +@@ -369,7 +369,7 @@ + + + +- ++ + + + +@@ -378,7 +378,7 @@ + + + +- ++ + + + +@@ -387,91 +387,91 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -480,7 +480,7 @@ + + + +- ++ + + + +@@ -489,7 +489,7 @@ + + + +- ++ + + + +@@ -498,7 +498,7 @@ + + + +- ++ + + + +@@ -507,7 +507,7 @@ + + + +- ++ + + + +@@ -516,7 +516,7 @@ + + + +- ++ + + + +@@ -525,7 +525,7 @@ + + + +- ++ + + + +@@ -534,7 +534,7 @@ + + + +- ++ + + + +@@ -543,7 +543,7 @@ + + + +- ++ + + + +@@ -552,22 +552,22 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -575,98 +575,97 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/master-7.scores b/pengine/test10/master-7.scores +index 7569625..37fc6cd 100644 +--- a/pengine/test10/master-7.scores ++++ b/pengine/test10/master-7.scores +@@ -1,4 +1,3 @@ +- notice: Fencing c001n01: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n01: 0 + clone_color: DoFencing allocation score on c001n02: 0 +diff --git a/pengine/test10/master-7.summary b/pengine/test10/master-7.summary +index d76051e..4c58184 100644 +--- a/pengine/test10/master-7.summary ++++ b/pengine/test10/master-7.summary +@@ -28,9 +28,8 @@ Online: [ c001n02 c001n03 c001n08 ] + ocf_msdummy:6 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n02 + ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08 + +- notice: Fencing c001n01: node is unclean + Transition Summary: +- * Fence (reboot) c001n01 ++ * Fence (reboot) c001n01 'peer is no longer part of the cluster' + * Move DcIPaddr (Started c001n01 -> c001n03) + * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) + * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) +@@ -60,18 +59,17 @@ Executing cluster transition: + * Resource action: ocf_msdummy:7 monitor on c001n02 + * Pseudo action: master_rsc_1_demote_0 + * Fencing c001n01 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: DcIPaddr_stop_0 + * Resource action: heartbeat_192.168.100.182 stop on c001n03 +- * Resource action: lsb_dummy start on c001n08 + * Pseudo action: rsc_c001n01_stop_0 + * Pseudo action: DoFencing_stop_0 + * Pseudo action: ocf_msdummy:0_demote_0 + * Pseudo action: master_rsc_1_demoted_0 + * Pseudo action: master_rsc_1_stop_0 ++ * Pseudo action: stonith_complete + * Resource action: DcIPaddr start on c001n03 + * Resource action: ocf_192.168.100.181 stop on c001n03 +- * Resource action: lsb_dummy monitor=5000 on c001n08 ++ * Resource action: lsb_dummy start on c001n08 + * Resource action: rsc_c001n01 start on c001n03 + * Pseudo action: child_DoFencing:0_stop_0 + * Pseudo action: DoFencing_stopped_0 +@@ -85,6 +83,7 @@ Executing cluster transition: + * Resource action: ocf_192.168.100.181 start on c001n02 + * Resource action: heartbeat_192.168.100.182 start on c001n02 + * Resource action: ocf_192.168.100.183 start on c001n02 ++ * Resource action: lsb_dummy monitor=5000 on c001n08 + * Resource action: rsc_c001n01 monitor=5000 on c001n03 + * Pseudo action: group-1_running_0 + * Resource action: ocf_192.168.100.181 monitor=5000 on c001n02 +diff --git a/pengine/test10/master-8.exp b/pengine/test10/master-8.exp +index 6843922..e680354 100644 +--- a/pengine/test10/master-8.exp ++++ b/pengine/test10/master-8.exp +@@ -1,69 +1,69 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -71,88 +71,88 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -161,13 +161,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -176,22 +176,22 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -200,93 +200,93 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -295,63 +295,63 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -360,7 +360,7 @@ + + + +- ++ + + + +@@ -369,7 +369,7 @@ + + + +- ++ + + + +@@ -378,7 +378,7 @@ + + + +- ++ + + + +@@ -387,102 +387,102 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -491,7 +491,7 @@ + + + +- ++ + + + +@@ -500,7 +500,7 @@ + + + +- ++ + + + +@@ -509,7 +509,7 @@ + + + +- ++ + + + +@@ -518,7 +518,7 @@ + + + +- ++ + + + +@@ -527,7 +527,7 @@ + + + +- ++ + + + +@@ -536,7 +536,7 @@ + + + +- ++ + + + +@@ -545,7 +545,7 @@ + + + +- ++ + + + +@@ -554,7 +554,7 @@ + + + +- ++ + + + +@@ -563,7 +563,7 @@ + + + +- ++ + + + +@@ -572,22 +572,22 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -595,152 +595,151 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/master-8.scores b/pengine/test10/master-8.scores +index 880d7fb..c9d1cc9 100644 +--- a/pengine/test10/master-8.scores ++++ b/pengine/test10/master-8.scores +@@ -1,4 +1,3 @@ +- notice: Fencing c001n01: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n01: 0 + clone_color: DoFencing allocation score on c001n02: 0 +diff --git a/pengine/test10/master-8.summary b/pengine/test10/master-8.summary +index 7d0b840..e8d90d2 100644 +--- a/pengine/test10/master-8.summary ++++ b/pengine/test10/master-8.summary +@@ -28,9 +28,8 @@ Online: [ c001n02 c001n03 c001n08 ] + ocf_msdummy:6 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n02 + ocf_msdummy:7 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n08 + +- notice: Fencing c001n01: node is unclean + Transition Summary: +- * Fence (reboot) c001n01 ++ * Fence (reboot) c001n01 'peer is no longer part of the cluster' + * Move DcIPaddr (Started c001n01 -> c001n03) + * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) + * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) +@@ -61,18 +60,17 @@ Executing cluster transition: + * Resource action: ocf_msdummy:7 monitor on c001n02 + * Pseudo action: master_rsc_1_demote_0 + * Fencing c001n01 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: DcIPaddr_stop_0 + * Resource action: heartbeat_192.168.100.182 stop on c001n03 +- * Resource action: lsb_dummy start on c001n08 + * Pseudo action: rsc_c001n01_stop_0 + * Pseudo action: DoFencing_stop_0 + * Pseudo action: ocf_msdummy:0_demote_0 + * Pseudo action: master_rsc_1_demoted_0 + * Pseudo action: master_rsc_1_stop_0 ++ * Pseudo action: stonith_complete + * Resource action: DcIPaddr start on c001n03 + * Resource action: ocf_192.168.100.181 stop on c001n03 +- * Resource action: lsb_dummy monitor=5000 on c001n08 ++ * Resource action: lsb_dummy start on c001n08 + * Resource action: rsc_c001n01 start on c001n03 + * Pseudo action: child_DoFencing:0_stop_0 + * Pseudo action: DoFencing_stopped_0 +@@ -86,6 +84,7 @@ Executing cluster transition: + * Resource action: ocf_192.168.100.181 start on c001n02 + * Resource action: heartbeat_192.168.100.182 start on c001n02 + * Resource action: ocf_192.168.100.183 start on c001n02 ++ * Resource action: lsb_dummy monitor=5000 on c001n08 + * Resource action: rsc_c001n01 monitor=5000 on c001n03 + * Resource action: ocf_msdummy:0 start on c001n03 + * Pseudo action: master_rsc_1_running_0 +diff --git a/pengine/test10/migrate-fencing.exp b/pengine/test10/migrate-fencing.exp +index a670395..5ef64d7 100644 +--- a/pengine/test10/migrate-fencing.exp ++++ b/pengine/test10/migrate-fencing.exp +@@ -1,533 +1,533 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -536,7 +536,7 @@ + + + +- ++ + + + +@@ -545,149 +545,148 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/migrate-fencing.scores b/pengine/test10/migrate-fencing.scores +index 37d42b4..3febd69 100644 +--- a/pengine/test10/migrate-fencing.scores ++++ b/pengine/test10/migrate-fencing.scores +@@ -1,4 +1,3 @@ +- notice: Fencing pcmk-4: node is unclean + Allocation scores: + clone_color: Connectivity allocation score on pcmk-1: 0 + clone_color: Connectivity allocation score on pcmk-2: 0 +diff --git a/pengine/test10/migrate-fencing.summary b/pengine/test10/migrate-fencing.summary +index fdb905e..7468fe2 100644 +--- a/pengine/test10/migrate-fencing.summary ++++ b/pengine/test10/migrate-fencing.summary +@@ -21,9 +21,8 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Masters: [ pcmk-4 ] + Slaves: [ pcmk-1 pcmk-2 pcmk-3 ] + +- notice: Fencing pcmk-4: node is unclean + Transition Summary: +- * Fence (reboot) pcmk-4 ++ * Fence (reboot) pcmk-4 'termination was requested' + * Stop FencingChild:0 (pcmk-4) + * Move r192.168.101.181 (Started pcmk-4 -> pcmk-1) + * Move r192.168.101.182 (Started pcmk-4 -> pcmk-1) +@@ -39,27 +38,27 @@ Executing cluster transition: + * Resource action: stateful-1:3 monitor=15000 on pcmk-3 + * Resource action: stateful-1:2 monitor=15000 on pcmk-2 + * Fencing pcmk-4 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: Fencing_stop_0 + * Pseudo action: rsc_pcmk-4_stop_0 + * Pseudo action: lsb-dummy_stop_0 +- * Resource action: migrator migrate_to on pcmk-1 + * Pseudo action: Connectivity_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: FencingChild:0_stop_0 + * Pseudo action: Fencing_stopped_0 + * Pseudo action: group-1_stop_0 + * Pseudo action: r192.168.101.183_stop_0 + * Resource action: rsc_pcmk-4 start on pcmk-2 +- * Resource action: migrator migrate_from on pcmk-3 +- * Resource action: migrator stop on pcmk-1 ++ * Resource action: migrator migrate_to on pcmk-1 + * Pseudo action: ping-1:0_stop_0 + * Pseudo action: Connectivity_stopped_0 + * Pseudo action: r192.168.101.182_stop_0 + * Resource action: rsc_pcmk-4 monitor=5000 on pcmk-2 +- * Pseudo action: migrator_start_0 ++ * Resource action: migrator migrate_from on pcmk-3 ++ * Resource action: migrator stop on pcmk-1 + * Pseudo action: r192.168.101.181_stop_0 +- * Resource action: migrator monitor=10000 on pcmk-3 ++ * Pseudo action: migrator_start_0 + * Pseudo action: group-1_stopped_0 ++ * Resource action: migrator monitor=10000 on pcmk-3 + * Pseudo action: master-1_demote_0 + * Pseudo action: stateful-1:0_demote_0 + * Pseudo action: master-1_demoted_0 +diff --git a/pengine/test10/rec-node-10.exp b/pengine/test10/rec-node-10.exp +index 309764a..a0606d9 100644 +--- a/pengine/test10/rec-node-10.exp ++++ b/pengine/test10/rec-node-10.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-11.exp b/pengine/test10/rec-node-11.exp +index 473c1a7..6c7efcc 100644 +--- a/pengine/test10/rec-node-11.exp ++++ b/pengine/test10/rec-node-11.exp +@@ -1,23 +1,23 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -26,113 +26,113 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -141,50 +141,50 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -193,76 +193,75 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/rec-node-11.scores b/pengine/test10/rec-node-11.scores +index 86c2bf5..482deeb 100644 +--- a/pengine/test10/rec-node-11.scores ++++ b/pengine/test10/rec-node-11.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node1: node is unclean + Allocation scores: + group_color: group1 allocation score on node1: 0 + group_color: group1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-11.summary b/pengine/test10/rec-node-11.summary +index f45f0a8..e884ffa 100644 +--- a/pengine/test10/rec-node-11.summary ++++ b/pengine/test10/rec-node-11.summary +@@ -9,9 +9,8 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 + rsc3 (heartbeat:apache): Started node2 + +- notice: Fencing node1: node is unclean + Transition Summary: +- * Fence (reboot) node1 ++ * Fence (reboot) node1 'peer process is no longer available' + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +@@ -22,9 +21,9 @@ Executing cluster transition: + * Resource action: rsc1 monitor on node2 + * Resource action: rsc2 monitor on node2 + * Fencing node1 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: group1_stop_0 + * Pseudo action: rsc2_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: group1_stopped_0 + * Resource action: rsc3 stop on node2 +diff --git a/pengine/test10/rec-node-12.scores b/pengine/test10/rec-node-12.scores +index 4fd9ff5..153c232 100644 +--- a/pengine/test10/rec-node-12.scores ++++ b/pengine/test10/rec-node-12.scores +@@ -1,4 +1,3 @@ +- notice: Fencing c001n02: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n01: 0 + clone_color: DoFencing allocation score on c001n02: 0 +diff --git a/pengine/test10/rec-node-12.summary b/pengine/test10/rec-node-12.summary +index 562cae9..8b2d4e6 100644 +--- a/pengine/test10/rec-node-12.summary ++++ b/pengine/test10/rec-node-12.summary +@@ -14,9 +14,8 @@ Online: [ c001n01 c001n03 c001n08 ] + child_DoFencing:2 (stonith:ssh): Stopped + child_DoFencing:3 (stonith:ssh): Stopped + +- notice: Fencing c001n02: node is unclean + Transition Summary: +- * Fence (reboot) c001n02 ++ * Fence (reboot) c001n02 'node is unclean' + * Start DcIPaddr (c001n08) + * Start rsc_c001n08 (c001n08) + * Start rsc_c001n02 (c001n01) +diff --git a/pengine/test10/rec-node-13.exp b/pengine/test10/rec-node-13.exp +index 3b25e9f..a289600 100644 +--- a/pengine/test10/rec-node-13.exp ++++ b/pengine/test10/rec-node-13.exp +@@ -1,83 +1,82 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/rec-node-13.scores b/pengine/test10/rec-node-13.scores +index bdb5004..dda1134 100644 +--- a/pengine/test10/rec-node-13.scores ++++ b/pengine/test10/rec-node-13.scores +@@ -1,4 +1,3 @@ +- notice: Fencing c001n04: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n02: 0 + clone_color: DoFencing allocation score on c001n03: 0 +diff --git a/pengine/test10/rec-node-13.summary b/pengine/test10/rec-node-13.summary +index d3b4d65..9873757 100644 +--- a/pengine/test10/rec-node-13.summary ++++ b/pengine/test10/rec-node-13.summary +@@ -33,15 +33,14 @@ OFFLINE: [ c001n03 c001n05 ] + ocf_msdummy:10 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n06 + ocf_msdummy:11 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n07 + +- notice: Fencing c001n04: node is unclean + Transition Summary: +- * Fence (reboot) c001n04 ++ * Fence (reboot) c001n04 'ocf_msdummy:6 failed there' + * Stop ocf_msdummy:6 (c001n04) + + Executing cluster transition: + * Fencing c001n04 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: master_rsc_1_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: ocf_msdummy:6_stop_0 + * Pseudo action: master_rsc_1_stopped_0 + * Pseudo action: all_stopped +diff --git a/pengine/test10/rec-node-14.exp b/pengine/test10/rec-node-14.exp +index 30dbcdb..308ab7e 100644 +--- a/pengine/test10/rec-node-14.exp ++++ b/pengine/test10/rec-node-14.exp +@@ -1,68 +1,67 @@ + + + +- +- +- +- +- +- ++ ++ ++ + + + +- ++ + + + + + +- +- +- +- +- +- ++ ++ ++ + + + +- ++ + + + + + +- +- ++ ++ + +- ++ + + + +- ++ ++ ++ ++ ++ + + + +- +- +- ++ ++ ++ ++ ++ ++ + + + +- ++ + + + + + +- +- +- ++ ++ ++ ++ ++ ++ + +- +- +- +- +- ++ + + +- +diff --git a/pengine/test10/rec-node-14.scores b/pengine/test10/rec-node-14.scores +index 1be723e..0322231 100644 +--- a/pengine/test10/rec-node-14.scores ++++ b/pengine/test10/rec-node-14.scores +@@ -1,6 +1,3 @@ +- notice: Fencing node1: node is unclean +- notice: Fencing node2: node is unclean +- notice: Fencing node3: node is unclean + Allocation scores: + native_color: lsb_dummy allocation score on node1: 0 + native_color: lsb_dummy allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-14.summary b/pengine/test10/rec-node-14.summary +index a707aee..b5f105e 100644 +--- a/pengine/test10/rec-node-14.summary ++++ b/pengine/test10/rec-node-14.summary +@@ -7,13 +7,10 @@ Node node3 (uuid3): UNCLEAN (offline) + stonith-1 (stonith:dummy): Stopped + lsb_dummy (lsb:/usr/lib/heartbeat/cts/LSBDummy): Stopped + +- notice: Fencing node1: node is unclean +- notice: Fencing node2: node is unclean +- notice: Fencing node3: node is unclean + Transition Summary: +- * Fence (reboot) node3 +- * Fence (reboot) node2 +- * Fence (reboot) node1 ++ * Fence (reboot) node3 'peer is no longer part of the cluster' ++ * Fence (reboot) node2 'peer is no longer part of the cluster' ++ * Fence (reboot) node1 'peer is no longer part of the cluster' + + Executing cluster transition: + * Fencing node1 (reboot) +diff --git a/pengine/test10/rec-node-15.exp b/pengine/test10/rec-node-15.exp +index 3810636..3cd3e81 100644 +--- a/pengine/test10/rec-node-15.exp ++++ b/pengine/test10/rec-node-15.exp +@@ -1,26 +1,26 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -38,28 +38,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -67,214 +67,214 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -282,204 +282,203 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/rec-node-15.scores b/pengine/test10/rec-node-15.scores +index d8b9617..03dc6e6 100644 +--- a/pengine/test10/rec-node-15.scores ++++ b/pengine/test10/rec-node-15.scores +@@ -1,4 +1,3 @@ +- notice: Fencing sapcl03: node is unclean + Allocation scores: + group_color: Filesystem_13 allocation score on sapcl01: 0 + group_color: Filesystem_13 allocation score on sapcl02: 0 +diff --git a/pengine/test10/rec-node-15.summary b/pengine/test10/rec-node-15.summary +index 863bfd4..760942c 100644 +--- a/pengine/test10/rec-node-15.summary ++++ b/pengine/test10/rec-node-15.summary +@@ -20,9 +20,8 @@ Online: [ sapcl01 ] + oracle_24 (ocf::heartbeat:oracle): Stopped + oralsnr_25 (ocf::heartbeat:oralsnr): Stopped + +- notice: Fencing sapcl03: node is unclean + Transition Summary: +- * Fence (reboot) sapcl03 ++ * Fence (reboot) sapcl03 'peer is no longer part of the cluster' + * Start stonith-1 (sapcl01) + * Move IPaddr_192_168_1_102 (Started sapcl02 -> sapcl01) + * Move LVM_12 (Started sapcl02 -> sapcl01) +@@ -40,20 +39,14 @@ Executing cluster transition: + * Resource action: Filesystem_13 stop on sapcl02 + * Pseudo action: oracle_start_0 + * Fencing sapcl03 (reboot) +- * Pseudo action: stonith_complete + * Resource action: LVM_12 stop on sapcl02 ++ * Pseudo action: stonith_complete ++ * Resource action: IPaddr_192_168_1_102 stop on sapcl02 + * Resource action: IPaddr_192_168_1_104 start on sapcl01 + * Resource action: LVM_22 start on sapcl01 + * Resource action: Filesystem_23 start on sapcl01 + * Resource action: oracle_24 start on sapcl01 + * Resource action: oralsnr_25 start on sapcl01 +- * Resource action: IPaddr_192_168_1_102 stop on sapcl02 +- * Pseudo action: oracle_running_0 +- * Resource action: IPaddr_192_168_1_104 monitor=5000 on sapcl01 +- * Resource action: LVM_22 monitor=120000 on sapcl01 +- * Resource action: Filesystem_23 monitor=120000 on sapcl01 +- * Resource action: oracle_24 monitor=120000 on sapcl01 +- * Resource action: oralsnr_25 monitor=120000 on sapcl01 + * Pseudo action: all_stopped + * Resource action: stonith-1 start on sapcl01 + * Pseudo action: app02_stopped_0 +@@ -61,6 +54,12 @@ Executing cluster transition: + * Resource action: IPaddr_192_168_1_102 start on sapcl01 + * Resource action: LVM_12 start on sapcl01 + * Resource action: Filesystem_13 start on sapcl01 ++ * Pseudo action: oracle_running_0 ++ * Resource action: IPaddr_192_168_1_104 monitor=5000 on sapcl01 ++ * Resource action: LVM_22 monitor=120000 on sapcl01 ++ * Resource action: Filesystem_23 monitor=120000 on sapcl01 ++ * Resource action: oracle_24 monitor=120000 on sapcl01 ++ * Resource action: oralsnr_25 monitor=120000 on sapcl01 + * Pseudo action: app02_running_0 + * Resource action: IPaddr_192_168_1_102 monitor=5000 on sapcl01 + * Resource action: LVM_12 monitor=120000 on sapcl01 +diff --git a/pengine/test10/rec-node-2.scores b/pengine/test10/rec-node-2.scores +index 6efa92d..68e176f 100644 +--- a/pengine/test10/rec-node-2.scores ++++ b/pengine/test10/rec-node-2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node1: node is unclean + Allocation scores: + group_color: group1 allocation score on node1: 0 + group_color: group1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-2.summary b/pengine/test10/rec-node-2.summary +index ecf6562..4106abf 100644 +--- a/pengine/test10/rec-node-2.summary ++++ b/pengine/test10/rec-node-2.summary +@@ -13,9 +13,8 @@ Online: [ node2 ] + rsc5 (heartbeat:apache): Stopped + rsc6 (heartbeat:apache): Stopped + +- notice: Fencing node1: node is unclean + Transition Summary: +- * Fence (reboot) node1 ++ * Fence (reboot) node1 'node is unclean' + * Start stonith-1 (node2) + * Start rsc1 (node2) + * Start rsc2 (node2) +diff --git a/pengine/test10/rec-node-4.exp b/pengine/test10/rec-node-4.exp +index 00f975e..2aaa34b 100644 +--- a/pengine/test10/rec-node-4.exp ++++ b/pengine/test10/rec-node-4.exp +@@ -1,23 +1,23 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -26,41 +26,41 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -69,41 +69,41 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -112,44 +112,43 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/rec-node-4.scores b/pengine/test10/rec-node-4.scores +index e25a30f..1d008ec 100644 +--- a/pengine/test10/rec-node-4.scores ++++ b/pengine/test10/rec-node-4.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node1: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-4.summary b/pengine/test10/rec-node-4.summary +index f46a3bb..4a39615 100644 +--- a/pengine/test10/rec-node-4.summary ++++ b/pengine/test10/rec-node-4.summary +@@ -7,9 +7,8 @@ Online: [ node2 ] + rsc1 (heartbeat:apache): Started node1 (UNCLEAN) + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + +- notice: Fencing node1: node is unclean + Transition Summary: +- * Fence (reboot) node1 ++ * Fence (reboot) node1 'peer is no longer part of the cluster' + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +@@ -19,9 +18,9 @@ Executing cluster transition: + * Resource action: rsc1 monitor on node2 + * Resource action: rsc2 monitor on node2 + * Fencing node1 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + * Resource action: stonith-1 start on node2 + * Resource action: rsc1 start on node2 +diff --git a/pengine/test10/rec-node-5.exp b/pengine/test10/rec-node-5.exp +index abae4db..7099236 100644 +--- a/pengine/test10/rec-node-5.exp ++++ b/pengine/test10/rec-node-5.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -23,20 +23,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-6.exp b/pengine/test10/rec-node-6.exp +index 00f975e..2aaa34b 100644 +--- a/pengine/test10/rec-node-6.exp ++++ b/pengine/test10/rec-node-6.exp +@@ -1,23 +1,23 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -26,41 +26,41 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -69,41 +69,41 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -112,44 +112,43 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/rec-node-6.scores b/pengine/test10/rec-node-6.scores +index e25a30f..1d008ec 100644 +--- a/pengine/test10/rec-node-6.scores ++++ b/pengine/test10/rec-node-6.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node1: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-6.summary b/pengine/test10/rec-node-6.summary +index 37257fe..6cb8d01 100644 +--- a/pengine/test10/rec-node-6.summary ++++ b/pengine/test10/rec-node-6.summary +@@ -7,9 +7,8 @@ Online: [ node2 ] + rsc1 (heartbeat:apache): Started node1 + rsc2 (heartbeat:apache): Started node1 + +- notice: Fencing node1: node is unclean + Transition Summary: +- * Fence (reboot) node1 ++ * Fence (reboot) node1 'peer process is no longer available' + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +@@ -19,9 +18,9 @@ Executing cluster transition: + * Resource action: rsc1 monitor on node2 + * Resource action: rsc2 monitor on node2 + * Fencing node1 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + * Resource action: stonith-1 start on node2 + * Resource action: rsc1 start on node2 +diff --git a/pengine/test10/rec-node-7.exp b/pengine/test10/rec-node-7.exp +index 00f975e..2aaa34b 100644 +--- a/pengine/test10/rec-node-7.exp ++++ b/pengine/test10/rec-node-7.exp +@@ -1,23 +1,23 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -26,41 +26,41 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -69,41 +69,41 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -112,44 +112,43 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/rec-node-7.scores b/pengine/test10/rec-node-7.scores +index e25a30f..1d008ec 100644 +--- a/pengine/test10/rec-node-7.scores ++++ b/pengine/test10/rec-node-7.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node1: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: 0 +diff --git a/pengine/test10/rec-node-7.summary b/pengine/test10/rec-node-7.summary +index f46a3bb..4a39615 100644 +--- a/pengine/test10/rec-node-7.summary ++++ b/pengine/test10/rec-node-7.summary +@@ -7,9 +7,8 @@ Online: [ node2 ] + rsc1 (heartbeat:apache): Started node1 (UNCLEAN) + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + +- notice: Fencing node1: node is unclean + Transition Summary: +- * Fence (reboot) node1 ++ * Fence (reboot) node1 'peer is no longer part of the cluster' + * Start stonith-1 (node2) + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +@@ -19,9 +18,9 @@ Executing cluster transition: + * Resource action: rsc1 monitor on node2 + * Resource action: rsc2 monitor on node2 + * Fencing node1 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + * Resource action: stonith-1 start on node2 + * Resource action: rsc1 start on node2 +diff --git a/pengine/test10/rec-node-8.exp b/pengine/test10/rec-node-8.exp +index 78a972c..334e62a 100644 +--- a/pengine/test10/rec-node-8.exp ++++ b/pengine/test10/rec-node-8.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-5.exp b/pengine/test10/rec-rsc-5.exp +index a690d68..b1f151b 100644 +--- a/pengine/test10/rec-rsc-5.exp ++++ b/pengine/test10/rec-rsc-5.exp +@@ -1,23 +1,23 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -26,41 +26,41 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -69,41 +69,41 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -112,44 +112,43 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/rec-rsc-5.scores b/pengine/test10/rec-rsc-5.scores +index ebaedbd..1c640ae 100644 +--- a/pengine/test10/rec-rsc-5.scores ++++ b/pengine/test10/rec-rsc-5.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node2: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: 0 + native_color: rsc1 allocation score on node2: -INFINITY +diff --git a/pengine/test10/rec-rsc-5.summary b/pengine/test10/rec-rsc-5.summary +index 6bc4010..28e29b5 100644 +--- a/pengine/test10/rec-rsc-5.summary ++++ b/pengine/test10/rec-rsc-5.summary +@@ -7,9 +7,8 @@ Online: [ node1 ] + rsc1 (heartbeat:apache): FAILED node2 + rsc2 (heartbeat:apache): Started node2 + +- notice: Fencing node2: node is unclean + Transition Summary: +- * Fence (reboot) node2 ++ * Fence (reboot) node2 'rsc1 failed there' + * Start stonith-1 (node1) + * Recover rsc1 (Started node2 -> node1) + * Move rsc2 (Started node2 -> node1) +@@ -19,9 +18,9 @@ Executing cluster transition: + * Resource action: rsc1 monitor on node1 + * Resource action: rsc2 monitor on node1 + * Fencing node2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + * Resource action: stonith-1 start on node1 + * Resource action: rsc1 start on node1 +diff --git a/pengine/test10/remote-fence-before-reconnect.exp b/pengine/test10/remote-fence-before-reconnect.exp +index 54c9106..736f64f 100644 +--- a/pengine/test10/remote-fence-before-reconnect.exp ++++ b/pengine/test10/remote-fence-before-reconnect.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -11,90 +11,90 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-fence-before-reconnect.scores b/pengine/test10/remote-fence-before-reconnect.scores +index 9423693..fb46919 100644 +--- a/pengine/test10/remote-fence-before-reconnect.scores ++++ b/pengine/test10/remote-fence-before-reconnect.scores +@@ -1,4 +1,3 @@ +- notice: Fencing c7auto4: node is unclean + Allocation scores: + native_color: c7auto4 allocation score on c7auto1: -INFINITY + native_color: c7auto4 allocation score on c7auto2: -INFINITY +diff --git a/pengine/test10/remote-fence-before-reconnect.summary b/pengine/test10/remote-fence-before-reconnect.summary +index f477884..7a22551 100644 +--- a/pengine/test10/remote-fence-before-reconnect.summary ++++ b/pengine/test10/remote-fence-before-reconnect.summary +@@ -11,16 +11,15 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + fake4 (ocf::heartbeat:Dummy): Started c7auto2 + fake5 (ocf::heartbeat:Dummy): Started c7auto3 + +- notice: Fencing c7auto4: node is unclean + Transition Summary: +- * Fence (reboot) c7auto4 ++ * Fence (reboot) c7auto4 'remote connection is unrecoverable' + * Stop c7auto4 (c7auto1) + * Move fake2 (Started c7auto4 -> c7auto1) + + Executing cluster transition: + * Fencing c7auto4 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: fake2_stop_0 ++ * Pseudo action: stonith_complete + * Resource action: c7auto4 stop on c7auto1 + * Resource action: fake2 start on c7auto1 + * Pseudo action: all_stopped +diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp +index 2e341bd..5bf7eb3 100644 +--- a/pengine/test10/remote-fence-unclean-3.exp ++++ b/pengine/test10/remote-fence-unclean-3.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -75,27 +75,27 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -107,7 +107,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/remote-fence-unclean-3.scores b/pengine/test10/remote-fence-unclean-3.scores +index 283a599..77d71d6 100644 +--- a/pengine/test10/remote-fence-unclean-3.scores ++++ b/pengine/test10/remote-fence-unclean-3.scores +@@ -1,4 +1,3 @@ +- notice: Fencing overcloud-novacompute-0: node is unclean + Allocation scores: + clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 + clone_color: galera-bundle-master allocation score on galera-bundle-1: 0 +diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary +index ec5d304..6d15598 100644 +--- a/pengine/test10/remote-fence-unclean-3.summary ++++ b/pengine/test10/remote-fence-unclean-3.summary +@@ -33,9 +33,8 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund + Docker container: openstack-cinder-backup [192.168.24.1:8787/tripleoupstream/centos-binary-cinder-backup:latest] + openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1 + +- notice: Fencing overcloud-novacompute-0: node is unclean + Transition Summary: +- * Fence (reboot) overcloud-novacompute-0 ++ * Fence (reboot) overcloud-novacompute-0 'the connection is unrecoverable' + * Start fence1 (overcloud-controller-0) + * Stop overcloud-novacompute-0 (overcloud-controller-0) + +diff --git a/pengine/test10/remote-fence-unclean.exp b/pengine/test10/remote-fence-unclean.exp +index d2c3617..3a07384 100644 +--- a/pengine/test10/remote-fence-unclean.exp ++++ b/pengine/test10/remote-fence-unclean.exp +@@ -1,23 +1,23 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +@@ -40,83 +40,83 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -125,36 +125,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -163,49 +163,49 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-fence-unclean.scores b/pengine/test10/remote-fence-unclean.scores +index 038ca2e..8d29662 100644 +--- a/pengine/test10/remote-fence-unclean.scores ++++ b/pengine/test10/remote-fence-unclean.scores +@@ -1,4 +1,3 @@ +- notice: Fencing remote1: node is unclean + Allocation scores: + native_color: FAKE1 allocation score on 18builder: 0 + native_color: FAKE1 allocation score on 18node1: 0 +diff --git a/pengine/test10/remote-fence-unclean.summary b/pengine/test10/remote-fence-unclean.summary +index 8f37909..9830b96 100644 +--- a/pengine/test10/remote-fence-unclean.summary ++++ b/pengine/test10/remote-fence-unclean.summary +@@ -10,9 +10,8 @@ Online: [ 18builder 18node1 18node2 ] + FAKE3 (ocf::heartbeat:Dummy): Started 18builder + FAKE4 (ocf::heartbeat:Dummy): Started 18node1 + +- notice: Fencing remote1: node is unclean + Transition Summary: +- * Fence (reboot) remote1 ++ * Fence (reboot) remote1 'remote connection is unrecoverable' + * Recover remote1 (Started 18node1) + * Move FAKE2 (Started remote1 -> 18builder) + * Move FAKE3 (Started 18builder -> 18node1) +@@ -22,18 +21,18 @@ Executing cluster transition: + * Resource action: FAKE3 stop on 18builder + * Resource action: FAKE4 stop on 18node1 + * Fencing remote1 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: FAKE2_stop_0 +- * Resource action: FAKE3 start on 18node1 +- * Resource action: FAKE4 start on 18node2 ++ * Pseudo action: stonith_complete + * Resource action: remote1 stop on 18node1 + * Resource action: FAKE2 start on 18builder +- * Resource action: FAKE3 monitor=60000 on 18node1 +- * Resource action: FAKE4 monitor=60000 on 18node2 ++ * Resource action: FAKE3 start on 18node1 ++ * Resource action: FAKE4 start on 18node2 + * Pseudo action: all_stopped + * Resource action: remote1 start on 18node1 + * Resource action: remote1 monitor=60000 on 18node1 + * Resource action: FAKE2 monitor=60000 on 18builder ++ * Resource action: FAKE3 monitor=60000 on 18node1 ++ * Resource action: FAKE4 monitor=60000 on 18node2 + + Revised cluster status: + Online: [ 18builder 18node1 18node2 ] +diff --git a/pengine/test10/remote-fence-unclean2.exp b/pengine/test10/remote-fence-unclean2.exp +index c28832b..c9c45ae 100644 +--- a/pengine/test10/remote-fence-unclean2.exp ++++ b/pengine/test10/remote-fence-unclean2.exp +@@ -1,52 +1,52 @@ + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-fence-unclean2.scores b/pengine/test10/remote-fence-unclean2.scores +index 937d0eb..10fc7fd 100644 +--- a/pengine/test10/remote-fence-unclean2.scores ++++ b/pengine/test10/remote-fence-unclean2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing rhel7-alt4: node is unclean + Allocation scores: + native_color: fake allocation score on rhel7-alt1: 0 + native_color: fake allocation score on rhel7-alt2: 0 +diff --git a/pengine/test10/remote-fence-unclean2.summary b/pengine/test10/remote-fence-unclean2.summary +index 060d55e..bf7f907 100644 +--- a/pengine/test10/remote-fence-unclean2.summary ++++ b/pengine/test10/remote-fence-unclean2.summary +@@ -9,15 +9,14 @@ OFFLINE: [ rhel7-alt3 ] + rhel7-alt4 (ocf::pacemaker:remote): Stopped + fake (ocf::heartbeat:Dummy): Started rhel7-alt4 (UNCLEAN) + +- notice: Fencing rhel7-alt4: node is unclean + Transition Summary: +- * Fence (reboot) rhel7-alt4 ++ * Fence (reboot) rhel7-alt4 'fake is active there (fencing will be revoked if remote connection can be re-established elsewhere)' + * Stop fake (rhel7-alt4) + + Executing cluster transition: + * Fencing rhel7-alt4 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: fake_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + + Revised cluster status: +diff --git a/pengine/test10/remote-partial-migrate2.exp b/pengine/test10/remote-partial-migrate2.exp +index 54d1ded..abf281f 100644 +--- a/pengine/test10/remote-partial-migrate2.exp ++++ b/pengine/test10/remote-partial-migrate2.exp +@@ -1,195 +1,195 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -198,42 +198,42 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -242,36 +242,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -280,36 +280,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -318,42 +318,42 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -362,42 +362,42 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -406,36 +406,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -444,36 +444,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -482,42 +482,42 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -526,36 +526,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -564,42 +564,42 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -608,90 +608,90 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -700,36 +700,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -738,42 +738,42 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -782,88 +782,88 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-partial-migrate2.scores b/pengine/test10/remote-partial-migrate2.scores +index ce095b1..6965507 100644 +--- a/pengine/test10/remote-partial-migrate2.scores ++++ b/pengine/test10/remote-partial-migrate2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing pcmk4: node is unclean + Allocation scores: + native_color: FAKE1 allocation score on pcmk1: -INFINITY + native_color: FAKE1 allocation score on pcmk2: 0 +diff --git a/pengine/test10/remote-partial-migrate2.summary b/pengine/test10/remote-partial-migrate2.summary +index c9f9592..17fee27 100644 +--- a/pengine/test10/remote-partial-migrate2.summary ++++ b/pengine/test10/remote-partial-migrate2.summary +@@ -62,9 +62,8 @@ RemoteOFFLINE: [ pcmk_remote4 ] + FAKE49 (ocf::heartbeat:Dummy): Started pcmk_remote3 + FAKE50 (ocf::heartbeat:Dummy): Started pcmk_remote5 + +- notice: Fencing pcmk4: node is unclean + Transition Summary: +- * Fence (reboot) pcmk4 ++ * Fence (reboot) pcmk4 'peer is no longer part of the cluster' + * Migrate pcmk_remote2 (Started pcmk3 -> pcmk1) + * Start pcmk_remote4 (pcmk2) + * Migrate pcmk_remote5 (Started pcmk1 -> pcmk2) +diff --git a/pengine/test10/remote-recover-all.exp b/pengine/test10/remote-recover-all.exp +index 6c1164c..b0af5c4 100644 +--- a/pengine/test10/remote-recover-all.exp ++++ b/pengine/test10/remote-recover-all.exp +@@ -1,72 +1,72 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -122,116 +122,116 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -239,76 +239,76 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -321,23 +321,23 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + +@@ -350,68 +350,68 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -423,7 +423,7 @@ + + + +- ++ + + + +@@ -431,211 +431,211 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -644,36 +644,36 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -682,65 +682,65 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -748,7 +748,7 @@ + + + +- ++ + + + +@@ -757,13 +757,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -772,91 +772,91 @@ + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-recover-all.scores b/pengine/test10/remote-recover-all.scores +index 947f228..43cac62 100644 +--- a/pengine/test10/remote-recover-all.scores ++++ b/pengine/test10/remote-recover-all.scores +@@ -1,6 +1,3 @@ +- notice: Fencing controller-1: node is unclean +- notice: Fencing galera-2: node is unclean +- notice: Fencing messaging-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary +index 6868533..35f3d58 100644 +--- a/pengine/test10/remote-recover-all.summary ++++ b/pengine/test10/remote-recover-all.summary +@@ -37,13 +37,10 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + +- notice: Fencing controller-1: node is unclean +- notice: Fencing galera-2: node is unclean +- notice: Fencing messaging-1: node is unclean + Transition Summary: +- * Fence (reboot) messaging-1 +- * Fence (reboot) galera-2 +- * Fence (reboot) controller-1 ++ * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' ++ * Fence (reboot) galera-2 'resources are active and the connection is unrecoverable' ++ * Fence (reboot) controller-1 'peer is no longer part of the cluster' + * Stop messaging-1 (controller-1) + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +diff --git a/pengine/test10/remote-recover-connection.exp b/pengine/test10/remote-recover-connection.exp +index 62860f0..cf74efb 100644 +--- a/pengine/test10/remote-recover-connection.exp ++++ b/pengine/test10/remote-recover-connection.exp +@@ -1,163 +1,163 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -187,22 +187,22 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -215,7 +215,7 @@ + + + +- ++ + + + +@@ -228,7 +228,7 @@ + + + +- ++ + + + +@@ -244,7 +244,7 @@ + + + +- ++ + + + +@@ -257,7 +257,7 @@ + + + +- ++ + + + +@@ -266,13 +266,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -284,28 +284,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -317,7 +317,7 @@ + + + +- ++ + + + +@@ -325,211 +325,211 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -538,33 +538,33 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -573,59 +573,59 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -633,76 +633,76 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-recover-connection.scores b/pengine/test10/remote-recover-connection.scores +index 3d3beae..56a18eb 100644 +--- a/pengine/test10/remote-recover-connection.scores ++++ b/pengine/test10/remote-recover-connection.scores +@@ -1,4 +1,3 @@ +- notice: Fencing controller-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recover-connection.summary b/pengine/test10/remote-recover-connection.summary +index f21b2b5..e0a9d0e 100644 +--- a/pengine/test10/remote-recover-connection.summary ++++ b/pengine/test10/remote-recover-connection.summary +@@ -37,9 +37,8 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + +- notice: Fencing controller-1: node is unclean + Transition Summary: +- * Fence (reboot) controller-1 ++ * Fence (reboot) controller-1 'peer is no longer part of the cluster' + * Move messaging-1 (Started controller-1 -> controller-2) + * Move galera-0 (Started controller-1 -> controller-2) + * Move galera-2 (Started controller-1 -> controller-2) +@@ -62,7 +61,6 @@ Executing cluster transition: + * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 + * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 + * Fencing controller-1 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: messaging-1_stop_0 + * Pseudo action: galera-0_stop_0 + * Pseudo action: galera-2_stop_0 +@@ -73,6 +71,7 @@ Executing cluster transition: + * Pseudo action: redis-master_stop_0 + * Pseudo action: haproxy-clone_stop_0 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 ++ * Pseudo action: stonith_complete + * Resource action: messaging-1 start on controller-2 + * Resource action: galera-0 start on controller-2 + * Resource action: galera-2 start on controller-2 +diff --git a/pengine/test10/remote-recover-fail.exp b/pengine/test10/remote-recover-fail.exp +index be45b88..bd014ae 100644 +--- a/pengine/test10/remote-recover-fail.exp ++++ b/pengine/test10/remote-recover-fail.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -24,7 +24,7 @@ + + + +- ++ + + + +@@ -40,83 +40,83 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -125,7 +125,7 @@ + + + +- ++ + + + +@@ -134,71 +134,60 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -207,14 +196,25 @@ + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-recover-fail.scores b/pengine/test10/remote-recover-fail.scores +index 02a0643..8fc7805 100644 +--- a/pengine/test10/remote-recover-fail.scores ++++ b/pengine/test10/remote-recover-fail.scores +@@ -1,4 +1,3 @@ +- notice: Fencing rhel7-auto4: node is unclean + Allocation scores: + native_color: FAKE1 allocation score on rhel7-auto1: 0 + native_color: FAKE1 allocation score on rhel7-auto2: 0 +diff --git a/pengine/test10/remote-recover-fail.summary b/pengine/test10/remote-recover-fail.summary +index fd518eb..302b716 100644 +--- a/pengine/test10/remote-recover-fail.summary ++++ b/pengine/test10/remote-recover-fail.summary +@@ -13,9 +13,8 @@ OFFLINE: [ rhel7-auto1 ] + FAKE5 (ocf::heartbeat:Dummy): Started rhel7-auto3 + FAKE6 (ocf::heartbeat:Dummy): Started rhel7-auto4 (UNCLEAN) + +- notice: Fencing rhel7-auto4: node is unclean + Transition Summary: +- * Fence (reboot) rhel7-auto4 ++ * Fence (reboot) rhel7-auto4 'FAKE2 is thought to be active there' + * Recover rhel7-auto4 (Started rhel7-auto2) + * Start FAKE1 (rhel7-auto2) + * Move FAKE2 (Started rhel7-auto4 -> rhel7-auto3) +@@ -25,16 +24,16 @@ Executing cluster transition: + * Resource action: FAKE3 monitor=10000 on rhel7-auto2 + * Resource action: FAKE4 monitor=10000 on rhel7-auto3 + * Fencing rhel7-auto4 (reboot) +- * Pseudo action: stonith_complete +- * Resource action: FAKE1 start on rhel7-auto2 + * Pseudo action: FAKE2_stop_0 + * Pseudo action: FAKE6_stop_0 ++ * Pseudo action: stonith_complete + * Resource action: rhel7-auto4 stop on rhel7-auto2 +- * Resource action: FAKE1 monitor=10000 on rhel7-auto2 ++ * Resource action: FAKE1 start on rhel7-auto2 + * Resource action: FAKE2 start on rhel7-auto3 + * Resource action: FAKE6 start on rhel7-auto2 + * Pseudo action: all_stopped + * Resource action: rhel7-auto4 start on rhel7-auto2 ++ * Resource action: FAKE1 monitor=10000 on rhel7-auto2 + * Resource action: FAKE2 monitor=10000 on rhel7-auto3 + * Resource action: FAKE6 monitor=10000 on rhel7-auto2 + * Resource action: rhel7-auto4 monitor=60000 on rhel7-auto2 +diff --git a/pengine/test10/remote-recover-no-resources.exp b/pengine/test10/remote-recover-no-resources.exp +index ba9b17b..987acfd 100644 +--- a/pengine/test10/remote-recover-no-resources.exp ++++ b/pengine/test10/remote-recover-no-resources.exp +@@ -1,69 +1,69 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -119,107 +119,107 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -232,23 +232,23 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + +@@ -261,68 +261,68 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -334,7 +334,7 @@ + + + +- ++ + + + +@@ -342,211 +342,211 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -555,36 +555,36 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -593,65 +593,65 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -659,7 +659,7 @@ + + + +- ++ + + + +@@ -668,88 +668,88 @@ + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-recover-no-resources.scores b/pengine/test10/remote-recover-no-resources.scores +index 36a1ff4..e918fc6 100644 +--- a/pengine/test10/remote-recover-no-resources.scores ++++ b/pengine/test10/remote-recover-no-resources.scores +@@ -1,5 +1,3 @@ +- notice: Fencing controller-1: node is unclean +- notice: Fencing messaging-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary +index c3cd6a8..19b5181 100644 +--- a/pengine/test10/remote-recover-no-resources.summary ++++ b/pengine/test10/remote-recover-no-resources.summary +@@ -37,11 +37,9 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + +- notice: Fencing controller-1: node is unclean +- notice: Fencing messaging-1: node is unclean + Transition Summary: +- * Fence (reboot) messaging-1 +- * Fence (reboot) controller-1 ++ * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' ++ * Fence (reboot) controller-1 'peer is no longer part of the cluster' + * Stop messaging-1 (controller-1) + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +diff --git a/pengine/test10/remote-recover-unknown.exp b/pengine/test10/remote-recover-unknown.exp +index d63cf81..b8d51be 100644 +--- a/pengine/test10/remote-recover-unknown.exp ++++ b/pengine/test10/remote-recover-unknown.exp +@@ -1,69 +1,69 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -119,107 +119,107 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -232,23 +232,23 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + +@@ -261,68 +261,68 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -334,7 +334,7 @@ + + + +- ++ + + + +@@ -342,211 +342,211 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -555,36 +555,36 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -593,65 +593,65 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -659,112 +659,112 @@ + + + +- +- ++ ++ + +- ++ + + + + + +- ++ + + + + + +- +- ++ ++ + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-recover-unknown.scores b/pengine/test10/remote-recover-unknown.scores +index d7d7713..e918fc6 100644 +--- a/pengine/test10/remote-recover-unknown.scores ++++ b/pengine/test10/remote-recover-unknown.scores +@@ -1,6 +1,3 @@ +- notice: Fencing controller-1: node is unclean +- notice: Fencing galera-2: node is unclean +- notice: Fencing messaging-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary +index 5b03065..772a699 100644 +--- a/pengine/test10/remote-recover-unknown.summary ++++ b/pengine/test10/remote-recover-unknown.summary +@@ -37,13 +37,10 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + +- notice: Fencing controller-1: node is unclean +- notice: Fencing galera-2: node is unclean +- notice: Fencing messaging-1: node is unclean + Transition Summary: +- * Fence (reboot) messaging-1 +- * Fence (reboot) galera-2 +- * Fence (reboot) controller-1 ++ * Fence (reboot) galera-2 'resources are in an unknown state and the connection is unrecoverable' ++ * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' ++ * Fence (reboot) controller-1 'peer is no longer part of the cluster' + * Stop messaging-1 (controller-1) + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +@@ -72,33 +69,33 @@ Executing cluster transition: + * Pseudo action: redis-master_stop_0 + * Pseudo action: haproxy-clone_stop_0 + * Fencing galera-2 (reboot) +- * Pseudo action: redis_stop_0 +- * Pseudo action: redis-master_stopped_0 +- * Pseudo action: haproxy_stop_0 +- * Pseudo action: haproxy-clone_stopped_0 + * Fencing messaging-1 (reboot) + * Pseudo action: stonith_complete + * Resource action: galera-0 start on controller-2 + * Pseudo action: rabbitmq_post_notify_stop_0 + * Pseudo action: rabbitmq-clone_stop_0 + * Resource action: galera monitor=10000 on galera-0 +- * Pseudo action: redis-master_post_notify_stopped_0 +- * Pseudo action: ip-172.17.1.14_stop_0 +- * Pseudo action: ip-172.17.1.17_stop_0 +- * Pseudo action: ip-172.17.4.11_stop_0 ++ * Pseudo action: redis_stop_0 ++ * Pseudo action: redis-master_stopped_0 ++ * Pseudo action: haproxy_stop_0 ++ * Pseudo action: haproxy-clone_stopped_0 + * Resource action: galera-0 monitor=20000 on controller-2 + * Resource action: rabbitmq notify on messaging-2 + * Resource action: rabbitmq notify on messaging-0 + * Pseudo action: rabbitmq_notified_0 + * Pseudo action: rabbitmq_stop_0 + * Pseudo action: rabbitmq-clone_stopped_0 ++ * Pseudo action: redis-master_post_notify_stopped_0 ++ * Pseudo action: ip-172.17.1.14_stop_0 ++ * Pseudo action: ip-172.17.1.17_stop_0 ++ * Pseudo action: ip-172.17.4.11_stop_0 ++ * Pseudo action: messaging-1_stop_0 + * Resource action: redis notify on controller-0 + * Resource action: redis notify on controller-2 + * Pseudo action: redis-master_confirmed-post_notify_stopped_0 + * Resource action: ip-172.17.1.14 start on controller-2 + * Resource action: ip-172.17.1.17 start on controller-2 + * Resource action: ip-172.17.4.11 start on controller-2 +- * Pseudo action: messaging-1_stop_0 + * Pseudo action: redis_notified_0 + * Resource action: ip-172.17.1.14 monitor=10000 on controller-2 + * Resource action: ip-172.17.1.17 monitor=10000 on controller-2 +diff --git a/pengine/test10/remote-recover.exp b/pengine/test10/remote-recover.exp +index 918db8d..1fab2f1 100644 +--- a/pengine/test10/remote-recover.exp ++++ b/pengine/test10/remote-recover.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -23,20 +23,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -45,14 +45,14 @@ + + + +- ++ + + + + + + +- ++ + + + +diff --git a/pengine/test10/remote-recovery.exp b/pengine/test10/remote-recovery.exp +index 62860f0..cf74efb 100644 +--- a/pengine/test10/remote-recovery.exp ++++ b/pengine/test10/remote-recovery.exp +@@ -1,163 +1,163 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -187,22 +187,22 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -215,7 +215,7 @@ + + + +- ++ + + + +@@ -228,7 +228,7 @@ + + + +- ++ + + + +@@ -244,7 +244,7 @@ + + + +- ++ + + + +@@ -257,7 +257,7 @@ + + + +- ++ + + + +@@ -266,13 +266,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -284,28 +284,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -317,7 +317,7 @@ + + + +- ++ + + + +@@ -325,211 +325,211 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -538,33 +538,33 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -573,59 +573,59 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -633,76 +633,76 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-recovery.scores b/pengine/test10/remote-recovery.scores +index 3d3beae..56a18eb 100644 +--- a/pengine/test10/remote-recovery.scores ++++ b/pengine/test10/remote-recovery.scores +@@ -1,4 +1,3 @@ +- notice: Fencing controller-1: node is unclean + Allocation scores: + Using the original execution date of: 2017-05-03 13:33:24Z + clone_color: galera-master allocation score on controller-0: -INFINITY +diff --git a/pengine/test10/remote-recovery.summary b/pengine/test10/remote-recovery.summary +index f21b2b5..e0a9d0e 100644 +--- a/pengine/test10/remote-recovery.summary ++++ b/pengine/test10/remote-recovery.summary +@@ -37,9 +37,8 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + stonith-fence_ipmilan-525400b4f6bd (stonith:fence_ipmilan): Started controller-0 + stonith-fence_ipmilan-5254005bdbb5 (stonith:fence_ipmilan): Started controller-1 (UNCLEAN) + +- notice: Fencing controller-1: node is unclean + Transition Summary: +- * Fence (reboot) controller-1 ++ * Fence (reboot) controller-1 'peer is no longer part of the cluster' + * Move messaging-1 (Started controller-1 -> controller-2) + * Move galera-0 (Started controller-1 -> controller-2) + * Move galera-2 (Started controller-1 -> controller-2) +@@ -62,7 +61,6 @@ Executing cluster transition: + * Resource action: stonith-fence_ipmilan-525400b4f6bd monitor=60000 on controller-0 + * Pseudo action: stonith-fence_ipmilan-5254005bdbb5_stop_0 + * Fencing controller-1 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: messaging-1_stop_0 + * Pseudo action: galera-0_stop_0 + * Pseudo action: galera-2_stop_0 +@@ -73,6 +71,7 @@ Executing cluster transition: + * Pseudo action: redis-master_stop_0 + * Pseudo action: haproxy-clone_stop_0 + * Resource action: stonith-fence_ipmilan-5254005bdbb5 start on controller-2 ++ * Pseudo action: stonith_complete + * Resource action: messaging-1 start on controller-2 + * Resource action: galera-0 start on controller-2 + * Resource action: galera-2 start on controller-2 +diff --git a/pengine/test10/remote-unclean2.exp b/pengine/test10/remote-unclean2.exp +index 045e976..ca0b3ba 100644 +--- a/pengine/test10/remote-unclean2.exp ++++ b/pengine/test10/remote-unclean2.exp +@@ -1,23 +1,23 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -36,46 +36,46 @@ + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/remote-unclean2.scores b/pengine/test10/remote-unclean2.scores +index 2c2c9a5..72a5953 100644 +--- a/pengine/test10/remote-unclean2.scores ++++ b/pengine/test10/remote-unclean2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing rhel7-auto4: node is unclean + Allocation scores: + native_color: rhel7-auto4 allocation score on rhel7-auto1: 0 + native_color: rhel7-auto4 allocation score on rhel7-auto2: 0 +diff --git a/pengine/test10/remote-unclean2.summary b/pengine/test10/remote-unclean2.summary +index 06047d1..1408c09 100644 +--- a/pengine/test10/remote-unclean2.summary ++++ b/pengine/test10/remote-unclean2.summary +@@ -6,9 +6,8 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ] + shooter (stonith:fence_xvm): Started rhel7-auto2 + rhel7-auto4 (ocf::pacemaker:remote): FAILED rhel7-auto1 + +- notice: Fencing rhel7-auto4: node is unclean + Transition Summary: +- * Fence (reboot) rhel7-auto4 ++ * Fence (reboot) rhel7-auto4 'remote connection is unrecoverable' + * Recover rhel7-auto4 (Started rhel7-auto1) + + Executing cluster transition: +diff --git a/pengine/test10/start-then-stop-with-unfence.scores b/pengine/test10/start-then-stop-with-unfence.scores +index a53e9f2..d353bef 100644 +--- a/pengine/test10/start-then-stop-with-unfence.scores ++++ b/pengine/test10/start-then-stop-with-unfence.scores +@@ -1,4 +1,3 @@ +- notice: Unfencing rhel7-node1.example.com: Required by ip1 + Allocation scores: + clone_color: jrummy-clone allocation score on rhel7-node1.example.com: 500 + clone_color: jrummy-clone allocation score on rhel7-node2.example.com: 500 +diff --git a/pengine/test10/start-then-stop-with-unfence.summary b/pengine/test10/start-then-stop-with-unfence.summary +index bb376f5..5e127f7 100644 +--- a/pengine/test10/start-then-stop-with-unfence.summary ++++ b/pengine/test10/start-then-stop-with-unfence.summary +@@ -10,9 +10,8 @@ Online: [ rhel7-node1.example.com rhel7-node2.example.com ] + Started: [ rhel7-node2.example.com ] + Stopped: [ rhel7-node1.example.com ] + +- notice: Unfencing rhel7-node1.example.com: Required by ip1 + Transition Summary: +- * Fence (on) rhel7-node1.example.com ++ * Fence (on) rhel7-node1.example.com 'Required by ip1' + * Start mpath-node1 (rhel7-node1.example.com) + * Move ip1 (Started rhel7-node2.example.com -> rhel7-node1.example.com) + * Start jrummy:1 (rhel7-node1.example.com) +diff --git a/pengine/test10/stonith-0.exp b/pengine/test10/stonith-0.exp +index 2a830b7..c409992 100644 +--- a/pengine/test10/stonith-0.exp ++++ b/pengine/test10/stonith-0.exp +@@ -1,153 +1,153 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -156,13 +156,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -171,22 +171,22 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -194,172 +194,172 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -368,72 +368,72 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + +diff --git a/pengine/test10/stonith-0.scores b/pengine/test10/stonith-0.scores +index db3ddb9..7aab8cd 100644 +--- a/pengine/test10/stonith-0.scores ++++ b/pengine/test10/stonith-0.scores +@@ -1,5 +1,3 @@ +- notice: Fencing c001n03: node is unclean +- notice: Fencing c001n05: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on c001n02: 0 + clone_color: DoFencing allocation score on c001n03: 0 +diff --git a/pengine/test10/stonith-0.summary b/pengine/test10/stonith-0.summary +index 6983a90..1062520 100644 +--- a/pengine/test10/stonith-0.summary ++++ b/pengine/test10/stonith-0.summary +@@ -36,11 +36,9 @@ Online: [ c001n02 c001n04 c001n06 c001n07 c001n08 ] + ocf_msdummy:12 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n06 + ocf_msdummy:13 (ocf::heartbeat:/usr/lib/heartbeat/cts/OCFMSDummy): Slave c001n06 + +- notice: Fencing c001n03: node is unclean +- notice: Fencing c001n05: node is unclean + Transition Summary: +- * Fence (reboot) c001n05 +- * Fence (reboot) c001n03 ++ * Fence (reboot) c001n05 'ocf_192.168.100.183 failed there' ++ * Fence (reboot) c001n03 'ocf_192.168.100.183 failed there' + * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) + * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) + * Recover ocf_192.168.100.183 (Started c001n03 -> c001n02) +@@ -51,12 +49,12 @@ Executing cluster transition: + * Resource action: child_DoFencing:4 monitor=20000 on c001n08 + * Fencing c001n05 (reboot) + * Fencing c001n03 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: group-1_stop_0 + * Pseudo action: ocf_192.168.100.183_stop_0 + * Pseudo action: ocf_192.168.100.183_stop_0 + * Pseudo action: rsc_c001n05_stop_0 + * Pseudo action: rsc_c001n07_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: heartbeat_192.168.100.182_stop_0 + * Resource action: rsc_c001n05 start on c001n07 + * Resource action: rsc_c001n07 start on c001n07 +diff --git a/pengine/test10/stonith-1.exp b/pengine/test10/stonith-1.exp +index f7499ad..2ccd1ef 100644 +--- a/pengine/test10/stonith-1.exp ++++ b/pengine/test10/stonith-1.exp +@@ -1,22 +1,22 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -24,7 +24,7 @@ + + + +- ++ + + + +@@ -35,36 +35,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -73,48 +73,48 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -123,48 +123,48 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -173,63 +173,63 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -237,362 +237,361 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/stonith-1.scores b/pengine/test10/stonith-1.scores +index 196b9f5..7422355 100644 +--- a/pengine/test10/stonith-1.scores ++++ b/pengine/test10/stonith-1.scores +@@ -1,4 +1,3 @@ +- notice: Fencing sles-3: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on sles-1: 0 + clone_color: DoFencing allocation score on sles-2: 0 +diff --git a/pengine/test10/stonith-1.summary b/pengine/test10/stonith-1.summary +index 111bcbe..03c8a39 100644 +--- a/pengine/test10/stonith-1.summary ++++ b/pengine/test10/stonith-1.summary +@@ -27,9 +27,8 @@ Online: [ sles-1 sles-2 sles-4 ] + ocf_msdummy:6 (ocf::heartbeat:Stateful): Stopped + ocf_msdummy:7 (ocf::heartbeat:Stateful): Stopped + +- notice: Fencing sles-3: node is unclean + Transition Summary: +- * Fence (reboot) sles-3 ++ * Fence (reboot) sles-3 'peer is no longer part of the cluster' + * Start r192.168.100.183 (sles-1) + * Move migrator (Started sles-3 -> sles-4) + * Move rsc_sles-3 (Started sles-3 -> sles-4) +@@ -49,16 +48,14 @@ Executing cluster transition: + * Resource action: rsc_sles-4 monitor=5000 on sles-4 + * Pseudo action: DoFencing_stop_0 + * Fencing sles-3 (reboot) +- * Pseudo action: stonith_complete +- * Resource action: r192.168.100.183 start on sles-1 + * Pseudo action: migrator_stop_0 + * Pseudo action: rsc_sles-3_stop_0 + * Pseudo action: child_DoFencing:2_stop_0 + * Pseudo action: DoFencing_stopped_0 + * Pseudo action: DoFencing_start_0 + * Pseudo action: master_rsc_1_stop_0 +- * Pseudo action: group-1_running_0 +- * Resource action: r192.168.100.183 monitor=5000 on sles-1 ++ * Pseudo action: stonith_complete ++ * Resource action: r192.168.100.183 start on sles-1 + * Resource action: migrator start on sles-4 + * Resource action: rsc_sles-3 start on sles-4 + * Resource action: child_DoFencing:2 start on sles-4 +@@ -68,6 +65,8 @@ Executing cluster transition: + * Pseudo action: master_rsc_1_stopped_0 + * Pseudo action: master_rsc_1_start_0 + * Pseudo action: all_stopped ++ * Pseudo action: group-1_running_0 ++ * Resource action: r192.168.100.183 monitor=5000 on sles-1 + * Resource action: migrator monitor=10000 on sles-4 + * Resource action: rsc_sles-3 monitor=5000 on sles-4 + * Resource action: child_DoFencing:2 monitor=60000 on sles-4 +diff --git a/pengine/test10/stonith-2.exp b/pengine/test10/stonith-2.exp +index 1828377..e433e49 100644 +--- a/pengine/test10/stonith-2.exp ++++ b/pengine/test10/stonith-2.exp +@@ -1,64 +1,63 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/stonith-2.scores b/pengine/test10/stonith-2.scores +index 8c11561..e1dd011 100644 +--- a/pengine/test10/stonith-2.scores ++++ b/pengine/test10/stonith-2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing sles-5: node is unclean + Allocation scores: + clone_color: DoFencing allocation score on sles-1: 0 + clone_color: DoFencing allocation score on sles-2: 0 +diff --git a/pengine/test10/stonith-2.summary b/pengine/test10/stonith-2.summary +index c8e1d70..2a6c218 100644 +--- a/pengine/test10/stonith-2.summary ++++ b/pengine/test10/stonith-2.summary +@@ -32,9 +32,8 @@ Online: [ sles-1 sles-2 sles-3 sles-4 sles-6 ] + ocf_msdummy:10 (ocf::heartbeat:Stateful): Slave sles-2 + ocf_msdummy:11 (ocf::heartbeat:Stateful): Slave sles-3 + +- notice: Fencing sles-5: node is unclean + Transition Summary: +- * Fence (reboot) sles-5 ++ * Fence (reboot) sles-5 'peer is no longer part of the cluster' + * Start rsc_sles-5 (sles-6) + + Executing cluster transition: +diff --git a/pengine/test10/stonith-3.scores b/pengine/test10/stonith-3.scores +index eef30fc..a1bd3d9 100644 +--- a/pengine/test10/stonith-3.scores ++++ b/pengine/test10/stonith-3.scores +@@ -1,4 +1,3 @@ +- notice: Fencing rh5node1: node is unclean + Allocation scores: + clone_color: clnStonith allocation score on rh5node1: 0 + clone_color: clnStonith allocation score on rh5node2: 0 +diff --git a/pengine/test10/stonith-3.summary b/pengine/test10/stonith-3.summary +index 015dd77..cc10cf5 100644 +--- a/pengine/test10/stonith-3.summary ++++ b/pengine/test10/stonith-3.summary +@@ -7,9 +7,8 @@ Online: [ rh5node2 ] + Clone Set: clnStonith [grpStonith] + Stopped: [ rh5node1 rh5node2 ] + +- notice: Fencing rh5node1: node is unclean + Transition Summary: +- * Fence (reboot) rh5node1 ++ * Fence (reboot) rh5node1 'node is unclean' + * Start prmIpPostgreSQLDB (rh5node2) + * Start prmStonith:0 (rh5node2) + +diff --git a/pengine/test10/stonith-4.exp b/pengine/test10/stonith-4.exp +index 298a636..d3e9e67 100644 +--- a/pengine/test10/stonith-4.exp ++++ b/pengine/test10/stonith-4.exp +@@ -1,81 +1,81 @@ + + + +- +- +- +- +- +- ++ ++ ++ + + + +- ++ + + + + + +- +- +- +- +- +- ++ ++ ++ + + + +- ++ + + + + + +- +- ++ ++ + +- ++ + + + +- +- +- +- +- ++ + + + +- +- ++ ++ + +- ++ + + + +- ++ ++ ++ ++ ++ + + + +- +- +- ++ ++ ++ ++ ++ ++ + + + +- ++ + + + + + +- +- +- ++ ++ ++ ++ ++ ++ + + + +- ++ + + + +diff --git a/pengine/test10/stonith-4.scores b/pengine/test10/stonith-4.scores +index 0bf1cf5..ac6fe0b 100644 +--- a/pengine/test10/stonith-4.scores ++++ b/pengine/test10/stonith-4.scores +@@ -1,7 +1,3 @@ +- notice: Fencing pcmk-10: node is unclean +- notice: Fencing pcmk-5: node is unclean +- notice: Fencing pcmk-7: node is unclean +- notice: Fencing pcmk-8: node is unclean + Allocation scores: + native_color: Fencing allocation score on pcmk-10: 0 + native_color: Fencing allocation score on pcmk-11: 0 +diff --git a/pengine/test10/stonith-4.summary b/pengine/test10/stonith-4.summary +index 0c9b5bd..76b1f44 100644 +--- a/pengine/test10/stonith-4.summary ++++ b/pengine/test10/stonith-4.summary +@@ -13,15 +13,11 @@ OFFLINE: [ pcmk-4 pcmk-6 ] + + Fencing (stonith:fence_xvm): Stopped + +- notice: Fencing pcmk-10: node is unclean +- notice: Fencing pcmk-5: node is unclean +- notice: Fencing pcmk-7: node is unclean +- notice: Fencing pcmk-8: node is unclean + Transition Summary: +- * Fence (reboot) pcmk-8 +- * Fence (reboot) pcmk-7 +- * Fence (reboot) pcmk-5 +- * Fence (reboot) pcmk-10 ++ * Fence (reboot) pcmk-10 'peer process is no longer available' ++ * Fence (reboot) pcmk-8 'peer has not been seen by the cluster' ++ * Fence (reboot) pcmk-7 'peer failed the pacemaker membership criteria' ++ * Fence (reboot) pcmk-5 'peer has not been seen by the cluster' + * Start Fencing (pcmk-1 - blocked) + + Executing cluster transition: +diff --git a/pengine/test10/stop-failure-no-quorum.exp b/pengine/test10/stop-failure-no-quorum.exp +index cf284bd..ddd383d 100644 +--- a/pengine/test10/stop-failure-no-quorum.exp ++++ b/pengine/test10/stop-failure-no-quorum.exp +@@ -1,82 +1,82 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/stop-failure-no-quorum.scores b/pengine/test10/stop-failure-no-quorum.scores +index c1893b6..df30423 100644 +--- a/pengine/test10/stop-failure-no-quorum.scores ++++ b/pengine/test10/stop-failure-no-quorum.scores +@@ -1,4 +1,3 @@ +- notice: Fencing pcmk-2: node is unclean + Allocation scores: + clone_color: clvm-clone allocation score on pcmk-1: 0 + clone_color: clvm-clone allocation score on pcmk-2: -INFINITY +diff --git a/pengine/test10/stop-failure-no-quorum.summary b/pengine/test10/stop-failure-no-quorum.summary +index d91edc1..4e337ab 100644 +--- a/pengine/test10/stop-failure-no-quorum.summary ++++ b/pengine/test10/stop-failure-no-quorum.summary +@@ -14,9 +14,8 @@ Online: [ pcmk-1 ] + ClusterIP (ocf::heartbeat:IPaddr2): Stopped + Fencing (stonith:fence_xvm): Stopped + +- notice: Fencing pcmk-2: node is unclean + Transition Summary: +- * Fence (reboot) pcmk-2 ++ * Fence (reboot) pcmk-2 'clvm:0 failed there' + * Start dlm:0 (pcmk-1 - blocked) + * Stop clvm:0 (pcmk-2) + * Start clvm:2 (pcmk-1 - blocked) +@@ -25,8 +24,8 @@ Transition Summary: + + Executing cluster transition: + * Fencing pcmk-2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: clvm-clone_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: clvm_stop_0 + * Pseudo action: clvm-clone_stopped_0 + * Pseudo action: all_stopped +diff --git a/pengine/test10/stop-failure-with-fencing.exp b/pengine/test10/stop-failure-with-fencing.exp +index 3d8b34a..e7e7ca6 100644 +--- a/pengine/test10/stop-failure-with-fencing.exp ++++ b/pengine/test10/stop-failure-with-fencing.exp +@@ -1,49 +1,49 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -52,40 +52,40 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/stop-failure-with-fencing.scores b/pengine/test10/stop-failure-with-fencing.scores +index 0dc0813..cf62806 100644 +--- a/pengine/test10/stop-failure-with-fencing.scores ++++ b/pengine/test10/stop-failure-with-fencing.scores +@@ -1,4 +1,3 @@ +- notice: Fencing pcmk-2: node is unclean + Allocation scores: + clone_color: clvm-clone allocation score on pcmk-1: 0 + clone_color: clvm-clone allocation score on pcmk-2: -INFINITY +diff --git a/pengine/test10/stop-failure-with-fencing.summary b/pengine/test10/stop-failure-with-fencing.summary +index 79e6105..cdb39b0 100644 +--- a/pengine/test10/stop-failure-with-fencing.summary ++++ b/pengine/test10/stop-failure-with-fencing.summary +@@ -13,9 +13,8 @@ Online: [ pcmk-1 ] + ClusterIP (ocf::heartbeat:IPaddr2): Stopped + Fencing (stonith:fence_xvm): Stopped + +- notice: Fencing pcmk-2: node is unclean + Transition Summary: +- * Fence (reboot) pcmk-2 ++ * Fence (reboot) pcmk-2 'clvm:0 failed there' + * Start dlm:0 (pcmk-1 - blocked) + * Stop clvm:0 (pcmk-2) + * Start clvm:1 (pcmk-1 - blocked) +@@ -25,8 +24,8 @@ Transition Summary: + Executing cluster transition: + * Resource action: Fencing monitor on pcmk-1 + * Fencing pcmk-2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: clvm-clone_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: clvm_stop_0 + * Pseudo action: clvm-clone_stopped_0 + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealth1.scores b/pengine/test10/systemhealth1.scores +index c55f6e2..8a59654 100644 +--- a/pengine/test10/systemhealth1.scores ++++ b/pengine/test10/systemhealth1.scores +@@ -1,5 +1,3 @@ +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealth1.summary b/pengine/test10/systemhealth1.summary +index 335b94a..351d508 100644 +--- a/pengine/test10/systemhealth1.summary ++++ b/pengine/test10/systemhealth1.summary +@@ -7,11 +7,9 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d +- * Fence (reboot) hs21c ++ * Fence (reboot) hs21d 'node is unclean' ++ * Fence (reboot) hs21c 'node is unclean' + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealth2.scores b/pengine/test10/systemhealth2.scores +index b20bd49..8a59654 100644 +--- a/pengine/test10/systemhealth2.scores ++++ b/pengine/test10/systemhealth2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealth2.summary b/pengine/test10/systemhealth2.summary +index b04d5a1..db68783 100644 +--- a/pengine/test10/systemhealth2.summary ++++ b/pengine/test10/systemhealth2.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealth3.scores b/pengine/test10/systemhealth3.scores +index b20bd49..8a59654 100644 +--- a/pengine/test10/systemhealth3.scores ++++ b/pengine/test10/systemhealth3.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealth3.summary b/pengine/test10/systemhealth3.summary +index b04d5a1..db68783 100644 +--- a/pengine/test10/systemhealth3.summary ++++ b/pengine/test10/systemhealth3.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealthm1.scores b/pengine/test10/systemhealthm1.scores +index c55f6e2..8a59654 100644 +--- a/pengine/test10/systemhealthm1.scores ++++ b/pengine/test10/systemhealthm1.scores +@@ -1,5 +1,3 @@ +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthm1.summary b/pengine/test10/systemhealthm1.summary +index 335b94a..351d508 100644 +--- a/pengine/test10/systemhealthm1.summary ++++ b/pengine/test10/systemhealthm1.summary +@@ -7,11 +7,9 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d +- * Fence (reboot) hs21c ++ * Fence (reboot) hs21d 'node is unclean' ++ * Fence (reboot) hs21c 'node is unclean' + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealthm2.scores b/pengine/test10/systemhealthm2.scores +index b20bd49..8a59654 100644 +--- a/pengine/test10/systemhealthm2.scores ++++ b/pengine/test10/systemhealthm2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthm2.summary b/pengine/test10/systemhealthm2.summary +index b04d5a1..db68783 100644 +--- a/pengine/test10/systemhealthm2.summary ++++ b/pengine/test10/systemhealthm2.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealthm3.scores b/pengine/test10/systemhealthm3.scores +index 00b8d64..7e8804e 100644 +--- a/pengine/test10/systemhealthm3.scores ++++ b/pengine/test10/systemhealthm3.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: -INFINITY + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthm3.summary b/pengine/test10/systemhealthm3.summary +index 55d5cf7..836d8f5 100644 +--- a/pengine/test10/systemhealthm3.summary ++++ b/pengine/test10/systemhealthm3.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/systemhealthn1.scores b/pengine/test10/systemhealthn1.scores +index c55f6e2..8a59654 100644 +--- a/pengine/test10/systemhealthn1.scores ++++ b/pengine/test10/systemhealthn1.scores +@@ -1,5 +1,3 @@ +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthn1.summary b/pengine/test10/systemhealthn1.summary +index 335b94a..351d508 100644 +--- a/pengine/test10/systemhealthn1.summary ++++ b/pengine/test10/systemhealthn1.summary +@@ -7,11 +7,9 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d +- * Fence (reboot) hs21c ++ * Fence (reboot) hs21d 'node is unclean' ++ * Fence (reboot) hs21c 'node is unclean' + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealthn2.scores b/pengine/test10/systemhealthn2.scores +index b20bd49..8a59654 100644 +--- a/pengine/test10/systemhealthn2.scores ++++ b/pengine/test10/systemhealthn2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthn2.summary b/pengine/test10/systemhealthn2.summary +index b04d5a1..db68783 100644 +--- a/pengine/test10/systemhealthn2.summary ++++ b/pengine/test10/systemhealthn2.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealthn3.scores b/pengine/test10/systemhealthn3.scores +index b20bd49..8a59654 100644 +--- a/pengine/test10/systemhealthn3.scores ++++ b/pengine/test10/systemhealthn3.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthn3.summary b/pengine/test10/systemhealthn3.summary +index b04d5a1..db68783 100644 +--- a/pengine/test10/systemhealthn3.summary ++++ b/pengine/test10/systemhealthn3.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + * Start stonith-1 (hs21c) + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) +diff --git a/pengine/test10/systemhealtho1.scores b/pengine/test10/systemhealtho1.scores +index c55f6e2..8a59654 100644 +--- a/pengine/test10/systemhealtho1.scores ++++ b/pengine/test10/systemhealtho1.scores +@@ -1,5 +1,3 @@ +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealtho1.summary b/pengine/test10/systemhealtho1.summary +index 335b94a..351d508 100644 +--- a/pengine/test10/systemhealtho1.summary ++++ b/pengine/test10/systemhealtho1.summary +@@ -7,11 +7,9 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d +- * Fence (reboot) hs21c ++ * Fence (reboot) hs21d 'node is unclean' ++ * Fence (reboot) hs21c 'node is unclean' + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealtho2.scores b/pengine/test10/systemhealtho2.scores +index 00b8d64..7e8804e 100644 +--- a/pengine/test10/systemhealtho2.scores ++++ b/pengine/test10/systemhealtho2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: -INFINITY + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealtho2.summary b/pengine/test10/systemhealtho2.summary +index 55d5cf7..836d8f5 100644 +--- a/pengine/test10/systemhealtho2.summary ++++ b/pengine/test10/systemhealtho2.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/systemhealtho3.scores b/pengine/test10/systemhealtho3.scores +index 00b8d64..7e8804e 100644 +--- a/pengine/test10/systemhealtho3.scores ++++ b/pengine/test10/systemhealtho3.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: -INFINITY + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealtho3.summary b/pengine/test10/systemhealtho3.summary +index 55d5cf7..836d8f5 100644 +--- a/pengine/test10/systemhealtho3.summary ++++ b/pengine/test10/systemhealtho3.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/systemhealthp1.scores b/pengine/test10/systemhealthp1.scores +index c55f6e2..8a59654 100644 +--- a/pengine/test10/systemhealthp1.scores ++++ b/pengine/test10/systemhealthp1.scores +@@ -1,5 +1,3 @@ +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 100 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthp1.summary b/pengine/test10/systemhealthp1.summary +index 335b94a..351d508 100644 +--- a/pengine/test10/systemhealthp1.summary ++++ b/pengine/test10/systemhealthp1.summary +@@ -7,11 +7,9 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21c: node is unclean +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d +- * Fence (reboot) hs21c ++ * Fence (reboot) hs21d 'node is unclean' ++ * Fence (reboot) hs21c 'node is unclean' + + Executing cluster transition: + * Fencing hs21d (reboot) +diff --git a/pengine/test10/systemhealthp2.scores b/pengine/test10/systemhealthp2.scores +index 7ca75bf..20a08f2 100644 +--- a/pengine/test10/systemhealthp2.scores ++++ b/pengine/test10/systemhealthp2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: 0 + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthp2.summary b/pengine/test10/systemhealthp2.summary +index 6c44a74..36d9675 100644 +--- a/pengine/test10/systemhealthp2.summary ++++ b/pengine/test10/systemhealthp2.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + * Start apache_1 (hs21c) + * Start nfs_1 (hs21c) + +diff --git a/pengine/test10/systemhealthp3.scores b/pengine/test10/systemhealthp3.scores +index 00b8d64..7e8804e 100644 +--- a/pengine/test10/systemhealthp3.scores ++++ b/pengine/test10/systemhealthp3.scores +@@ -1,4 +1,3 @@ +- notice: Fencing hs21d: node is unclean + Allocation scores: + native_color: apache_1 allocation score on hs21c: -INFINITY + native_color: apache_1 allocation score on hs21d: 0 +diff --git a/pengine/test10/systemhealthp3.summary b/pengine/test10/systemhealthp3.summary +index 55d5cf7..836d8f5 100644 +--- a/pengine/test10/systemhealthp3.summary ++++ b/pengine/test10/systemhealthp3.summary +@@ -7,9 +7,8 @@ Online: [ hs21c ] + apache_1 (ocf::heartbeat:apache): Stopped + nfs_1 (ocf::heartbeat:Filesystem): Stopped + +- notice: Fencing hs21d: node is unclean + Transition Summary: +- * Fence (reboot) hs21d ++ * Fence (reboot) hs21d 'node is unclean' + + Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c +diff --git a/pengine/test10/ticket-clone-21.exp b/pengine/test10/ticket-clone-21.exp +index 23266d4..bc25bf2 100644 +--- a/pengine/test10/ticket-clone-21.exp ++++ b/pengine/test10/ticket-clone-21.exp +@@ -1,137 +1,136 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- ++ ++ ++ + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + +- +- +- +- +- +- +- +- +- + +- ++ + + +- ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/ticket-clone-21.scores b/pengine/test10/ticket-clone-21.scores +index e0fc0d8..89e2d0f 100644 +--- a/pengine/test10/ticket-clone-21.scores ++++ b/pengine/test10/ticket-clone-21.scores +@@ -1,5 +1,3 @@ +- notice: Fencing node1: node is unclean +- notice: Fencing node2: node is unclean + Allocation scores: + clone_color: clone1 allocation score on node1: 0 + clone_color: clone1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-clone-21.summary b/pengine/test10/ticket-clone-21.summary +index 13d2887..ce5335f 100644 +--- a/pengine/test10/ticket-clone-21.summary ++++ b/pengine/test10/ticket-clone-21.summary +@@ -6,11 +6,9 @@ Online: [ node1 node2 ] + Clone Set: clone1 [rsc1] + Started: [ node1 node2 ] + +- notice: Fencing node1: node is unclean +- notice: Fencing node2: node is unclean + Transition Summary: +- * Fence (reboot) node2 +- * Fence (reboot) node1 ++ * Fence (reboot) node2 'deadman ticket was lost' ++ * Fence (reboot) node1 'deadman ticket was lost' + * Stop rsc_stonith (node1) + * Stop rsc1:0 (node1) + * Stop rsc1:1 (node2) +@@ -19,8 +17,8 @@ Executing cluster transition: + * Fencing node1 (reboot) + * Pseudo action: rsc_stonith_stop_0 + * Fencing node2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: clone1_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: rsc1:1_stop_0 + * Pseudo action: rsc1:0_stop_0 + * Pseudo action: clone1_stopped_0 +diff --git a/pengine/test10/ticket-clone-9.exp b/pengine/test10/ticket-clone-9.exp +index 23266d4..bc25bf2 100644 +--- a/pengine/test10/ticket-clone-9.exp ++++ b/pengine/test10/ticket-clone-9.exp +@@ -1,137 +1,136 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- ++ ++ ++ + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + +- +- +- +- +- +- +- +- +- + +- ++ + + +- ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/ticket-clone-9.scores b/pengine/test10/ticket-clone-9.scores +index e0fc0d8..89e2d0f 100644 +--- a/pengine/test10/ticket-clone-9.scores ++++ b/pengine/test10/ticket-clone-9.scores +@@ -1,5 +1,3 @@ +- notice: Fencing node1: node is unclean +- notice: Fencing node2: node is unclean + Allocation scores: + clone_color: clone1 allocation score on node1: 0 + clone_color: clone1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-clone-9.summary b/pengine/test10/ticket-clone-9.summary +index 13d2887..ce5335f 100644 +--- a/pengine/test10/ticket-clone-9.summary ++++ b/pengine/test10/ticket-clone-9.summary +@@ -6,11 +6,9 @@ Online: [ node1 node2 ] + Clone Set: clone1 [rsc1] + Started: [ node1 node2 ] + +- notice: Fencing node1: node is unclean +- notice: Fencing node2: node is unclean + Transition Summary: +- * Fence (reboot) node2 +- * Fence (reboot) node1 ++ * Fence (reboot) node2 'deadman ticket was lost' ++ * Fence (reboot) node1 'deadman ticket was lost' + * Stop rsc_stonith (node1) + * Stop rsc1:0 (node1) + * Stop rsc1:1 (node2) +@@ -19,8 +17,8 @@ Executing cluster transition: + * Fencing node1 (reboot) + * Pseudo action: rsc_stonith_stop_0 + * Fencing node2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: clone1_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: rsc1:1_stop_0 + * Pseudo action: rsc1:0_stop_0 + * Pseudo action: clone1_stopped_0 +diff --git a/pengine/test10/ticket-group-21.exp b/pengine/test10/ticket-group-21.exp +index 82b3fdb..4a51d5f 100644 +--- a/pengine/test10/ticket-group-21.exp ++++ b/pengine/test10/ticket-group-21.exp +@@ -1,107 +1,106 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/ticket-group-21.scores b/pengine/test10/ticket-group-21.scores +index 8f01379..e5b8167 100644 +--- a/pengine/test10/ticket-group-21.scores ++++ b/pengine/test10/ticket-group-21.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node2: node is unclean + Allocation scores: + group_color: group1 allocation score on node1: 0 + group_color: group1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-group-21.summary b/pengine/test10/ticket-group-21.summary +index 80fb33f..385e44f 100644 +--- a/pengine/test10/ticket-group-21.summary ++++ b/pengine/test10/ticket-group-21.summary +@@ -7,17 +7,16 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): Started node2 + rsc2 (ocf::pacemaker:Dummy): Started node2 + +- notice: Fencing node2: node is unclean + Transition Summary: +- * Fence (reboot) node2 ++ * Fence (reboot) node2 'deadman ticket was lost' + * Stop rsc1 (node2) + * Stop rsc2 (node2) + + Executing cluster transition: + * Fencing node2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: group1_stop_0 + * Pseudo action: rsc2_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: all_stopped + * Pseudo action: group1_stopped_0 +diff --git a/pengine/test10/ticket-group-9.exp b/pengine/test10/ticket-group-9.exp +index 82b3fdb..4a51d5f 100644 +--- a/pengine/test10/ticket-group-9.exp ++++ b/pengine/test10/ticket-group-9.exp +@@ -1,107 +1,106 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/ticket-group-9.scores b/pengine/test10/ticket-group-9.scores +index 8f01379..e5b8167 100644 +--- a/pengine/test10/ticket-group-9.scores ++++ b/pengine/test10/ticket-group-9.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node2: node is unclean + Allocation scores: + group_color: group1 allocation score on node1: 0 + group_color: group1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-group-9.summary b/pengine/test10/ticket-group-9.summary +index 80fb33f..385e44f 100644 +--- a/pengine/test10/ticket-group-9.summary ++++ b/pengine/test10/ticket-group-9.summary +@@ -7,17 +7,16 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): Started node2 + rsc2 (ocf::pacemaker:Dummy): Started node2 + +- notice: Fencing node2: node is unclean + Transition Summary: +- * Fence (reboot) node2 ++ * Fence (reboot) node2 'deadman ticket was lost' + * Stop rsc1 (node2) + * Stop rsc2 (node2) + + Executing cluster transition: + * Fencing node2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: group1_stop_0 + * Pseudo action: rsc2_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: all_stopped + * Pseudo action: group1_stopped_0 +diff --git a/pengine/test10/ticket-master-21.exp b/pengine/test10/ticket-master-21.exp +index 71e57be..76114c0 100644 +--- a/pengine/test10/ticket-master-21.exp ++++ b/pengine/test10/ticket-master-21.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -22,55 +22,55 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -78,74 +78,73 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/ticket-master-21.scores b/pengine/test10/ticket-master-21.scores +index 89eac5e..dc79d67 100644 +--- a/pengine/test10/ticket-master-21.scores ++++ b/pengine/test10/ticket-master-21.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node1: node is unclean + Allocation scores: + clone_color: ms1 allocation score on node1: 0 + clone_color: ms1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-master-21.summary b/pengine/test10/ticket-master-21.summary +index dc2f5db..8fc50eb 100644 +--- a/pengine/test10/ticket-master-21.summary ++++ b/pengine/test10/ticket-master-21.summary +@@ -7,9 +7,8 @@ Online: [ node1 node2 ] + Masters: [ node1 ] + Slaves: [ node2 ] + +- notice: Fencing node1: node is unclean + Transition Summary: +- * Fence (reboot) node1 ++ * Fence (reboot) node1 'deadman ticket was lost' + * Move rsc_stonith (Started node1 -> node2) + * Demote rsc1:0 (Master -> Stopped node1) + +@@ -17,11 +16,11 @@ Executing cluster transition: + * Pseudo action: rsc_stonith_stop_0 + * Pseudo action: ms1_demote_0 + * Fencing node1 (reboot) +- * Pseudo action: stonith_complete + * Resource action: rsc_stonith start on node2 + * Pseudo action: rsc1:1_demote_0 + * Pseudo action: ms1_demoted_0 + * Pseudo action: ms1_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: rsc1:1_stop_0 + * Pseudo action: ms1_stopped_0 + * Pseudo action: all_stopped +diff --git a/pengine/test10/ticket-master-9.exp b/pengine/test10/ticket-master-9.exp +index 71e57be..76114c0 100644 +--- a/pengine/test10/ticket-master-9.exp ++++ b/pengine/test10/ticket-master-9.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -22,55 +22,55 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -78,74 +78,73 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/ticket-master-9.scores b/pengine/test10/ticket-master-9.scores +index 89eac5e..dc79d67 100644 +--- a/pengine/test10/ticket-master-9.scores ++++ b/pengine/test10/ticket-master-9.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node1: node is unclean + Allocation scores: + clone_color: ms1 allocation score on node1: 0 + clone_color: ms1 allocation score on node2: 0 +diff --git a/pengine/test10/ticket-master-9.summary b/pengine/test10/ticket-master-9.summary +index dc2f5db..8fc50eb 100644 +--- a/pengine/test10/ticket-master-9.summary ++++ b/pengine/test10/ticket-master-9.summary +@@ -7,9 +7,8 @@ Online: [ node1 node2 ] + Masters: [ node1 ] + Slaves: [ node2 ] + +- notice: Fencing node1: node is unclean + Transition Summary: +- * Fence (reboot) node1 ++ * Fence (reboot) node1 'deadman ticket was lost' + * Move rsc_stonith (Started node1 -> node2) + * Demote rsc1:0 (Master -> Stopped node1) + +@@ -17,11 +16,11 @@ Executing cluster transition: + * Pseudo action: rsc_stonith_stop_0 + * Pseudo action: ms1_demote_0 + * Fencing node1 (reboot) +- * Pseudo action: stonith_complete + * Resource action: rsc_stonith start on node2 + * Pseudo action: rsc1:1_demote_0 + * Pseudo action: ms1_demoted_0 + * Pseudo action: ms1_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: rsc1:1_stop_0 + * Pseudo action: ms1_stopped_0 + * Pseudo action: all_stopped +diff --git a/pengine/test10/ticket-primitive-21.exp b/pengine/test10/ticket-primitive-21.exp +index bbd949f..7895296 100644 +--- a/pengine/test10/ticket-primitive-21.exp ++++ b/pengine/test10/ticket-primitive-21.exp +@@ -1,53 +1,52 @@ + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/ticket-primitive-21.scores b/pengine/test10/ticket-primitive-21.scores +index d1147ca..ed3f3cd 100644 +--- a/pengine/test10/ticket-primitive-21.scores ++++ b/pengine/test10/ticket-primitive-21.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node2: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: -INFINITY +diff --git a/pengine/test10/ticket-primitive-21.summary b/pengine/test10/ticket-primitive-21.summary +index d3d02fe..9b91672 100644 +--- a/pengine/test10/ticket-primitive-21.summary ++++ b/pengine/test10/ticket-primitive-21.summary +@@ -5,15 +5,14 @@ Online: [ node1 node2 ] + rsc_stonith (stonith:null): Started node1 + rsc1 (ocf::pacemaker:Dummy): Started node2 + +- notice: Fencing node2: node is unclean + Transition Summary: +- * Fence (reboot) node2 ++ * Fence (reboot) node2 'deadman ticket was lost' + * Stop rsc1 (node2) + + Executing cluster transition: + * Fencing node2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + + Revised cluster status: +diff --git a/pengine/test10/ticket-primitive-9.exp b/pengine/test10/ticket-primitive-9.exp +index bbd949f..7895296 100644 +--- a/pengine/test10/ticket-primitive-9.exp ++++ b/pengine/test10/ticket-primitive-9.exp +@@ -1,53 +1,52 @@ + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +diff --git a/pengine/test10/ticket-primitive-9.scores b/pengine/test10/ticket-primitive-9.scores +index d1147ca..ed3f3cd 100644 +--- a/pengine/test10/ticket-primitive-9.scores ++++ b/pengine/test10/ticket-primitive-9.scores +@@ -1,4 +1,3 @@ +- notice: Fencing node2: node is unclean + Allocation scores: + native_color: rsc1 allocation score on node1: -INFINITY + native_color: rsc1 allocation score on node2: -INFINITY +diff --git a/pengine/test10/ticket-primitive-9.summary b/pengine/test10/ticket-primitive-9.summary +index d3d02fe..9b91672 100644 +--- a/pengine/test10/ticket-primitive-9.summary ++++ b/pengine/test10/ticket-primitive-9.summary +@@ -5,15 +5,14 @@ Online: [ node1 node2 ] + rsc_stonith (stonith:null): Started node1 + rsc1 (ocf::pacemaker:Dummy): Started node2 + +- notice: Fencing node2: node is unclean + Transition Summary: +- * Fence (reboot) node2 ++ * Fence (reboot) node2 'deadman ticket was lost' + * Stop rsc1 (node2) + + Executing cluster transition: + * Fencing node2 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + + Revised cluster status: +diff --git a/pengine/test10/unfence-definition.scores b/pengine/test10/unfence-definition.scores +index 3d5aa9f..d9fa9bb 100644 +--- a/pengine/test10/unfence-definition.scores ++++ b/pengine/test10/unfence-definition.scores +@@ -1,6 +1,3 @@ +- notice: Fencing virt-4: node is unclean +- notice: Unfencing virt-1: Device definition changed +- notice: Unfencing virt-3: Required by dlm:2 + Allocation scores: + clone_color: clvmd-clone allocation score on virt-1: 0 + clone_color: clvmd-clone allocation score on virt-2: 0 +diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary +index a31cc20..d776228 100644 +--- a/pengine/test10/unfence-definition.summary ++++ b/pengine/test10/unfence-definition.summary +@@ -11,13 +11,10 @@ Online: [ virt-1 virt-2 virt-3 ] + Started: [ virt-1 ] + Stopped: [ virt-2 virt-3 virt-4 ] + +- notice: Unfencing virt-1: Device definition changed +- notice: Unfencing virt-3: Required by dlm:2 +- notice: Fencing virt-4: node is unclean + Transition Summary: +- * Fence (reboot) virt-4 +- * Fence (on) virt-3 +- * Fence (on) virt-1 ++ * Fence (reboot) virt-4 'node is unclean' ++ * Fence (on) virt-3 'Required by dlm:2' ++ * Fence (on) virt-1 'Device definition changed' + * Restart fencing (Started virt-1) + * Restart dlm:0 (Started virt-1) + * Start dlm:2 (virt-3) +diff --git a/pengine/test10/unfence-parameters.scores b/pengine/test10/unfence-parameters.scores +index 5089910..d9fa9bb 100644 +--- a/pengine/test10/unfence-parameters.scores ++++ b/pengine/test10/unfence-parameters.scores +@@ -1,7 +1,3 @@ +- notice: Fencing virt-4: node is unclean +- notice: Unfencing virt-1: Device parameters changed (reload) +- notice: Unfencing virt-2: Device parameters changed (reload) +- notice: Unfencing virt-3: Device parameters changed (reload) + Allocation scores: + clone_color: clvmd-clone allocation score on virt-1: 0 + clone_color: clvmd-clone allocation score on virt-2: 0 +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +index 8effc63..e71eafc 100644 +--- a/pengine/test10/unfence-parameters.summary ++++ b/pengine/test10/unfence-parameters.summary +@@ -11,15 +11,11 @@ Online: [ virt-1 virt-2 virt-3 ] + Started: [ virt-1 ] + Stopped: [ virt-2 virt-3 virt-4 ] + +- notice: Unfencing virt-1: Device parameters changed (reload) +- notice: Unfencing virt-2: Device parameters changed (reload) +- notice: Unfencing virt-3: Device parameters changed (reload) +- notice: Fencing virt-4: node is unclean + Transition Summary: +- * Fence (reboot) virt-4 +- * Fence (on) virt-3 +- * Fence (on) virt-2 +- * Fence (on) virt-1 ++ * Fence (reboot) virt-4 'node is unclean' ++ * Fence (on) virt-3 'Device parameters changed (reload)' ++ * Fence (on) virt-2 'Device parameters changed (reload)' ++ * Fence (on) virt-1 'Device parameters changed (reload)' + * Restart fencing (Started virt-1) + * Restart dlm:0 (Started virt-1) + * Restart dlm:1 (Started virt-2) +diff --git a/pengine/test10/unfence-startup.scores b/pengine/test10/unfence-startup.scores +index 0172a9d..d9fa9bb 100644 +--- a/pengine/test10/unfence-startup.scores ++++ b/pengine/test10/unfence-startup.scores +@@ -1,5 +1,3 @@ +- notice: Fencing virt-4: node is unclean +- notice: Unfencing virt-3: Required by dlm:2 + Allocation scores: + clone_color: clvmd-clone allocation score on virt-1: 0 + clone_color: clvmd-clone allocation score on virt-2: 0 +diff --git a/pengine/test10/unfence-startup.summary b/pengine/test10/unfence-startup.summary +index 6a49f42..276358c 100644 +--- a/pengine/test10/unfence-startup.summary ++++ b/pengine/test10/unfence-startup.summary +@@ -11,11 +11,9 @@ Online: [ virt-1 virt-2 virt-3 ] + Started: [ virt-1 ] + Stopped: [ virt-2 virt-3 virt-4 ] + +- notice: Unfencing virt-3: Required by dlm:2 +- notice: Fencing virt-4: node is unclean + Transition Summary: +- * Fence (reboot) virt-4 +- * Fence (on) virt-3 ++ * Fence (reboot) virt-4 'node is unclean' ++ * Fence (on) virt-3 'Required by dlm:2' + * Start dlm:2 (virt-3) + * Start clvmd:1 (virt-2) + * Start clvmd:2 (virt-3) +diff --git a/pengine/test10/unrunnable-1.exp b/pengine/test10/unrunnable-1.exp +index 56fb4c0..d8f3b69 100644 +--- a/pengine/test10/unrunnable-1.exp ++++ b/pengine/test10/unrunnable-1.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -46,7 +46,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -91,7 +91,7 @@ + + + +- ++ + + + +@@ -100,19 +100,19 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-fail1.scores b/pengine/test10/whitebox-fail1.scores +index 02192a9..4fbbf29 100644 +--- a/pengine/test10/whitebox-fail1.scores ++++ b/pengine/test10/whitebox-fail1.scores +@@ -1,4 +1,3 @@ +- notice: Fencing lxc1: guest is unclean + Allocation scores: + clone_color: M-clone allocation score on 18node1: 0 + clone_color: M-clone allocation score on 18node2: 0 +diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary +index c7b5e19..14b1ee2 100644 +--- a/pengine/test10/whitebox-fail1.summary ++++ b/pengine/test10/whitebox-fail1.summary +@@ -13,9 +13,8 @@ Containers: [ lxc2:container2 ] + C (ocf::pacemaker:Dummy): Started lxc2 + D (ocf::pacemaker:Dummy): Started 18node1 + +- notice: Fencing lxc1: guest is unclean + Transition Summary: +- * Fence (reboot) lxc1 (resource: container1) ++ * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' + * Recover container1 (Started 18node2) + * Recover M:4 (Started lxc1) + * Recover B (Started lxc1) +diff --git a/pengine/test10/whitebox-fail2.scores b/pengine/test10/whitebox-fail2.scores +index 02192a9..4fbbf29 100644 +--- a/pengine/test10/whitebox-fail2.scores ++++ b/pengine/test10/whitebox-fail2.scores +@@ -1,4 +1,3 @@ +- notice: Fencing lxc1: guest is unclean + Allocation scores: + clone_color: M-clone allocation score on 18node1: 0 + clone_color: M-clone allocation score on 18node2: 0 +diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary +index 73e968e..06364fb 100644 +--- a/pengine/test10/whitebox-fail2.summary ++++ b/pengine/test10/whitebox-fail2.summary +@@ -13,9 +13,8 @@ Containers: [ lxc2:container2 ] + C (ocf::pacemaker:Dummy): Started lxc2 + D (ocf::pacemaker:Dummy): Started 18node1 + +- notice: Fencing lxc1: guest is unclean + Transition Summary: +- * Fence (reboot) lxc1 (resource: container1) ++ * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' + * Recover container1 (Started 18node2) + * Recover M:4 (Started lxc1) + * Recover B (Started lxc1) +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp +index e9390bf..9beb51d 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.exp ++++ b/pengine/test10/whitebox-imply-stop-on-fence.exp +@@ -1,36 +1,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -38,277 +38,277 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -320,89 +320,89 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -417,7 +417,7 @@ + + + +- ++ + + + +@@ -432,32 +432,19 @@ + + + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + + +@@ -467,43 +454,56 @@ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.scores b/pengine/test10/whitebox-imply-stop-on-fence.scores +index 5c3e325..e50f077 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.scores ++++ b/pengine/test10/whitebox-imply-stop-on-fence.scores +@@ -1,6 +1,3 @@ +- notice: Fencing kiff-01: node is unclean +- notice: Fencing lxc-01_kiff-01: guest is unclean +- notice: Fencing lxc-02_kiff-01: guest is unclean + Allocation scores: + clone_color: clvmd-clone allocation score on kiff-01: 0 + clone_color: clvmd-clone allocation score on kiff-02: 0 +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary +index a74bb72..9dab410 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.summary ++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary +@@ -24,13 +24,10 @@ Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ] + R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01 + +- notice: Fencing kiff-01: node is unclean +- notice: Fencing lxc-01_kiff-01: guest is unclean +- notice: Fencing lxc-02_kiff-01: guest is unclean + Transition Summary: +- * Fence (reboot) lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) +- * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) +- * Fence (reboot) kiff-01 ++ * Fence (reboot) lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) 'guest is unclean' ++ * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) 'guest is unclean' ++ * Fence (reboot) kiff-01 'peer is no longer part of the cluster' + * Move fence-kiff-02 (Started kiff-01 -> kiff-02) + * Stop dlm:0 (kiff-01) + * Stop clvmd:0 (kiff-01) +diff --git a/pengine/test10/whitebox-ms-ordering.scores b/pengine/test10/whitebox-ms-ordering.scores +index 72e6d25..f0e51f5 100644 +--- a/pengine/test10/whitebox-ms-ordering.scores ++++ b/pengine/test10/whitebox-ms-ordering.scores +@@ -1,5 +1,3 @@ +- notice: Fencing lxc1: guest is unclean +- notice: Fencing lxc2: guest is unclean + Allocation scores: + clone_color: lxc-ms-master allocation score on 18node1: 0 + clone_color: lxc-ms-master allocation score on 18node2: 0 +diff --git a/pengine/test10/whitebox-ms-ordering.summary b/pengine/test10/whitebox-ms-ordering.summary +index 41436f8..d8ff62c 100644 +--- a/pengine/test10/whitebox-ms-ordering.summary ++++ b/pengine/test10/whitebox-ms-ordering.summary +@@ -8,11 +8,9 @@ Online: [ 18node1 18node2 18node3 ] + Master/Slave Set: lxc-ms-master [lxc-ms] + Stopped: [ 18node1 18node2 18node3 ] + +- notice: Fencing lxc1: guest is unclean +- notice: Fencing lxc2: guest is unclean + Transition Summary: +- * Fence (reboot) lxc2 (resource: container2) +- * Fence (reboot) lxc1 (resource: container1) ++ * Fence (reboot) lxc2 (resource: container2) 'guest is unclean' ++ * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' + * Start container1 (18node1) + * Start container2 (18node1) + * Recover lxc-ms:0 (Master lxc1) +diff --git a/pengine/test10/whitebox-unexpectedly-running.scores b/pengine/test10/whitebox-unexpectedly-running.scores +index eb8a6f9..7f8a1d9 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.scores ++++ b/pengine/test10/whitebox-unexpectedly-running.scores +@@ -1,4 +1,3 @@ +- notice: Fencing remote1: guest is unclean + Allocation scores: + native_color: FAKE allocation score on 18builder: 0 + native_color: FAKE allocation score on remote1: -INFINITY +diff --git a/pengine/test10/whitebox-unexpectedly-running.summary b/pengine/test10/whitebox-unexpectedly-running.summary +index 3888aa2..0e69d3c 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.summary ++++ b/pengine/test10/whitebox-unexpectedly-running.summary +@@ -4,9 +4,8 @@ Online: [ 18builder ] + + FAKE (ocf::pacemaker:Dummy): FAILED 18builder + +- notice: Fencing remote1: guest is unclean + Transition Summary: +- * Fence (reboot) remote1 (resource: FAKE) ++ * Fence (reboot) remote1 (resource: FAKE) 'guest is unclean' + * Recover FAKE (Started 18builder) + * Start remote1 (18builder) + +-- +1.8.3.1 + + +From 681d290ca57849b72c2d3eac03ff74cbb63e2df8 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Mon, 31 Jul 2017 14:25:54 +1000 +Subject: [PATCH 05/12] Log: PE: Attempt to indicate why resources are changing + state + +--- + include/crm/pengine/internal.h | 7 +- + lib/pengine/utils.c | 81 +++++++++++++++++--- + pengine/native.c | 89 ++++++++++++---------- + pengine/test10/10-a-then-bm-b-move-a-clone.summary | 2 +- + .../11-a-then-bm-b-move-a-clone-starting.summary | 2 +- + pengine/test10/1484.summary | 2 +- + pengine/test10/1494.summary | 2 +- + .../test10/5-am-then-bm-a-not-migratable.summary | 2 +- + pengine/test10/594.summary | 4 +- + pengine/test10/662.summary | 2 +- + .../7-migrate-group-one-unmigratable.summary | 2 +- + pengine/test10/764.summary | 10 +-- + pengine/test10/797.summary | 12 +-- + pengine/test10/829.summary | 2 +- + .../9-am-then-bm-b-migrating-a-stopping.summary | 4 +- + pengine/test10/994-2.summary | 2 +- + pengine/test10/anti-colocation-order.summary | 4 +- + pengine/test10/asymmetrical-order-move.summary | 2 +- + pengine/test10/bug-1572-1.summary | 8 +- + pengine/test10/bug-1572-2.summary | 8 +- + pengine/test10/bug-1573.summary | 2 +- + pengine/test10/bug-1718.summary | 4 +- + pengine/test10/bug-5014-A-stop-B-started.summary | 2 +- + .../test10/bug-5014-A-stopped-B-stopped.summary | 2 +- + .../test10/bug-5014-CLONE-A-stop-B-started.summary | 2 +- + .../test10/bug-5014-CthenAthenB-C-stopped.summary | 4 +- + .../bug-5014-GROUP-A-stopped-B-started.summary | 2 +- + .../bug-5014-GROUP-A-stopped-B-stopped.summary | 2 +- + .../bug-5014-ordered-set-symmetrical-false.summary | 2 +- + .../bug-5014-ordered-set-symmetrical-true.summary | 4 +- + pengine/test10/bug-5028-bottom.summary | 2 +- + pengine/test10/bug-5140-require-all-false.summary | 6 +- + pengine/test10/bug-5186-partial-migrate.summary | 6 +- + pengine/test10/bug-cl-5170.summary | 4 +- + pengine/test10/bug-cl-5212.summary | 8 +- + pengine/test10/bug-cl-5247.summary | 2 +- + pengine/test10/bug-lf-2153.summary | 4 +- + pengine/test10/bug-lf-2171.summary | 8 +- + pengine/test10/bug-lf-2361.summary | 4 +- + pengine/test10/bug-lf-2422.summary | 24 +++--- + pengine/test10/bug-lf-2453.summary | 10 +-- + pengine/test10/bug-lf-2508.summary | 12 +-- + pengine/test10/bug-lf-2551.summary | 12 +-- + pengine/test10/bug-lf-2574.summary | 2 +- + pengine/test10/bug-lf-2619.summary | 2 +- + pengine/test10/bug-n-385265.summary | 2 +- + pengine/test10/bug-rh-1097457.summary | 8 +- + pengine/test10/bug-suse-707150.summary | 2 +- + .../test10/bundle-order-partial-start-2.summary | 4 +- + pengine/test10/bundle-order-partial-stop.summary | 34 ++++----- + pengine/test10/bundle-order-startup-clone.summary | 12 +-- + pengine/test10/bundle-order-stop-clone.summary | 8 +- + pengine/test10/bundle-order-stop.summary | 34 ++++----- + pengine/test10/clone-anon-dup.summary | 2 +- + pengine/test10/clone-anon-failcount.summary | 6 +- + pengine/test10/clone-interleave-2.summary | 4 +- + pengine/test10/clone-interleave-3.summary | 4 +- + pengine/test10/clone-max-zero.summary | 12 +-- + pengine/test10/clone-require-all-2.summary | 8 +- + pengine/test10/clone-require-all-3.summary | 8 +- + pengine/test10/clone-require-all-4.summary | 2 +- + pengine/test10/clone-require-all-6.summary | 4 +- + .../test10/clone_min_interleave_start_one.summary | 12 +-- + .../test10/clone_min_interleave_stop_one.summary | 2 +- + .../test10/clone_min_interleave_stop_two.summary | 16 ++-- + pengine/test10/clone_min_start_one.summary | 2 +- + pengine/test10/clone_min_stop_all.summary | 8 +- + pengine/test10/clone_min_stop_one.summary | 2 +- + pengine/test10/clone_min_stop_two.summary | 6 +- + pengine/test10/cloned-group-stop.summary | 32 ++++---- + pengine/test10/cloned-group.summary | 6 +- + pengine/test10/cloned_start_one.summary | 4 +- + pengine/test10/cloned_start_two.summary | 2 +- + pengine/test10/cloned_stop_one.summary | 4 +- + pengine/test10/cloned_stop_two.summary | 8 +- + .../colocation_constraint_stops_slave.summary | 2 +- + pengine/test10/complex_enforce_colo.summary | 70 ++++++++--------- + pengine/test10/container-2.summary | 2 +- + pengine/test10/container-3.summary | 2 +- + pengine/test10/container-group-2.summary | 2 +- + pengine/test10/enforce-colo1.summary | 8 +- + pengine/test10/group-fail.summary | 4 +- + pengine/test10/group-unmanaged-stopped.summary | 2 +- + pengine/test10/group10.summary | 4 +- + pengine/test10/group11.summary | 4 +- + pengine/test10/group13.summary | 2 +- + pengine/test10/group14.summary | 32 ++++---- + pengine/test10/group9.summary | 2 +- + pengine/test10/inc10.summary | 4 +- + pengine/test10/inc12.summary | 36 ++++----- + pengine/test10/inc2.summary | 2 +- + pengine/test10/inc3.summary | 2 +- + pengine/test10/inc4.summary | 2 +- + pengine/test10/inc6.summary | 4 +- + pengine/test10/inc9.summary | 6 +- + pengine/test10/interleave-pseudo-stop.summary | 8 +- + pengine/test10/interleave-restart.summary | 4 +- + pengine/test10/interleave-stop.summary | 8 +- + pengine/test10/master-7.summary | 4 +- + pengine/test10/master-8.summary | 2 +- + pengine/test10/master-9.summary | 24 +++--- + pengine/test10/master-asymmetrical-order.summary | 2 +- + pengine/test10/master-failed-demote-2.summary | 2 +- + pengine/test10/master-failed-demote.summary | 2 +- + pengine/test10/master-pseudo.summary | 2 +- + pengine/test10/master-stop.summary | 2 +- + pengine/test10/migrate-5.summary | 2 +- + pengine/test10/migrate-both-vms.summary | 20 ++--- + pengine/test10/migrate-fencing.summary | 4 +- + pengine/test10/migrate-shutdown.summary | 16 ++-- + pengine/test10/migrate-stop-complex.summary | 4 +- + pengine/test10/migrate-stop-start-complex.summary | 2 +- + pengine/test10/migrate-stop.summary | 2 +- + pengine/test10/monitor-recovery.summary | 2 +- + pengine/test10/notify-0.summary | 2 +- + pengine/test10/notify-1.summary | 2 +- + pengine/test10/notify-2.summary | 2 +- + pengine/test10/notify-3.summary | 2 +- + pengine/test10/novell-252693.summary | 10 +-- + pengine/test10/one-or-more-1.summary | 6 +- + pengine/test10/one-or-more-3.summary | 2 +- + pengine/test10/one-or-more-6.summary | 2 +- + pengine/test10/one-or-more-7.summary | 2 +- + pengine/test10/order-mandatory.summary | 4 +- + pengine/test10/order-required.summary | 4 +- + pengine/test10/order-wrong-kind.summary | 2 +- + pengine/test10/order7.summary | 2 +- + .../test10/order_constraint_stops_master.summary | 4 +- + .../test10/order_constraint_stops_slave.summary | 4 +- + pengine/test10/ordered-set-basic-startup.summary | 6 +- + pengine/test10/ordered-set-natural.summary | 12 +-- + pengine/test10/probe-2.summary | 12 +-- + pengine/test10/quorum-2.summary | 2 +- + pengine/test10/quorum-3.summary | 6 +- + pengine/test10/rec-node-10.summary | 6 +- + pengine/test10/rec-node-13.summary | 2 +- + pengine/test10/rec-node-8.summary | 4 +- + pengine/test10/rec-node-9.summary | 4 +- + pengine/test10/rec-rsc-9.summary | 4 +- + pengine/test10/remote-disable.summary | 2 +- + pengine/test10/remote-orphaned.summary | 4 +- + pengine/test10/remote-recover-all.summary | 10 +-- + pengine/test10/remote-recover-connection.summary | 4 +- + pengine/test10/remote-recover-no-resources.summary | 8 +- + pengine/test10/remote-recover-unknown.summary | 10 +-- + pengine/test10/remote-recovery.summary | 4 +- + pengine/test10/rsc-sets-clone.summary | 2 +- + pengine/test10/stop-failure-no-quorum.summary | 10 +-- + pengine/test10/stop-failure-with-fencing.summary | 10 +-- + pengine/test10/ticket-clone-14.summary | 4 +- + pengine/test10/ticket-clone-15.summary | 4 +- + pengine/test10/ticket-clone-17.summary | 4 +- + pengine/test10/ticket-clone-18.summary | 4 +- + pengine/test10/ticket-clone-20.summary | 4 +- + pengine/test10/ticket-clone-21.summary | 4 +- + pengine/test10/ticket-clone-23.summary | 4 +- + pengine/test10/ticket-clone-3.summary | 4 +- + pengine/test10/ticket-clone-6.summary | 4 +- + pengine/test10/ticket-clone-9.summary | 4 +- + pengine/test10/ticket-group-14.summary | 4 +- + pengine/test10/ticket-group-15.summary | 4 +- + pengine/test10/ticket-group-17.summary | 4 +- + pengine/test10/ticket-group-18.summary | 4 +- + pengine/test10/ticket-group-20.summary | 4 +- + pengine/test10/ticket-group-21.summary | 4 +- + pengine/test10/ticket-group-23.summary | 4 +- + pengine/test10/ticket-group-3.summary | 4 +- + pengine/test10/ticket-group-6.summary | 4 +- + pengine/test10/ticket-group-9.summary | 4 +- + pengine/test10/ticket-master-14.summary | 2 +- + pengine/test10/ticket-master-15.summary | 2 +- + pengine/test10/ticket-master-3.summary | 2 +- + pengine/test10/ticket-rsc-sets-10.summary | 8 +- + pengine/test10/ticket-rsc-sets-12.summary | 4 +- + pengine/test10/ticket-rsc-sets-13.summary | 8 +- + pengine/test10/ticket-rsc-sets-14.summary | 8 +- + pengine/test10/ticket-rsc-sets-3.summary | 8 +- + pengine/test10/ticket-rsc-sets-7.summary | 8 +- + pengine/test10/ticket-rsc-sets-9.summary | 8 +- + pengine/test10/unfence-definition.summary | 4 +- + pengine/test10/unfence-parameters.summary | 6 +- + pengine/test10/unmanaged-block-restart.summary | 4 +- + pengine/test10/unmanaged-stop-1.summary | 2 +- + pengine/test10/unmanaged-stop-2.summary | 2 +- + pengine/test10/unmanaged-stop-3.summary | 2 +- + pengine/test10/unmanaged-stop-4.summary | 2 +- + pengine/test10/unrunnable-1.summary | 18 ++--- + pengine/test10/unrunnable-2.summary | 2 +- + pengine/test10/utilization-order2.summary | 2 +- + pengine/test10/utilization-order4.summary | 8 +- + pengine/test10/whitebox-fail1.summary | 2 +- + .../test10/whitebox-imply-stop-on-fence.summary | 6 +- + pengine/test10/whitebox-move.summary | 4 +- + pengine/test10/whitebox-ms-ordering-move.summary | 2 +- + pengine/test10/whitebox-orphan-ms.summary | 8 +- + pengine/test10/whitebox-orphaned.summary | 6 +- + pengine/test10/whitebox-stop.summary | 6 +- + 197 files changed, 704 insertions(+), 633 deletions(-) + +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index 99e2055..5db90aa 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -277,8 +277,11 @@ action_t *pe_fence_op(node_t * node, const char *op, bool optional, const char * + void trigger_unfencing( + resource_t * rsc, node_t *node, const char *reason, action_t *dependency, pe_working_set_t * data_set); + +-void pe_action_required_worker(pe_action_t *action, const char *reason, const char *function, long line); +-#define pe_action_required(action, reason) pe_action_required_worker(action, reason, __FUNCTION__, __LINE__) ++void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite); ++void pe_action_set_flag_reason(const char *function, long line, pe_action_t *action, pe_action_t *reason, const char *text, enum pe_action_flags flags, bool overwrite); ++ ++#define pe_action_required(action, reason, text) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, text, pe_action_optional, FALSE) ++#define pe_action_implies(action, reason, flag) pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, reason, NULL, flag, FALSE) + + void set_bit_recursive(resource_t * rsc, unsigned long long flag); + void clear_bit_recursive(resource_t * rsc, unsigned long long flag); +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index a53be6a..bc34323 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -500,7 +503,7 @@ custom_action(resource_t * rsc, char *key, const char *task, + + } else if (action->node == NULL) { + pe_rsc_trace(rsc, "Unset runnable on %s", action->uuid); +- pe_clear_action_bit(action, pe_action_runnable); ++ pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "node availability", pe_action_runnable, TRUE); + + } else if (is_not_set(rsc->flags, pe_rsc_managed) + && g_hash_table_lookup(action->meta, XML_LRM_ATTR_INTERVAL) == NULL) { +@@ -527,6 +530,7 @@ custom_action(resource_t * rsc, char *key, const char *task, + + } else if (action->needs == rsc_req_nothing) { + pe_rsc_trace(rsc, "Action %s does not require anything", action->uuid); ++ free(action->reason); action->reason = NULL; + pe_set_action_bit(action, pe_action_runnable); + #if 0 + /* +@@ -540,20 +543,21 @@ custom_action(resource_t * rsc, char *key, const char *task, + #endif + } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE + && data_set->no_quorum_policy == no_quorum_stop) { +- pe_clear_action_bit(action, pe_action_runnable); ++ pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "no quorum", pe_action_runnable, TRUE); + crm_debug("%s\t%s (cancelled : quorum)", action->node->details->uname, action->uuid); + + } else if (is_set(data_set->flags, pe_flag_have_quorum) == FALSE + && data_set->no_quorum_policy == no_quorum_freeze) { + pe_rsc_trace(rsc, "Check resource is already active: %s %s %s %s", rsc->id, action->uuid, role2text(rsc->next_role), role2text(rsc->role)); + if (rsc->fns->active(rsc, TRUE) == FALSE || rsc->next_role > rsc->role) { +- pe_clear_action_bit(action, pe_action_runnable); ++ pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, "quorum freeze", pe_action_runnable, TRUE); + pe_rsc_debug(rsc, "%s\t%s (cancelled : quorum freeze)", + action->node->details->uname, action->uuid); + } + + } else { + pe_rsc_trace(rsc, "Action %s is runnable", action->uuid); ++ free(action->reason); action->reason = NULL; + pe_set_action_bit(action, pe_action_runnable); + } + +@@ -1880,7 +1882,7 @@ rsc_action_digest(resource_t * rsc, const char *task, const char *key, + data->digest_secure_calc = calculate_operation_digest(data->params_secure, op_version); + } + +- if(crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) != NULL) { ++ if(xml_op && crm_element_value(xml_op, XML_LRM_ATTR_RESTART_DIGEST) != NULL) { + data->params_restart = copy_xml(data->params_all); + if (restart_list) { + filter_parameters(data->params_restart, restart_list, TRUE); +@@ -2108,7 +2110,7 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + } + + if(optional == FALSE && pe_can_fence(data_set, node)) { +- pe_action_required(stonith_op, reason); ++ pe_action_required(stonith_op, NULL, reason); + } else if(reason && stonith_op->reason == NULL) { + stonith_op->reason = strdup(reason); + } +@@ -2187,12 +2189,73 @@ add_tag_ref(GHashTable * tags, const char * tag_name, const char * obj_ref) + return TRUE; + } + +-void pe_action_required_worker(pe_action_t *action, const char *reason, const char *function, long line) ++void pe_action_set_flag_reason(const char *function, long line, ++ pe_action_t *action, pe_action_t *reason, const char *text, ++ enum pe_action_flags flags, bool overwrite) ++{ ++ bool unset = FALSE; ++ bool update = FALSE; ++ const char *change = NULL; ++ ++ if(is_set(flags, pe_action_runnable)) { ++ unset = TRUE; ++ change = "unrunnable"; ++ } else if(is_set(flags, pe_action_optional)) { ++ unset = TRUE; ++ change = "required"; ++ } else if(is_set(flags, pe_action_failure_is_fatal)) { ++ change = "fatally failed"; ++ } else if(is_set(flags, pe_action_migrate_runnable)) { ++ unset = TRUE; ++ overwrite = TRUE; ++ change = "unrunnable"; ++ } else if(is_set(flags, pe_action_dangle)) { ++ change = "dangling"; ++ } else if(is_set(flags, pe_action_requires_any)) { ++ change = "required"; ++ } else { ++ crm_err("Unknown flag change to %s by %s: 0x%.16x", flags, action->uuid, reason->uuid); ++ } ++ ++ if(unset) { ++ if(is_set(action->flags, flags)) { ++ action->flags = crm_clear_bit(function, line, action->uuid, action->flags, flags); ++ update = TRUE; ++ } ++ ++ } else { ++ if(is_not_set(action->flags, flags)) { ++ action->flags = crm_set_bit(function, line, action->uuid, action->flags, flags); ++ update = TRUE; ++ } ++ } ++ ++ if((change && update) || text) { ++ char *reason_text = NULL; ++ if(reason == NULL) { ++ pe_action_set_reason(action, text, overwrite); ++ ++ } else if(reason->rsc == NULL) { ++ reason_text = crm_strdup_printf("%s %s%c %s", change, reason->task, text?':':0, text?text:""); ++ } else { ++ reason_text = crm_strdup_printf("%s %s %s%c %s", change, reason->rsc->id, reason->task, text?':':0, text?text:"NA"); ++ } ++ ++ if(reason_text && action->rsc != reason->rsc) { ++ pe_action_set_reason(action, reason_text, overwrite); ++ } ++ free(reason_text); ++ } ++ } ++ ++void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrite) + { +- if(is_set(action->flags, pe_action_optional)) { +- action->flags = crm_clear_bit(function, line, action->uuid, action->flags, pe_action_optional); +- if(action->reason == NULL) { ++ if(action->reason == NULL || overwrite) { ++ free(action->reason); ++ if(reason) { + action->reason = strdup(reason); ++ } else { ++ action->reason = NULL; + } + } + } +diff --git a/pengine/native.c b/pengine/native.c +index 16c9d5a..63666fd 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -1886,8 +1886,9 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + } else if (!(first->flags & pe_action_runnable)) { + /* prevent 'then' action from happening if 'first' is not runnable and + * 'then' has not yet occurred. */ +- pe_clear_action_bit(then, pe_action_runnable); +- pe_clear_action_bit(then, pe_action_optional); ++ pe_action_implies(then, first, pe_action_optional); ++ pe_action_implies(then, first, pe_action_runnable); ++ + pe_rsc_trace(then->rsc, "Unset optional and runnable on %s", then->uuid); + } else { + /* ignore... then is allowed to start/stop if it wants to. */ +@@ -1895,10 +1896,10 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + } + + if (type & pe_order_implies_first) { +- if ((filter & pe_action_optional) && (flags & pe_action_optional) == 0) { ++ if (is_set(filter, pe_action_optional) && is_not_set(flags /* Should be then_flags? */, pe_action_optional)) { ++ // Needs is_set(first_flags, pe_action_optional) too? + pe_rsc_trace(first->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid); +- +- pe_clear_action_bit(first, pe_action_optional); ++ pe_action_implies(first, then, pe_action_optional); + } + + if (is_set(flags, pe_action_migrate_runnable) && +@@ -1907,7 +1908,7 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + + pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", + first->uuid, then->uuid); +- pe_clear_action_bit(first, pe_action_migrate_runnable); ++ pe_action_implies(first, then, pe_action_migrate_runnable); + } + } + +@@ -1915,13 +1916,13 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + if ((filter & pe_action_optional) && + ((then->flags & pe_action_optional) == FALSE) && + then->rsc && (then->rsc->role == RSC_ROLE_MASTER)) { +- pe_clear_action_bit(first, pe_action_optional); ++ pe_action_implies(first, then, pe_action_optional); + + if (is_set(first->flags, pe_action_migrate_runnable) && + is_set(then->flags, pe_action_migrate_runnable) == FALSE) { + + pe_rsc_trace(first->rsc, "Unset migrate runnable on %s because of %s", first->uuid, then->uuid); +- pe_clear_action_bit(first, pe_action_migrate_runnable); ++ pe_action_implies(first, then, pe_action_migrate_runnable); + } + pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", first->uuid, then->uuid); + } +@@ -1934,13 +1935,12 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + ((then->flags & pe_action_runnable) == FALSE)) { + + pe_rsc_trace(then->rsc, "Unset runnable on %s because %s is neither runnable or migratable", first->uuid, then->uuid); +- pe_clear_action_bit(first, pe_action_runnable); ++ pe_action_implies(first, then, pe_action_runnable); + } + + if ((then->flags & pe_action_optional) == 0) { +- + pe_rsc_trace(then->rsc, "Unset optional on %s because %s is not optional", first->uuid, then->uuid); +- pe_clear_action_bit(first, pe_action_optional); ++ pe_action_implies(first, then, pe_action_optional); + } + } + +@@ -1948,7 +1948,7 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + && is_set(filter, pe_action_optional)) { + + if ((first->flags & pe_action_runnable) == FALSE) { +- pe_clear_action_bit(then, pe_action_migrate_runnable); ++ pe_action_implies(then, first, pe_action_migrate_runnable); + pe_clear_action_bit(then, pe_action_pseudo); + pe_rsc_trace(then->rsc, "Unset pseudo on %s because %s is not runnable", then->uuid, first->uuid); + } +@@ -1960,8 +1960,8 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + && is_set(then->flags, pe_action_runnable) + && is_set(flags, pe_action_runnable) == FALSE) { + pe_rsc_trace(then->rsc, "Unset runnable on %s because of %s", then->uuid, first->uuid); +- pe_clear_action_bit(then, pe_action_runnable); +- pe_clear_action_bit(then, pe_action_migrate_runnable); ++ pe_action_implies(then, first, pe_action_runnable); ++ pe_action_implies(then, first, pe_action_migrate_runnable); + } + + if (is_set(type, pe_order_implies_then) +@@ -1972,7 +1972,7 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + /* in this case, treat migrate_runnable as if first is optional */ + if (is_set(first->flags, pe_action_migrate_runnable) == FALSE) { + pe_rsc_trace(then->rsc, "Unset optional on %s because of %s", then->uuid, first->uuid); +- pe_clear_action_bit(then, pe_action_optional); ++ pe_action_implies(then, first, pe_action_optional); + } + } + +@@ -1996,14 +1996,14 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + if (is_set(first->flags, pe_action_runnable) + || is_not_set(then->flags, pe_action_optional)) { + pe_rsc_trace(first->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); +- pe_clear_action_bit(first, pe_action_optional); ++ pe_action_implies(first, then, pe_action_optional); + } + } + + if (reason && is_not_set(first->flags, pe_action_optional) + && is_not_set(first->flags, pe_action_runnable)) { + pe_rsc_trace(then->rsc, "Handling %s: %s -> %s", reason, first->uuid, then->uuid); +- pe_clear_action_bit(then, pe_action_runnable); ++ pe_action_implies(then, first, pe_action_runnable); + } + + if (reason && +@@ -2011,7 +2011,7 @@ native_update_actions(action_t * first, action_t * then, node_t * node, enum pe_ + is_set(first->flags, pe_action_migrate_runnable) && + is_not_set(then->flags, pe_action_migrate_runnable)) { + +- pe_clear_action_bit(first, pe_action_migrate_runnable); ++ pe_action_implies(first, then, pe_action_migrate_runnable); + } + + } +@@ -2090,6 +2090,7 @@ native_rsc_location(resource_t * rsc, rsc_to_node_t * constraint) + } else { + other_node = node_copy(node); + ++ pe_rsc_trace(rsc, "%s: %d (insert %d)", other_node->details->uname, other_node->weight, constraint->discover_mode); + g_hash_table_insert(rsc->allowed_nodes, (gpointer) other_node->details->id, other_node); + } + +@@ -2130,12 +2131,16 @@ native_expand(resource_t * rsc, pe_working_set_t * data_set) + } + } + +-#define log_change(fmt, args...) do { \ +- if(terminal) { \ +- printf(" * "fmt"\n", ##args); \ +- } else { \ +- crm_notice(fmt, ##args); \ +- } \ ++#define log_change(a, fmt, args...) do { \ ++ if(a && a->reason && terminal) { \ ++ printf(" * "fmt" \tdue to %s\n", ##args, a->reason); \ ++ } else if(a && a->reason) { \ ++ crm_notice(fmt" \tdue to %s", ##args, a->reason); \ ++ } else if(terminal) { \ ++ printf(" * "fmt"\n", ##args); \ ++ } else { \ ++ crm_notice(fmt, ##args); \ ++ } \ + } while(0) + + #define STOP_SANITY_ASSERT(lineno) do { \ +@@ -2264,34 +2269,34 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + CRM_CHECK(next != NULL,); + if (next == NULL) { + } else if (migrate_to && is_set(migrate_to->flags, pe_action_runnable) && current) { +- log_change("Migrate %s\t(%s %s -> %s)", ++ log_change(start, "Migrate %s\t(%s %s -> %s)", + rsc->id, role2text(rsc->role), current->details->uname, + next->details->uname); + + } else if (is_set(rsc->flags, pe_rsc_reload)) { +- log_change("Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); ++ log_change(start, "Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); + + } else if (start == NULL || is_set(start->flags, pe_action_optional)) { + pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, role2text(rsc->role), + next->details->uname); + + } else if (start && is_set(start->flags, pe_action_runnable) == FALSE) { +- log_change("Stop %s\t(%s %s%s)", rsc->id, role2text(rsc->role), current?current->details->uname:"N/A", ++ log_change(start, "Stop %s\t(%s %s%s)", rsc->id, role2text(rsc->role), current?current->details->uname:"N/A", + stop && is_not_set(stop->flags, pe_action_runnable) ? " - blocked" : ""); + STOP_SANITY_ASSERT(__LINE__); + + } else if (moving && current) { +- log_change("%s %s\t(%s %s -> %s)", ++ log_change(stop, "%s %s\t(%s %s -> %s)", + is_set(rsc->flags, pe_rsc_failed) ? "Recover" : "Move ", + rsc->id, role2text(rsc->role), + current->details->uname, next->details->uname); + + } else if (is_set(rsc->flags, pe_rsc_failed)) { +- log_change("Recover %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); ++ log_change(stop, "Recover %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); + STOP_SANITY_ASSERT(__LINE__); + + } else { +- log_change("Restart %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); ++ log_change(start, "Restart %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); + /* STOP_SANITY_ASSERT(__LINE__); False positive for migrate-fail-7 */ + } + +@@ -2308,7 +2313,7 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + allowed = TRUE; + } + +- log_change("Demote %s\t(%s -> %s %s%s)", ++ log_change(demote, "Demote %s\t(%s -> %s %s%s)", + rsc->id, + role2text(rsc->role), + role2text(rsc->next_role), +@@ -2317,16 +2322,16 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + if (stop != NULL && is_not_set(stop->flags, pe_action_optional) + && rsc->next_role > RSC_ROLE_STOPPED && moving == FALSE) { + if (is_set(rsc->flags, pe_rsc_failed)) { +- log_change("Recover %s\t(%s %s)", ++ log_change(stop, "Recover %s\t(%s %s)", + rsc->id, role2text(rsc->role), next->details->uname); + STOP_SANITY_ASSERT(__LINE__); + + } else if (is_set(rsc->flags, pe_rsc_reload)) { +- log_change("Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), ++ log_change(start, "Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), + next->details->uname); + + } else { +- log_change("Restart %s\t(%s %s)", ++ log_change(start, "Restart %s\t(%s %s)", + rsc->id, role2text(rsc->next_role), next->details->uname); + STOP_SANITY_ASSERT(__LINE__); + } +@@ -2355,15 +2360,15 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + allowed = TRUE; + } + +- log_change("Stop %s\t(%s%s)", rsc->id, node->details->uname, +- allowed ? "" : " - blocked"); ++ log_change(start, "Stop %s\t(%s%s) %s", rsc->id, node->details->uname, ++ allowed ? "" : " - blocked", stop->reason?stop->reason:""); + } + + free(key); + } + + if (moving) { +- log_change("Move %s\t(%s %s -> %s)", ++ log_change(stop, "Move %s\t(%s %s -> %s)", + rsc->id, role2text(rsc->next_role), current->details->uname, + next->details->uname); + STOP_SANITY_ASSERT(__LINE__); +@@ -2378,7 +2383,7 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + + CRM_CHECK(next != NULL,); + if (next != NULL) { +- log_change("Start %s\t(%s%s)", rsc->id, next->details->uname, ++ log_change(start, "Start %s\t(%s%s)", rsc->id, next->details->uname, + allowed ? "" : " - blocked"); + } + if (allowed == FALSE) { +@@ -2393,17 +2398,17 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + if (stop != NULL && is_not_set(stop->flags, pe_action_optional) + && rsc->role > RSC_ROLE_STOPPED) { + if (is_set(rsc->flags, pe_rsc_failed)) { +- log_change("Recover %s\t(%s %s)", ++ log_change(stop, "Recover %s\t(%s %s)", + rsc->id, role2text(rsc->role), next?next->details->uname:NULL); + STOP_SANITY_ASSERT(__LINE__); + + } else if (is_set(rsc->flags, pe_rsc_reload)) { +- log_change("Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), ++ log_change(start, "Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), + next?next->details->uname:NULL); + STOP_SANITY_ASSERT(__LINE__); + + } else { +- log_change("Restart %s\t(%s %s)", ++ log_change(start, "Restart %s\t(%s %s)", + rsc->id, role2text(rsc->role), next?next->details->uname:NULL); + STOP_SANITY_ASSERT(__LINE__); + } +@@ -2413,7 +2418,7 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + allowed = TRUE; + } + +- log_change("Promote %s\t(%s -> %s %s%s)", ++ log_change(promote, "Promote %s\t(%s -> %s %s%s)", + rsc->id, + role2text(rsc->role), + role2text(rsc->next_role), +diff --git a/pengine/test10/10-a-then-bm-b-move-a-clone.summary b/pengine/test10/10-a-then-bm-b-move-a-clone.summary +index 07b4d2b..b39963f 100644 +--- a/pengine/test10/10-a-then-bm-b-move-a-clone.summary ++++ b/pengine/test10/10-a-then-bm-b-move-a-clone.summary +@@ -8,7 +8,7 @@ Online: [ f20node2 ] + vm (ocf::heartbeat:Dummy): Started f20node1 + + Transition Summary: +- * Stop myclone:1 (f20node1) ++ * Stop myclone:1 (f20node1) due to node availability + * Migrate vm (Started f20node1 -> f20node2) + + Executing cluster transition: +diff --git a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.summary b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.summary +index 4b8e1d0..498cd2b 100644 +--- a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.summary ++++ b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.summary +@@ -10,7 +10,7 @@ Online: [ f20node2 ] + + Transition Summary: + * Move myclone:0 (Started f20node1 -> f20node2) +- * Move vm (Started f20node1 -> f20node2) ++ * Move vm (Started f20node1 -> f20node2) due to unrunnable myclone-clone stop + + Executing cluster transition: + * Resource action: myclone monitor on f20node2 +diff --git a/pengine/test10/1484.summary b/pengine/test10/1484.summary +index 52cfebe..4b5d8a6 100644 +--- a/pengine/test10/1484.summary ++++ b/pengine/test10/1484.summary +@@ -6,7 +6,7 @@ OFFLINE: [ hb3 ] + the-future-of-vaj (ocf::heartbeat:Dummy): FAILED hb2 + + Transition Summary: +- * Stop the-future-of-vaj (hb2) ++ * Stop the-future-of-vaj (hb2) due to node availability + + Executing cluster transition: + * Resource action: the-future-of-vaj stop on hb2 +diff --git a/pengine/test10/1494.summary b/pengine/test10/1494.summary +index 29c3cd6..be13fdd 100644 +--- a/pengine/test10/1494.summary ++++ b/pengine/test10/1494.summary +@@ -8,7 +8,7 @@ OFFLINE: [ hb3 ] + ima_rscid:1 (ocf::heartbeat:Dummy): Started hb2 + + Transition Summary: +- * Stop ima_rscid:0 (hb1) ++ * Stop ima_rscid:0 (hb1) due to node availability + + Executing cluster transition: + * Pseudo action: ima_cloneid_stop_0 +diff --git a/pengine/test10/5-am-then-bm-a-not-migratable.summary b/pengine/test10/5-am-then-bm-a-not-migratable.summary +index bc73910..7e95dbb 100644 +--- a/pengine/test10/5-am-then-bm-a-not-migratable.summary ++++ b/pengine/test10/5-am-then-bm-a-not-migratable.summary +@@ -7,7 +7,7 @@ Online: [ 18node1 18node2 18node3 ] + + Transition Summary: + * Move A (Started 18node1 -> 18node2) +- * Move B (Started 18node2 -> 18node1) ++ * Move B (Started 18node2 -> 18node1) due to unrunnable A stop + + Executing cluster transition: + * Resource action: B stop on 18node2 +diff --git a/pengine/test10/594.summary b/pengine/test10/594.summary +index 44ed113..de1d179 100644 +--- a/pengine/test10/594.summary ++++ b/pengine/test10/594.summary +@@ -17,8 +17,8 @@ Transition Summary: + * Fence (reboot) hadev3 'peer is no longer part of the cluster' + * Move DcIPaddr (Started hadev2 -> hadev1) + * Move rsc_hadev2 (Started hadev2 -> hadev1) +- * Stop child_DoFencing:0 (hadev2) +- * Stop child_DoFencing:2 (hadev1) ++ * Stop child_DoFencing:0 (hadev2) due to node availability ++ * Stop child_DoFencing:2 (hadev1) due to node availability + + Executing cluster transition: + * Resource action: DcIPaddr monitor on hadev1 +diff --git a/pengine/test10/662.summary b/pengine/test10/662.summary +index 1726f35..4a9d911 100644 +--- a/pengine/test10/662.summary ++++ b/pengine/test10/662.summary +@@ -16,7 +16,7 @@ Online: [ c001n02 c001n03 c001n04 c001n09 ] + Transition Summary: + * Shutdown c001n02 + * Move rsc_c001n02 (Started c001n02 -> c001n03) +- * Stop child_DoFencing:0 (c001n02) ++ * Stop child_DoFencing:0 (c001n02) due to node availability + + Executing cluster transition: + * Resource action: DcIPaddr monitor on c001n04 +diff --git a/pengine/test10/7-migrate-group-one-unmigratable.summary b/pengine/test10/7-migrate-group-one-unmigratable.summary +index ab9bcd9..cf1b370 100644 +--- a/pengine/test10/7-migrate-group-one-unmigratable.summary ++++ b/pengine/test10/7-migrate-group-one-unmigratable.summary +@@ -10,7 +10,7 @@ Online: [ 18node1 18node2 18node3 ] + Transition Summary: + * Migrate A (Started 18node1 -> 18node2) + * Move B (Started 18node1 -> 18node2) +- * Move C (Started 18node1 -> 18node2) ++ * Move C (Started 18node1 -> 18node2) due to unrunnable B stop + + Executing cluster transition: + * Pseudo action: thegroup_stop_0 +diff --git a/pengine/test10/764.summary b/pengine/test10/764.summary +index 2d43eae..0d5c612 100644 +--- a/pengine/test10/764.summary ++++ b/pengine/test10/764.summary +@@ -15,11 +15,11 @@ OFFLINE: [ posic042 posic044 ] + child_DoFencing:3 (stonith:ssh): Stopped + + Transition Summary: +- * Stop DcIPaddr (Started posic043) +- * Stop rsc_posic041 (Started posic041) +- * Stop rsc_posic042 (Started posic041) +- * Stop rsc_posic043 (Started posic043) +- * Stop rsc_posic044 (Started posic041) ++ * Stop DcIPaddr (Started posic043) due to no quorum ++ * Stop rsc_posic041 (Started posic041) due to no quorum ++ * Stop rsc_posic042 (Started posic041) due to no quorum ++ * Stop rsc_posic043 (Started posic043) due to no quorum ++ * Stop rsc_posic044 (Started posic041) due to no quorum + + Executing cluster transition: + * Resource action: DcIPaddr monitor on posic041 +diff --git a/pengine/test10/797.summary b/pengine/test10/797.summary +index 3184eae..9e94003 100644 +--- a/pengine/test10/797.summary ++++ b/pengine/test10/797.summary +@@ -16,13 +16,13 @@ Online: [ c001n01 c001n02 c001n03 ] + + Transition Summary: + * Shutdown c001n02 +- * Stop DcIPaddr (Started c001n03) +- * Stop rsc_c001n08 (Started c001n02) +- * Stop rsc_c001n02 (Started c001n02) +- * Stop rsc_c001n03 (Started c001n03) +- * Stop rsc_c001n01 (Started c001n01) ++ * Stop DcIPaddr (Started c001n03) due to no quorum ++ * Stop rsc_c001n08 (Started c001n02) due to no quorum ++ * Stop rsc_c001n02 (Started c001n02) due to no quorum ++ * Stop rsc_c001n03 (Started c001n03) due to no quorum ++ * Stop rsc_c001n01 (Started c001n01) due to no quorum + * Restart child_DoFencing:0 (Started c001n01) +- * Stop child_DoFencing:1 (c001n02) ++ * Stop child_DoFencing:1 (c001n02) due to node availability + + Executing cluster transition: + * Resource action: DcIPaddr monitor on c001n02 +diff --git a/pengine/test10/829.summary b/pengine/test10/829.summary +index 9e66733..feca908 100644 +--- a/pengine/test10/829.summary ++++ b/pengine/test10/829.summary +@@ -17,7 +17,7 @@ Online: [ c001n01 c001n03 c001n08 ] + Transition Summary: + * Fence (reboot) c001n02 'peer is no longer part of the cluster' + * Move rsc_c001n02 (Started c001n02 -> c001n01) +- * Stop child_DoFencing:0 (c001n02) ++ * Stop child_DoFencing:0 (c001n02) due to node availability + + Executing cluster transition: + * Resource action: DcIPaddr monitor on c001n03 +diff --git a/pengine/test10/9-am-then-bm-b-migrating-a-stopping.summary b/pengine/test10/9-am-then-bm-b-migrating-a-stopping.summary +index 37d16be..44fc1a2 100644 +--- a/pengine/test10/9-am-then-bm-b-migrating-a-stopping.summary ++++ b/pengine/test10/9-am-then-bm-b-migrating-a-stopping.summary +@@ -7,8 +7,8 @@ Online: [ 18node1 18node2 18node3 ] + B (ocf::heartbeat:Dummy): Started 18node2 + + Transition Summary: +- * Stop A (18node1) +- * Stop B (Started 18node2) ++ * Stop A (18node1) due to node availability ++ * Stop B (Started 18node2) due to unrunnable A start + + Executing cluster transition: + * Resource action: B stop on 18node2 +diff --git a/pengine/test10/994-2.summary b/pengine/test10/994-2.summary +index dc4b9d1..a1d477f 100644 +--- a/pengine/test10/994-2.summary ++++ b/pengine/test10/994-2.summary +@@ -11,7 +11,7 @@ Online: [ paul ] + + Transition Summary: + * Recover postfix_9 (Started paul) +- * Restart depends (Started paul) ++ * Restart depends (Started paul) due to required group_1 running + + Executing cluster transition: + * Resource action: depends stop on paul +diff --git a/pengine/test10/anti-colocation-order.summary b/pengine/test10/anti-colocation-order.summary +index 2447501..052043a 100644 +--- a/pengine/test10/anti-colocation-order.summary ++++ b/pengine/test10/anti-colocation-order.summary +@@ -13,8 +13,8 @@ Online: [ node2 ] + Transition Summary: + * Move rsc1 (Started node1 -> node2) + * Move rsc2 (Started node1 -> node2) +- * Stop rsc3 (node2) +- * Stop rsc4 (node2) ++ * Stop rsc3 (node2) due to node availability ++ * Stop rsc4 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/asymmetrical-order-move.summary b/pengine/test10/asymmetrical-order-move.summary +index a9c1d8f..503813c 100644 +--- a/pengine/test10/asymmetrical-order-move.summary ++++ b/pengine/test10/asymmetrical-order-move.summary +@@ -9,7 +9,7 @@ Online: [ sle12sp2-1 sle12sp2-2 ] + dummy2 (ocf::pacemaker:Dummy): Started sle12sp2-1 + + Transition Summary: +- * Stop dummy2 (Started sle12sp2-1) ++ * Stop dummy2 (Started sle12sp2-1) due to unrunnable dummy1 start + + Executing cluster transition: + * Resource action: dummy2 stop on sle12sp2-1 +diff --git a/pengine/test10/bug-1572-1.summary b/pengine/test10/bug-1572-1.summary +index 6c37bb4..6a24614 100644 +--- a/pengine/test10/bug-1572-1.summary ++++ b/pengine/test10/bug-1572-1.summary +@@ -12,11 +12,11 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ] + + Transition Summary: + * Shutdown arc-dknightlx +- * Stop rsc_drbd_7788:0 (arc-dknightlx) ++ * Stop rsc_drbd_7788:0 (arc-dknightlx) due to node availability + * Restart rsc_drbd_7788:1 (Master arc-tkincaidlx.wsicorp.com) +- * Restart fs_mirror (Started arc-tkincaidlx.wsicorp.com) +- * Restart pgsql_5555 (Started arc-tkincaidlx.wsicorp.com) +- * Restart IPaddr_147_81_84_133 (Started arc-tkincaidlx.wsicorp.com) ++ * Restart fs_mirror (Started arc-tkincaidlx.wsicorp.com) due to required ms_drbd_7788 notified ++ * Restart pgsql_5555 (Started arc-tkincaidlx.wsicorp.com) due to required fs_mirror start ++ * Restart IPaddr_147_81_84_133 (Started arc-tkincaidlx.wsicorp.com) due to required pgsql_5555 start + + Executing cluster transition: + * Pseudo action: ms_drbd_7788_pre_notify_demote_0 +diff --git a/pengine/test10/bug-1572-2.summary b/pengine/test10/bug-1572-2.summary +index a4235a7..96574cf 100644 +--- a/pengine/test10/bug-1572-2.summary ++++ b/pengine/test10/bug-1572-2.summary +@@ -12,11 +12,11 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ] + + Transition Summary: + * Shutdown arc-dknightlx +- * Stop rsc_drbd_7788:0 (arc-dknightlx) ++ * Stop rsc_drbd_7788:0 (arc-dknightlx) due to node availability + * Demote rsc_drbd_7788:1 (Master -> Slave arc-tkincaidlx.wsicorp.com) +- * Stop fs_mirror (arc-tkincaidlx.wsicorp.com) +- * Stop pgsql_5555 (arc-tkincaidlx.wsicorp.com) +- * Stop IPaddr_147_81_84_133 (arc-tkincaidlx.wsicorp.com) ++ * Stop fs_mirror (arc-tkincaidlx.wsicorp.com) due to node availability ++ * Stop pgsql_5555 (arc-tkincaidlx.wsicorp.com) due to node availability ++ * Stop IPaddr_147_81_84_133 (arc-tkincaidlx.wsicorp.com) due to node availability + + Executing cluster transition: + * Pseudo action: ms_drbd_7788_pre_notify_demote_0 +diff --git a/pengine/test10/bug-1573.summary b/pengine/test10/bug-1573.summary +index 8fb2820..02d93a6 100644 +--- a/pengine/test10/bug-1573.summary ++++ b/pengine/test10/bug-1573.summary +@@ -12,7 +12,7 @@ OFFLINE: [ xen-c ] + + Transition Summary: + * Shutdown xen-b +- * Stop IPaddr_192_168_1_102 (xen-b) ++ * Stop IPaddr_192_168_1_102 (xen-b) due to node availability + + Executing cluster transition: + * Pseudo action: group_11_stop_0 +diff --git a/pengine/test10/bug-1718.summary b/pengine/test10/bug-1718.summary +index 622a5a8..b539e4e 100644 +--- a/pengine/test10/bug-1718.summary ++++ b/pengine/test10/bug-1718.summary +@@ -13,8 +13,8 @@ OFFLINE: [ defiant.ds9 warbird.ds9 ] + resource_dummy (ocf::heartbeat:Dummy): Started ops.ds9 + + Transition Summary: +- * Stop resource_IP3 (Started ops.ds9) +- * Stop resource_dummy (Started ops.ds9) ++ * Stop resource_IP3 (Started ops.ds9) due to unrunnable Web_Group running ++ * Stop resource_dummy (Started ops.ds9) due to required resource_IP3 start + + Executing cluster transition: + * Pseudo action: group_fUN_stop_0 +diff --git a/pengine/test10/bug-5014-A-stop-B-started.summary b/pengine/test10/bug-5014-A-stop-B-started.summary +index 20af4ab..a00dfd7 100644 +--- a/pengine/test10/bug-5014-A-stop-B-started.summary ++++ b/pengine/test10/bug-5014-A-stop-B-started.summary +@@ -7,7 +7,7 @@ Online: [ fc16-builder ] + ClusterIP2 (ocf::heartbeat:IPaddr2): Started fc16-builder + + Transition Summary: +- * Stop ClusterIP (fc16-builder) ++ * Stop ClusterIP (fc16-builder) due to node availability + + Executing cluster transition: + * Resource action: ClusterIP stop on fc16-builder +diff --git a/pengine/test10/bug-5014-A-stopped-B-stopped.summary b/pengine/test10/bug-5014-A-stopped-B-stopped.summary +index 970d81c..95e5b60 100644 +--- a/pengine/test10/bug-5014-A-stopped-B-stopped.summary ++++ b/pengine/test10/bug-5014-A-stopped-B-stopped.summary +@@ -7,7 +7,7 @@ Online: [ fc16-builder ] + ClusterIP2 (ocf::heartbeat:IPaddr2): Stopped + + Transition Summary: +- * Start ClusterIP2 (fc16-builder - blocked) ++ * Start ClusterIP2 (fc16-builder - blocked) due to unrunnable ClusterIP start + + Executing cluster transition: + * Resource action: ClusterIP monitor on fc16-builder +diff --git a/pengine/test10/bug-5014-CLONE-A-stop-B-started.summary b/pengine/test10/bug-5014-CLONE-A-stop-B-started.summary +index 5e9bec0..c3fb6d5 100644 +--- a/pengine/test10/bug-5014-CLONE-A-stop-B-started.summary ++++ b/pengine/test10/bug-5014-CLONE-A-stop-B-started.summary +@@ -9,7 +9,7 @@ Online: [ fc16-builder ] + Started: [ fc16-builder ] + + Transition Summary: +- * Stop ClusterIP:0 (fc16-builder) ++ * Stop ClusterIP:0 (fc16-builder) due to node availability + + Executing cluster transition: + * Pseudo action: clone1_stop_0 +diff --git a/pengine/test10/bug-5014-CthenAthenB-C-stopped.summary b/pengine/test10/bug-5014-CthenAthenB-C-stopped.summary +index 991e618..fe12fe6 100644 +--- a/pengine/test10/bug-5014-CthenAthenB-C-stopped.summary ++++ b/pengine/test10/bug-5014-CthenAthenB-C-stopped.summary +@@ -8,8 +8,8 @@ Online: [ fc16-builder ] + ClusterIP3 (ocf::heartbeat:IPaddr2): Stopped ( disabled ) + + Transition Summary: +- * Start ClusterIP (fc16-builder - blocked) +- * Start ClusterIP2 (fc16-builder - blocked) ++ * Start ClusterIP (fc16-builder - blocked) due to unrunnable ClusterIP3 start ++ * Start ClusterIP2 (fc16-builder - blocked) due to unrunnable ClusterIP start + + Executing cluster transition: + * Resource action: ClusterIP monitor on fc16-builder +diff --git a/pengine/test10/bug-5014-GROUP-A-stopped-B-started.summary b/pengine/test10/bug-5014-GROUP-A-stopped-B-started.summary +index 67df8d7..5ea35cb 100644 +--- a/pengine/test10/bug-5014-GROUP-A-stopped-B-started.summary ++++ b/pengine/test10/bug-5014-GROUP-A-stopped-B-started.summary +@@ -9,7 +9,7 @@ Online: [ fc16-builder ] + ClusterIP2 (ocf::heartbeat:IPaddr2): Started fc16-builder + + Transition Summary: +- * Stop ClusterIP (fc16-builder) ++ * Stop ClusterIP (fc16-builder) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/bug-5014-GROUP-A-stopped-B-stopped.summary b/pengine/test10/bug-5014-GROUP-A-stopped-B-stopped.summary +index e5251a0..f9b6448 100644 +--- a/pengine/test10/bug-5014-GROUP-A-stopped-B-stopped.summary ++++ b/pengine/test10/bug-5014-GROUP-A-stopped-B-stopped.summary +@@ -9,7 +9,7 @@ Online: [ fc16-builder ] + ClusterIP2 (ocf::heartbeat:IPaddr2): Stopped + + Transition Summary: +- * Start ClusterIP2 (fc16-builder - blocked) ++ * Start ClusterIP2 (fc16-builder - blocked) due to unrunnable group1 running + + Executing cluster transition: + +diff --git a/pengine/test10/bug-5014-ordered-set-symmetrical-false.summary b/pengine/test10/bug-5014-ordered-set-symmetrical-false.summary +index 4322db3..89b3416 100644 +--- a/pengine/test10/bug-5014-ordered-set-symmetrical-false.summary ++++ b/pengine/test10/bug-5014-ordered-set-symmetrical-false.summary +@@ -9,7 +9,7 @@ OFFLINE: [ fc16-builder2 ] + C (ocf::pacemaker:Dummy): Started fc16-builder ( disabled ) + + Transition Summary: +- * Stop C (fc16-builder) ++ * Stop C (fc16-builder) due to node availability + + Executing cluster transition: + * Resource action: C stop on fc16-builder +diff --git a/pengine/test10/bug-5014-ordered-set-symmetrical-true.summary b/pengine/test10/bug-5014-ordered-set-symmetrical-true.summary +index af74ba6..7d94d60 100644 +--- a/pengine/test10/bug-5014-ordered-set-symmetrical-true.summary ++++ b/pengine/test10/bug-5014-ordered-set-symmetrical-true.summary +@@ -9,8 +9,8 @@ OFFLINE: [ fc16-builder2 ] + C (ocf::pacemaker:Dummy): Started fc16-builder ( disabled ) + + Transition Summary: +- * Stop A (Started fc16-builder) +- * Stop C (fc16-builder) ++ * Stop A (Started fc16-builder) due to required C start ++ * Stop C (fc16-builder) due to node availability + + Executing cluster transition: + * Resource action: A stop on fc16-builder +diff --git a/pengine/test10/bug-5028-bottom.summary b/pengine/test10/bug-5028-bottom.summary +index b43ba4e..de24e29 100644 +--- a/pengine/test10/bug-5028-bottom.summary ++++ b/pengine/test10/bug-5028-bottom.summary +@@ -8,7 +8,7 @@ Online: [ bl460g6a bl460g6b ] + + Transition Summary: + * Shutdown bl460g6a +- * Stop dummy02 (bl460g6a) ++ * Stop dummy02 (bl460g6a) due to node availability + + Executing cluster transition: + * Pseudo action: dummy-g_stop_0 +diff --git a/pengine/test10/bug-5140-require-all-false.summary b/pengine/test10/bug-5140-require-all-false.summary +index e06e969..cf5193c 100644 +--- a/pengine/test10/bug-5140-require-all-false.summary ++++ b/pengine/test10/bug-5140-require-all-false.summary +@@ -36,9 +36,9 @@ Node hex-3: OFFLINE (standby) + cluster-md0 (ocf::heartbeat:Raid1): Stopped + + Transition Summary: +- * Stop dlm:0 (hex-2) +- * Stop clvmd:0 (hex-2) +- * Stop o2cb:0 (hex-2) ++ * Stop dlm:0 (hex-2) due to node availability ++ * Stop clvmd:0 (hex-2) due to node availability ++ * Stop o2cb:0 (hex-2) due to node availability + + Executing cluster transition: + * Pseudo action: baseclone_stop_0 +diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary +index c2685d8..cd01706 100644 +--- a/pengine/test10/bug-5186-partial-migrate.summary ++++ b/pengine/test10/bug-5186-partial-migrate.summary +@@ -30,9 +30,9 @@ Transition Summary: + * Move prmVM2 (Started bl460g1n7 -> bl460g1n8) + * Move prmStonith8-1 (Started bl460g1n7 -> bl460g1n6) + * Move prmStonith8-2 (Started bl460g1n7 -> bl460g1n6) +- * Stop prmDiskd1:0 (bl460g1n7) +- * Stop prmDiskd2:0 (bl460g1n7) +- * Stop prmPing:0 (bl460g1n7) ++ * Stop prmDiskd1:0 (bl460g1n7) due to node availability ++ * Stop prmDiskd2:0 (bl460g1n7) due to node availability ++ * Stop prmPing:0 (bl460g1n7) due to node availability + + Executing cluster transition: + * Resource action: prmVM2 stop on bl460g1n6 +diff --git a/pengine/test10/bug-cl-5170.summary b/pengine/test10/bug-cl-5170.summary +index c932a86..70b589e 100644 +--- a/pengine/test10/bug-cl-5170.summary ++++ b/pengine/test10/bug-cl-5170.summary +@@ -11,8 +11,8 @@ Online: [ TCS-2 ] + Stopped: [ TCS-1 ] + + Transition Summary: +- * Stop ip_trf (TCS-2) +- * Stop ip_mgmt (TCS-2) ++ * Stop ip_trf (TCS-2) due to node availability ++ * Stop ip_mgmt (TCS-2) due to node availability + + Executing cluster transition: + * Pseudo action: svc_stop_0 +diff --git a/pengine/test10/bug-cl-5212.summary b/pengine/test10/bug-cl-5212.summary +index b5d5146..92d3af3 100644 +--- a/pengine/test10/bug-cl-5212.summary ++++ b/pengine/test10/bug-cl-5212.summary +@@ -22,11 +22,11 @@ Online: [ srv03 ] + Transition Summary: + * Stop prmStonith1-1 (Started srv02 - blocked) + * Stop prmStonith2-1 (Started srv01 - blocked) +- * Stop prmStonith3-1 (srv01 - blocked) +- * Stop pgsql:0 (srv02 - blocked) ++ * Stop prmStonith3-1 (srv01 - blocked) due to node availability ++ * Stop pgsql:0 (srv02 - blocked) due to node availability + * Demote pgsql:1 (Master -> Stopped srv01 - blocked) +- * Stop prmPingd:0 (srv02 - blocked) +- * Stop prmPingd:1 (srv01 - blocked) ++ * Stop prmPingd:0 (srv02 - blocked) due to node availability ++ * Stop prmPingd:1 (srv01 - blocked) due to node availability + + Executing cluster transition: + * Pseudo action: grpStonith1_stop_0 +diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary +index fc97e8d..0f328cc 100644 +--- a/pengine/test10/bug-cl-5247.summary ++++ b/pengine/test10/bug-cl-5247.summary +@@ -19,7 +19,7 @@ Containers: [ pgsr01:prmDB1 ] + + Transition Summary: + * Fence (off) pgsr02 (resource: prmDB2) 'guest is unclean' +- * Stop prmDB2 (bl460g8n4) ++ * Stop prmDB2 (bl460g8n4) due to node availability + * Restart prmStonith1-2 (Started bl460g8n4) + * Restart prmStonith2-2 (Started bl460g8n3) + * Recover vip-master (Started pgsr02 -> pgsr01) +diff --git a/pengine/test10/bug-lf-2153.summary b/pengine/test10/bug-lf-2153.summary +index 9995475..e95713e 100644 +--- a/pengine/test10/bug-lf-2153.summary ++++ b/pengine/test10/bug-lf-2153.summary +@@ -18,8 +18,8 @@ Online: [ alice ] + res_portblock_iscsivg01_unblock (ocf::heartbeat:portblock): Started alice + + Transition Summary: +- * Stop res_drbd_iscsivg01:0 (bob) +- * Stop res_tgtd:0 (bob) ++ * Stop res_drbd_iscsivg01:0 (bob) due to node availability ++ * Stop res_tgtd:0 (bob) due to node availability + + Executing cluster transition: + * Pseudo action: ms_drbd_iscsivg01_pre_notify_stop_0 +diff --git a/pengine/test10/bug-lf-2171.summary b/pengine/test10/bug-lf-2171.summary +index b1413c3..8c1d8a4 100644 +--- a/pengine/test10/bug-lf-2171.summary ++++ b/pengine/test10/bug-lf-2171.summary +@@ -10,10 +10,10 @@ Online: [ xenserver1 xenserver2 ] + res_Dummy3 (ocf::heartbeat:Dummy): Started xenserver1 + + Transition Summary: +- * Stop res_Dummy1:0 (xenserver1) +- * Stop res_Dummy1:1 (xenserver2) +- * Stop res_Dummy2 (Started xenserver1) +- * Stop res_Dummy3 (Started xenserver1) ++ * Stop res_Dummy1:0 (xenserver1) due to node availability ++ * Stop res_Dummy1:1 (xenserver2) due to node availability ++ * Stop res_Dummy2 (Started xenserver1) due to unrunnable cl_res_Dummy1 running ++ * Stop res_Dummy3 (Started xenserver1) due to unrunnable cl_res_Dummy1 running + + Executing cluster transition: + * Pseudo action: gr_Dummy_stop_0 +diff --git a/pengine/test10/bug-lf-2361.summary b/pengine/test10/bug-lf-2361.summary +index b81456c..3089e04 100644 +--- a/pengine/test10/bug-lf-2361.summary ++++ b/pengine/test10/bug-lf-2361.summary +@@ -11,8 +11,8 @@ Online: [ alice.demo bob.demo ] + Transition Summary: + * Start stateful:0 (alice.demo) + * Start stateful:1 (bob.demo) +- * Start dummy2:0 (alice.demo - blocked) +- * Start dummy2:1 (bob.demo - blocked) ++ * Start dummy2:0 (alice.demo - blocked) due to unrunnable dummy1 start ++ * Start dummy2:1 (bob.demo - blocked) due to unrunnable dummy1 start + + Executing cluster transition: + * Pseudo action: ms_stateful_pre_notify_start_0 +diff --git a/pengine/test10/bug-lf-2422.summary b/pengine/test10/bug-lf-2422.summary +index 54341c9..5e22e67 100644 +--- a/pengine/test10/bug-lf-2422.summary ++++ b/pengine/test10/bug-lf-2422.summary +@@ -10,18 +10,18 @@ Online: [ qa-suse-1 qa-suse-2 qa-suse-3 qa-suse-4 ] + Started: [ qa-suse-1 qa-suse-2 qa-suse-3 qa-suse-4 ] + + Transition Summary: +- * Stop o2cb:0 (qa-suse-1) +- * Stop cmirror:0 (qa-suse-1) +- * Stop o2cb:1 (qa-suse-4) +- * Stop cmirror:1 (qa-suse-4) +- * Stop o2cb:2 (qa-suse-3) +- * Stop cmirror:2 (qa-suse-3) +- * Stop o2cb:3 (qa-suse-2) +- * Stop cmirror:3 (qa-suse-2) +- * Stop ocfs:0 (qa-suse-1) +- * Stop ocfs:1 (qa-suse-4) +- * Stop ocfs:2 (qa-suse-3) +- * Stop ocfs:3 (qa-suse-2) ++ * Stop o2cb:0 (qa-suse-1) due to node availability ++ * Stop cmirror:0 (qa-suse-1) due to node availability ++ * Stop o2cb:1 (qa-suse-4) due to node availability ++ * Stop cmirror:1 (qa-suse-4) due to node availability ++ * Stop o2cb:2 (qa-suse-3) due to node availability ++ * Stop cmirror:2 (qa-suse-3) due to node availability ++ * Stop o2cb:3 (qa-suse-2) due to node availability ++ * Stop cmirror:3 (qa-suse-2) due to node availability ++ * Stop ocfs:0 (qa-suse-1) due to node availability ++ * Stop ocfs:1 (qa-suse-4) due to node availability ++ * Stop ocfs:2 (qa-suse-3) due to node availability ++ * Stop ocfs:3 (qa-suse-2) due to node availability + + Executing cluster transition: + * Resource action: sbd_stonith monitor=15000 on qa-suse-2 +diff --git a/pengine/test10/bug-lf-2453.summary b/pengine/test10/bug-lf-2453.summary +index 3ff1a6b..398868b 100644 +--- a/pengine/test10/bug-lf-2453.summary ++++ b/pengine/test10/bug-lf-2453.summary +@@ -10,11 +10,11 @@ Online: [ domu1 domu2 ] + Started: [ domu1 domu2 ] + + Transition Summary: +- * Stop PrimitiveResource1 (Started domu1) +- * Stop apache:0 (domu1) +- * Stop apache:1 (domu2) +- * Stop DummyResource:0 (Started domu1) +- * Stop DummyResource:1 (Started domu2) ++ * Stop PrimitiveResource1 (Started domu1) due to required CloneResource2 running ++ * Stop apache:0 (domu1) due to node availability ++ * Stop apache:1 (domu2) due to node availability ++ * Stop DummyResource:0 (Started domu1) due to unrunnable CloneResource1 running ++ * Stop DummyResource:1 (Started domu2) due to unrunnable CloneResource1 running + + Executing cluster transition: + * Resource action: PrimitiveResource1 stop on domu1 +diff --git a/pengine/test10/bug-lf-2508.summary b/pengine/test10/bug-lf-2508.summary +index ddaf1c3..5d0d90c 100644 +--- a/pengine/test10/bug-lf-2508.summary ++++ b/pengine/test10/bug-lf-2508.summary +@@ -38,13 +38,13 @@ Transition Summary: + * Fence (reboot) srv02 'peer is no longer part of the cluster' + * Start Dummy01 (srv01) + * Move Dummy02 (Started srv02 -> srv04) +- * Stop prmStonith1-1:1 (srv02) +- * Stop prmStonith1-3:1 (srv02) +- * Stop prmStonith3-1:0 (srv02) +- * Stop prmStonith3-3:0 (srv02) ++ * Stop prmStonith1-1:1 (srv02) due to node availability ++ * Stop prmStonith1-3:1 (srv02) due to node availability ++ * Stop prmStonith3-1:0 (srv02) due to node availability ++ * Stop prmStonith3-3:0 (srv02) due to node availability + * Start prmStonith3-3:1 (srv01) +- * Stop prmStonith4-1:1 (srv02) +- * Stop prmStonith4-3:1 (srv02) ++ * Stop prmStonith4-1:1 (srv02) due to node availability ++ * Stop prmStonith4-3:1 (srv02) due to node availability + + Executing cluster transition: + * Pseudo action: Group01_start_0 +diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary +index 1b57ea7..9537277 100644 +--- a/pengine/test10/bug-lf-2551.summary ++++ b/pengine/test10/bug-lf-2551.summary +@@ -84,12 +84,12 @@ Transition Summary: + * Fence (reboot) hex-9 'peer is no longer part of the cluster' + * Move fencing-sbd (Started hex-9 -> hex-0) + * Move dummy1 (Started hex-9 -> hex-0) +- * Stop dlm:3 (hex-9) +- * Stop o2cb:3 (hex-9) +- * Stop clvm:3 (hex-9) +- * Stop cmirrord:3 (hex-9) +- * Stop vg1:3 (hex-9) +- * Stop ocfs2-1:3 (hex-9) ++ * Stop dlm:3 (hex-9) due to node availability ++ * Stop o2cb:3 (hex-9) due to node availability ++ * Stop clvm:3 (hex-9) due to node availability ++ * Stop cmirrord:3 (hex-9) due to node availability ++ * Stop vg1:3 (hex-9) due to node availability ++ * Stop ocfs2-1:3 (hex-9) due to node availability + * Stop vm-03 (hex-9) + * Stop vm-06 (hex-9) + * Stop vm-09 (hex-9) +diff --git a/pengine/test10/bug-lf-2574.summary b/pengine/test10/bug-lf-2574.summary +index 3024a73..800453c 100644 +--- a/pengine/test10/bug-lf-2574.summary ++++ b/pengine/test10/bug-lf-2574.summary +@@ -12,7 +12,7 @@ Online: [ srv01 srv02 srv03 ] + + Transition Summary: + * Move main_rsc (Started srv01 -> srv03) +- * Stop prmPingd:0 (srv01) ++ * Stop prmPingd:0 (srv01) due to node availability + + Executing cluster transition: + * Resource action: main_rsc stop on srv01 +diff --git a/pengine/test10/bug-lf-2619.summary b/pengine/test10/bug-lf-2619.summary +index 2816ac3..9a2213d 100644 +--- a/pengine/test10/bug-lf-2619.summary ++++ b/pengine/test10/bug-lf-2619.summary +@@ -34,7 +34,7 @@ Transition Summary: + * Move prmFsPostgreSQLDB1-3 (Started act1 -> sby1) + * Move prmIpPostgreSQLDB1 (Started act1 -> sby1) + * Move prmApPostgreSQLDB1 (Started act1 -> sby1) +- * Stop prmPingd:0 (act1) ++ * Stop prmPingd:0 (act1) due to node availability + + Executing cluster transition: + * Pseudo action: grpPostgreSQLDB1_stop_0 +diff --git a/pengine/test10/bug-n-385265.summary b/pengine/test10/bug-n-385265.summary +index 8f95787..e5bc4bc 100644 +--- a/pengine/test10/bug-n-385265.summary ++++ b/pengine/test10/bug-n-385265.summary +@@ -7,7 +7,7 @@ Online: [ ih01 ih02 ] + resource_idvscommon (ocf::dfs:idvs): FAILED ih02 + + Transition Summary: +- * Stop resource_idvscommon (ih02) ++ * Stop resource_idvscommon (ih02) due to node availability + + Executing cluster transition: + * Pseudo action: group_common_stop_0 +diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary +index 29dd018..ee21df8 100644 +--- a/pengine/test10/bug-rh-1097457.summary ++++ b/pengine/test10/bug-rh-1097457.summary +@@ -35,10 +35,10 @@ Transition Summary: + * Fence (reboot) lamaVM2 (resource: VM2) 'guest is unclean' + * Recover VM2 (Started lama3) + * Recover FSlun3 (Started lamaVM2 -> lama2) +- * Restart FAKE4 (Started lamaVM2) +- * Restart FAKE4-IP (Started lamaVM2) +- * Restart FAKE6:2 (Started lamaVM2) +- * Restart lamaVM2 (Started lama3) ++ * Restart FAKE4 (Started lamaVM2) due to required VM2 start ++ * Restart FAKE4-IP (Started lamaVM2) due to required VM2 start ++ * Restart FAKE6:2 (Started lamaVM2) due to required VM2 start ++ * Restart lamaVM2 (Started lama3) due to required VM2 start + + Executing cluster transition: + * Resource action: lamaVM2 stop on lama3 +diff --git a/pengine/test10/bug-suse-707150.summary b/pengine/test10/bug-suse-707150.summary +index da1d5c4..d6922ab 100644 +--- a/pengine/test10/bug-suse-707150.summary ++++ b/pengine/test10/bug-suse-707150.summary +@@ -26,7 +26,7 @@ Transition Summary: + * Start o2cb:1 (hex-9) + * Start clvm:1 (hex-9) + * Start cmirrord:1 (hex-9) +- * Start vm-01 (hex-9 - blocked) ++ * Start vm-01 (hex-9 - blocked) due to unrunnable base-clone running + + Executing cluster transition: + * Resource action: vg1:1 monitor on hex-9 +diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary +index 5e3927c..d67f8fc 100644 +--- a/pengine/test10/bundle-order-partial-start-2.summary ++++ b/pengine/test10/bundle-order-partial-start-2.summary +@@ -22,8 +22,8 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq- + + Transition Summary: + * Start rabbitmq:0 (rabbitmq-bundle-0) +- * Restart galera-bundle-docker-0 (Started undercloud) +- * Restart galera-bundle-0 (Started undercloud) ++ * Restart galera-bundle-docker-0 (Started undercloud) due to required haproxy-bundle running ++ * Restart galera-bundle-0 (Started undercloud) due to required galera-bundle-docker-0 start + * Start galera:0 (galera-bundle-0) + * Promote redis:0 (Slave -> Master redis-bundle-0) + * Start haproxy-bundle-docker-0 (undercloud) +diff --git a/pengine/test10/bundle-order-partial-stop.summary b/pengine/test10/bundle-order-partial-stop.summary +index b30a237..e7bac73 100644 +--- a/pengine/test10/bundle-order-partial-stop.summary ++++ b/pengine/test10/bundle-order-partial-stop.summary +@@ -22,25 +22,25 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq- + + Transition Summary: + * Shutdown undercloud +- * Stop rabbitmq-bundle-docker-0 (undercloud) +- * Stop rabbitmq-bundle-0 (undercloud) +- * Stop rabbitmq:0 (Started rabbitmq-bundle-0) +- * Stop galera-bundle-docker-0 (undercloud) +- * Stop galera-bundle-0 (undercloud) ++ * Stop rabbitmq-bundle-docker-0 (undercloud) due to node availability ++ * Stop rabbitmq-bundle-0 (undercloud) due to node availability ++ * Stop rabbitmq:0 (Started rabbitmq-bundle-0) due to unrunnable rabbitmq-bundle-0 start ++ * Stop galera-bundle-docker-0 (undercloud) due to node availability ++ * Stop galera-bundle-0 (undercloud) due to node availability + * Demote galera:0 (Master -> Slave galera-bundle-0) +- * Restart galera:0 (Slave galera-bundle-0) +- * Stop redis-bundle-docker-0 (undercloud) +- * Stop redis-bundle-0 (undercloud) ++ * Restart galera:0 (Slave galera-bundle-0) due to unrunnable galera-bundle-0 start ++ * Stop redis-bundle-docker-0 (undercloud) due to node availability ++ * Stop redis-bundle-0 (undercloud) due to node availability + * Demote redis:0 (Master -> Slave redis-bundle-0) +- * Restart redis:0 (Slave redis-bundle-0) +- * Stop ip-192.168.122.254 (undercloud) +- * Stop ip-192.168.122.250 (undercloud) +- * Stop ip-192.168.122.249 (undercloud) +- * Stop ip-192.168.122.253 (undercloud) +- * Stop ip-192.168.122.247 (undercloud) +- * Stop ip-192.168.122.248 (undercloud) +- * Stop haproxy-bundle-docker-0 (undercloud) +- * Stop openstack-cinder-volume-docker-0 (undercloud) ++ * Restart redis:0 (Slave redis-bundle-0) due to unrunnable redis-bundle-0 start ++ * Stop ip-192.168.122.254 (undercloud) due to node availability ++ * Stop ip-192.168.122.250 (undercloud) due to node availability ++ * Stop ip-192.168.122.249 (undercloud) due to node availability ++ * Stop ip-192.168.122.253 (undercloud) due to node availability ++ * Stop ip-192.168.122.247 (undercloud) due to node availability ++ * Stop ip-192.168.122.248 (undercloud) due to node availability ++ * Stop haproxy-bundle-docker-0 (undercloud) due to node availability ++ * Stop openstack-cinder-volume-docker-0 (undercloud) due to node availability + + Executing cluster transition: + * Resource action: galera cancel=10000 on galera-bundle-0 +diff --git a/pengine/test10/bundle-order-startup-clone.summary b/pengine/test10/bundle-order-startup-clone.summary +index f3f8be0..0acfd1e 100644 +--- a/pengine/test10/bundle-order-startup-clone.summary ++++ b/pengine/test10/bundle-order-startup-clone.summary +@@ -13,12 +13,12 @@ RemoteOFFLINE: [ rabbitmq-bundle-0 ] + redis-bundle-0 (ocf::heartbeat:redis): Stopped + + Transition Summary: +- * Start storage:0 (metal-1 - blocked) +- * Start storage:1 (metal-2 - blocked) +- * Start storage:2 (metal-3 - blocked) +- * Start galera-bundle-docker-0 (metal-1 - blocked) +- * Start galera-bundle-0 (metal-1 - blocked) +- * Start galera:0 (galera-bundle-0 - blocked) ++ * Start storage:0 (metal-1 - blocked) due to unrunnable redis-bundle promoted ++ * Start storage:1 (metal-2 - blocked) due to unrunnable redis-bundle promoted ++ * Start storage:2 (metal-3 - blocked) due to unrunnable redis-bundle promoted ++ * Start galera-bundle-docker-0 (metal-1 - blocked) due to unrunnable storage-clone notified ++ * Start galera-bundle-0 (metal-1 - blocked) due to unrunnable galera-bundle-docker-0 start ++ * Start galera:0 (galera-bundle-0 - blocked) due to unrunnable galera-bundle-docker-0 start + * Start haproxy-bundle-docker-0 (metal-2) + * Start redis-bundle-docker-0 (metal-2) + * Start redis-bundle-0 (metal-2) +diff --git a/pengine/test10/bundle-order-stop-clone.summary b/pengine/test10/bundle-order-stop-clone.summary +index 404eecd..9a6b0f2 100644 +--- a/pengine/test10/bundle-order-stop-clone.summary ++++ b/pengine/test10/bundle-order-stop-clone.summary +@@ -21,10 +21,10 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund + redis-bundle-2 (ocf::heartbeat:redis): Master metal-3 + + Transition Summary: +- * Stop storage:0 (metal-1) +- * Stop galera-bundle-docker-0 (metal-1) +- * Stop galera-bundle-0 (Started metal-1) +- * Stop galera:0 (Slave galera-bundle-0) ++ * Stop storage:0 (metal-1) due to node availability ++ * Stop galera-bundle-docker-0 (metal-1) due to node availability ++ * Stop galera-bundle-0 (Started metal-1) due to unrunnable galera-bundle-docker-0 start ++ * Stop galera:0 (Slave galera-bundle-0) due to unrunnable galera-bundle-docker-0 start + + Executing cluster transition: + * Pseudo action: storage-clone_pre_notify_stop_0 +diff --git a/pengine/test10/bundle-order-stop.summary b/pengine/test10/bundle-order-stop.summary +index b30a237..e7bac73 100644 +--- a/pengine/test10/bundle-order-stop.summary ++++ b/pengine/test10/bundle-order-stop.summary +@@ -22,25 +22,25 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq- + + Transition Summary: + * Shutdown undercloud +- * Stop rabbitmq-bundle-docker-0 (undercloud) +- * Stop rabbitmq-bundle-0 (undercloud) +- * Stop rabbitmq:0 (Started rabbitmq-bundle-0) +- * Stop galera-bundle-docker-0 (undercloud) +- * Stop galera-bundle-0 (undercloud) ++ * Stop rabbitmq-bundle-docker-0 (undercloud) due to node availability ++ * Stop rabbitmq-bundle-0 (undercloud) due to node availability ++ * Stop rabbitmq:0 (Started rabbitmq-bundle-0) due to unrunnable rabbitmq-bundle-0 start ++ * Stop galera-bundle-docker-0 (undercloud) due to node availability ++ * Stop galera-bundle-0 (undercloud) due to node availability + * Demote galera:0 (Master -> Slave galera-bundle-0) +- * Restart galera:0 (Slave galera-bundle-0) +- * Stop redis-bundle-docker-0 (undercloud) +- * Stop redis-bundle-0 (undercloud) ++ * Restart galera:0 (Slave galera-bundle-0) due to unrunnable galera-bundle-0 start ++ * Stop redis-bundle-docker-0 (undercloud) due to node availability ++ * Stop redis-bundle-0 (undercloud) due to node availability + * Demote redis:0 (Master -> Slave redis-bundle-0) +- * Restart redis:0 (Slave redis-bundle-0) +- * Stop ip-192.168.122.254 (undercloud) +- * Stop ip-192.168.122.250 (undercloud) +- * Stop ip-192.168.122.249 (undercloud) +- * Stop ip-192.168.122.253 (undercloud) +- * Stop ip-192.168.122.247 (undercloud) +- * Stop ip-192.168.122.248 (undercloud) +- * Stop haproxy-bundle-docker-0 (undercloud) +- * Stop openstack-cinder-volume-docker-0 (undercloud) ++ * Restart redis:0 (Slave redis-bundle-0) due to unrunnable redis-bundle-0 start ++ * Stop ip-192.168.122.254 (undercloud) due to node availability ++ * Stop ip-192.168.122.250 (undercloud) due to node availability ++ * Stop ip-192.168.122.249 (undercloud) due to node availability ++ * Stop ip-192.168.122.253 (undercloud) due to node availability ++ * Stop ip-192.168.122.247 (undercloud) due to node availability ++ * Stop ip-192.168.122.248 (undercloud) due to node availability ++ * Stop haproxy-bundle-docker-0 (undercloud) due to node availability ++ * Stop openstack-cinder-volume-docker-0 (undercloud) due to node availability + + Executing cluster transition: + * Resource action: galera cancel=10000 on galera-bundle-0 +diff --git a/pengine/test10/clone-anon-dup.summary b/pengine/test10/clone-anon-dup.summary +index 6ad247f..843972d 100644 +--- a/pengine/test10/clone-anon-dup.summary ++++ b/pengine/test10/clone-anon-dup.summary +@@ -11,7 +11,7 @@ Online: [ wc01 wc02 wc03 ] + + Transition Summary: + * Start stonith-1 (wc01) +- * Stop apache2:2 (wc02) ++ * Stop apache2:2 (wc02) due to node availability + + Executing cluster transition: + * Resource action: stonith-1 monitor on wc03 +diff --git a/pengine/test10/clone-anon-failcount.summary b/pengine/test10/clone-anon-failcount.summary +index cd4349c..3fb39e3 100644 +--- a/pengine/test10/clone-anon-failcount.summary ++++ b/pengine/test10/clone-anon-failcount.summary +@@ -42,9 +42,9 @@ Transition Summary: + * Move UmDummy01 (Started srv01 -> srv04) + * Move UmDummy02 (Started srv01 -> srv04) + * Recover clnUMdummy01:0 (Started srv04) +- * Restart clnUMdummy02:0 (Started srv04) +- * Stop clnUMdummy01:1 (srv01) +- * Stop clnUMdummy02:1 (srv01) ++ * Restart clnUMdummy02:0 (Started srv04) due to required clnUMdummy01:0 start ++ * Stop clnUMdummy01:1 (srv01) due to node availability ++ * Stop clnUMdummy02:1 (srv01) due to node availability + + Executing cluster transition: + * Pseudo action: UMgroup01_stop_0 +diff --git a/pengine/test10/clone-interleave-2.summary b/pengine/test10/clone-interleave-2.summary +index b15a546..78d46cd 100644 +--- a/pengine/test10/clone-interleave-2.summary ++++ b/pengine/test10/clone-interleave-2.summary +@@ -11,8 +11,8 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Started: [ pcmk-1 pcmk-2 pcmk-3 ] + + Transition Summary: +- * Restart dummy (Started pcmk-1) +- * Stop child-2:0 (pcmk-1) ++ * Restart dummy (Started pcmk-1) due to required clone-3 running ++ * Stop child-2:0 (pcmk-1) due to node availability + * Stop child-3:0 (pcmk-1) + + Executing cluster transition: +diff --git a/pengine/test10/clone-interleave-3.summary b/pengine/test10/clone-interleave-3.summary +index 004fa54..8b13dc4 100644 +--- a/pengine/test10/clone-interleave-3.summary ++++ b/pengine/test10/clone-interleave-3.summary +@@ -12,9 +12,9 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Started: [ pcmk-1 pcmk-2 pcmk-3 ] + + Transition Summary: +- * Restart dummy (Started pcmk-1) ++ * Restart dummy (Started pcmk-1) due to required clone-3 running + * Recover child-2:0 (Started pcmk-1) +- * Restart child-3:0 (Started pcmk-1) ++ * Restart child-3:0 (Started pcmk-1) due to required child-2:0 start + + Executing cluster transition: + * Resource action: dummy stop on pcmk-1 +diff --git a/pengine/test10/clone-max-zero.summary b/pengine/test10/clone-max-zero.summary +index 38aa6ba..33c4e89 100644 +--- a/pengine/test10/clone-max-zero.summary ++++ b/pengine/test10/clone-max-zero.summary +@@ -14,12 +14,12 @@ Online: [ c001n11 c001n12 ] + Started: [ c001n11 c001n12 ] + + Transition Summary: +- * Stop dlm:0 (c001n12) +- * Stop dlm:1 (c001n11) +- * Stop o2cb:0 (c001n12) +- * Stop o2cb:1 (c001n11) +- * Stop ocfs2-1:0 (c001n12) +- * Stop ocfs2-1:1 (c001n11) ++ * Stop dlm:0 (c001n12) due to node availability ++ * Stop dlm:1 (c001n11) due to node availability ++ * Stop o2cb:0 (c001n12) due to node availability ++ * Stop o2cb:1 (c001n11) due to node availability ++ * Stop ocfs2-1:0 (c001n12) due to node availability ++ * Stop ocfs2-1:1 (c001n11) due to node availability + + Executing cluster transition: + * Pseudo action: c-ocfs2-1_stop_0 +diff --git a/pengine/test10/clone-require-all-2.summary b/pengine/test10/clone-require-all-2.summary +index d4b2519..f5861e7 100644 +--- a/pengine/test10/clone-require-all-2.summary ++++ b/pengine/test10/clone-require-all-2.summary +@@ -13,10 +13,10 @@ Online: [ rhel7-auto3 rhel7-auto4 ] + + Transition Summary: + * Move shooter (Started rhel7-auto1 -> rhel7-auto3) +- * Stop A:0 (rhel7-auto1) +- * Stop A:1 (rhel7-auto2) +- * Start B:0 (rhel7-auto4 - blocked) +- * Start B:1 (rhel7-auto3 - blocked) ++ * Stop A:0 (rhel7-auto1) due to node availability ++ * Stop A:1 (rhel7-auto2) due to node availability ++ * Start B:0 (rhel7-auto4 - blocked) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory ++ * Start B:1 (rhel7-auto3 - blocked) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory + + Executing cluster transition: + * Resource action: shooter stop on rhel7-auto1 +diff --git a/pengine/test10/clone-require-all-3.summary b/pengine/test10/clone-require-all-3.summary +index 68191b1..1c887e5 100644 +--- a/pengine/test10/clone-require-all-3.summary ++++ b/pengine/test10/clone-require-all-3.summary +@@ -14,10 +14,10 @@ Online: [ rhel7-auto3 rhel7-auto4 ] + + Transition Summary: + * Move shooter (Started rhel7-auto1 -> rhel7-auto3) +- * Stop A:0 (rhel7-auto1) +- * Stop A:1 (rhel7-auto2) +- * Stop B:0 (Started rhel7-auto3) +- * Stop B:1 (Started rhel7-auto4) ++ * Stop A:0 (rhel7-auto1) due to node availability ++ * Stop A:1 (rhel7-auto2) due to node availability ++ * Stop B:0 (Started rhel7-auto3) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory ++ * Stop B:1 (Started rhel7-auto4) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory + + Executing cluster transition: + * Resource action: shooter stop on rhel7-auto1 +diff --git a/pengine/test10/clone-require-all-4.summary b/pengine/test10/clone-require-all-4.summary +index 49ae3bd..90d2059 100644 +--- a/pengine/test10/clone-require-all-4.summary ++++ b/pengine/test10/clone-require-all-4.summary +@@ -13,7 +13,7 @@ Online: [ rhel7-auto2 rhel7-auto3 rhel7-auto4 ] + + Transition Summary: + * Move shooter (Started rhel7-auto1 -> rhel7-auto2) +- * Stop A:0 (rhel7-auto1) ++ * Stop A:0 (rhel7-auto1) due to node availability + + Executing cluster transition: + * Resource action: shooter stop on rhel7-auto1 +diff --git a/pengine/test10/clone-require-all-6.summary b/pengine/test10/clone-require-all-6.summary +index 6561ea3..20ccf76 100644 +--- a/pengine/test10/clone-require-all-6.summary ++++ b/pengine/test10/clone-require-all-6.summary +@@ -11,8 +11,8 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 rhel7-auto4 ] + Stopped: [ rhel7-auto2 ] + + Transition Summary: +- * Stop A:0 (rhel7-auto1) +- * Stop A:2 (rhel7-auto3) ++ * Stop A:0 (rhel7-auto1) due to node availability ++ * Stop A:2 (rhel7-auto3) due to node availability + + Executing cluster transition: + * Pseudo action: A-clone_stop_0 +diff --git a/pengine/test10/clone_min_interleave_start_one.summary b/pengine/test10/clone_min_interleave_start_one.summary +index b15f68a..4ee71c4 100644 +--- a/pengine/test10/clone_min_interleave_start_one.summary ++++ b/pengine/test10/clone_min_interleave_start_one.summary +@@ -12,12 +12,12 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + + Transition Summary: + * Start FAKE1:0 (c7auto1) +- * Start FAKE2:0 (c7auto2 - blocked) +- * Start FAKE2:1 (c7auto3 - blocked) +- * Start FAKE2:2 (c7auto1 - blocked) +- * Start FAKE3:0 (c7auto2 - blocked) +- * Start FAKE3:1 (c7auto3 - blocked) +- * Start FAKE3:2 (c7auto1 - blocked) ++ * Start FAKE2:0 (c7auto2 - blocked) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory ++ * Start FAKE2:1 (c7auto3 - blocked) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory ++ * Start FAKE2:2 (c7auto1 - blocked) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory ++ * Start FAKE3:0 (c7auto2 - blocked) due to unrunnable FAKE2:0 start ++ * Start FAKE3:1 (c7auto3 - blocked) due to unrunnable FAKE2:1 start ++ * Start FAKE3:2 (c7auto1 - blocked) due to unrunnable FAKE2:2 start + + Executing cluster transition: + * Pseudo action: FAKE1-clone_start_0 +diff --git a/pengine/test10/clone_min_interleave_stop_one.summary b/pengine/test10/clone_min_interleave_stop_one.summary +index 9280b7e..4fd094e 100644 +--- a/pengine/test10/clone_min_interleave_stop_one.summary ++++ b/pengine/test10/clone_min_interleave_stop_one.summary +@@ -11,7 +11,7 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + Started: [ c7auto1 c7auto2 c7auto3 ] + + Transition Summary: +- * Stop FAKE1:0 (c7auto3) ++ * Stop FAKE1:0 (c7auto3) due to node availability + + Executing cluster transition: + * Pseudo action: FAKE1-clone_stop_0 +diff --git a/pengine/test10/clone_min_interleave_stop_two.summary b/pengine/test10/clone_min_interleave_stop_two.summary +index fb28e0d..0866f3c 100644 +--- a/pengine/test10/clone_min_interleave_stop_two.summary ++++ b/pengine/test10/clone_min_interleave_stop_two.summary +@@ -11,14 +11,14 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + Started: [ c7auto1 c7auto2 c7auto3 ] + + Transition Summary: +- * Stop FAKE1:0 (c7auto3) +- * Stop FAKE1:2 (c7auto2) +- * Stop FAKE2:0 (Started c7auto3) +- * Stop FAKE2:1 (Started c7auto1) +- * Stop FAKE2:2 (Started c7auto2) +- * Stop FAKE3:0 (Started c7auto3) +- * Stop FAKE3:1 (Started c7auto1) +- * Stop FAKE3:2 (Started c7auto2) ++ * Stop FAKE1:0 (c7auto3) due to node availability ++ * Stop FAKE1:2 (c7auto2) due to node availability ++ * Stop FAKE2:0 (Started c7auto3) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory ++ * Stop FAKE2:1 (Started c7auto1) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory ++ * Stop FAKE2:2 (Started c7auto2) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory ++ * Stop FAKE3:0 (Started c7auto3) due to required FAKE2:0 start ++ * Stop FAKE3:1 (Started c7auto1) due to required FAKE2:1 start ++ * Stop FAKE3:2 (Started c7auto2) due to required FAKE2:2 start + + Executing cluster transition: + * Pseudo action: FAKE3-clone_stop_0 +diff --git a/pengine/test10/clone_min_start_one.summary b/pengine/test10/clone_min_start_one.summary +index ee33e01..196f1b3 100644 +--- a/pengine/test10/clone_min_start_one.summary ++++ b/pengine/test10/clone_min_start_one.summary +@@ -12,7 +12,7 @@ Online: [ c7auto3 c7auto4 ] + Transition Summary: + * Move shooter (Started c7auto1 -> c7auto3) + * Start FAKECLONE:0 (c7auto3) +- * Start FAKE (c7auto4 - blocked) ++ * Start FAKE (c7auto4 - blocked) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory + + Executing cluster transition: + * Resource action: shooter stop on c7auto1 +diff --git a/pengine/test10/clone_min_stop_all.summary b/pengine/test10/clone_min_stop_all.summary +index eb2944f..877d12f 100644 +--- a/pengine/test10/clone_min_stop_all.summary ++++ b/pengine/test10/clone_min_stop_all.summary +@@ -13,10 +13,10 @@ Online: [ c7auto4 ] + + Transition Summary: + * Move shooter (Started c7auto1 -> c7auto4) +- * Stop FAKECLONE:0 (c7auto1) +- * Stop FAKECLONE:1 (c7auto2) +- * Stop FAKECLONE:2 (c7auto3) +- * Stop FAKE (Started c7auto4) ++ * Stop FAKECLONE:0 (c7auto1) due to node availability ++ * Stop FAKECLONE:1 (c7auto2) due to node availability ++ * Stop FAKECLONE:2 (c7auto3) due to node availability ++ * Stop FAKE (Started c7auto4) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory + + Executing cluster transition: + * Resource action: shooter stop on c7auto1 +diff --git a/pengine/test10/clone_min_stop_one.summary b/pengine/test10/clone_min_stop_one.summary +index 9206a0d..3fdb675 100644 +--- a/pengine/test10/clone_min_stop_one.summary ++++ b/pengine/test10/clone_min_stop_one.summary +@@ -10,7 +10,7 @@ Online: [ c7auto1 c7auto3 c7auto4 ] + FAKE (ocf::heartbeat:Dummy): Started c7auto4 + + Transition Summary: +- * Stop FAKECLONE:1 (c7auto2) ++ * Stop FAKECLONE:1 (c7auto2) due to node availability + + Executing cluster transition: + * Pseudo action: FAKECLONE-clone_stop_0 +diff --git a/pengine/test10/clone_min_stop_two.summary b/pengine/test10/clone_min_stop_two.summary +index c009d7d..4d8c38f 100644 +--- a/pengine/test10/clone_min_stop_two.summary ++++ b/pengine/test10/clone_min_stop_two.summary +@@ -12,9 +12,9 @@ Online: [ c7auto3 c7auto4 ] + + Transition Summary: + * Move shooter (Started c7auto1 -> c7auto3) +- * Stop FAKECLONE:0 (c7auto1) +- * Stop FAKECLONE:1 (c7auto2) +- * Stop FAKE (Started c7auto4) ++ * Stop FAKECLONE:0 (c7auto1) due to node availability ++ * Stop FAKECLONE:1 (c7auto2) due to node availability ++ * Stop FAKE (Started c7auto4) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory + + Executing cluster transition: + * Resource action: shooter stop on c7auto1 +diff --git a/pengine/test10/cloned-group-stop.summary b/pengine/test10/cloned-group-stop.summary +index 8357c38..f7a980c 100644 +--- a/pengine/test10/cloned-group-stop.summary ++++ b/pengine/test10/cloned-group-stop.summary +@@ -18,22 +18,22 @@ Online: [ rhos4-node3 rhos4-node4 ] + Started: [ rhos4-node3 rhos4-node4 ] + + Transition Summary: +- * Stop qpidd:0 (rhos4-node4) +- * Stop qpidd:1 (rhos4-node3) +- * Stop keystone:0 (Started rhos4-node4) +- * Stop keystone:1 (Started rhos4-node3) +- * Stop glance-fs:0 (Started rhos4-node4) +- * Stop glance-registry:0 (Started rhos4-node4) +- * Stop glance-api:0 (Started rhos4-node4) +- * Stop glance-fs:1 (Started rhos4-node3) +- * Stop glance-registry:1 (Started rhos4-node3) +- * Stop glance-api:1 (Started rhos4-node3) +- * Stop cinder-api:0 (Started rhos4-node4) +- * Stop cinder-scheduler:0 (Started rhos4-node4) +- * Stop cinder-volume:0 (Started rhos4-node4) +- * Stop cinder-api:1 (Started rhos4-node3) +- * Stop cinder-scheduler:1 (Started rhos4-node3) +- * Stop cinder-volume:1 (Started rhos4-node3) ++ * Stop qpidd:0 (rhos4-node4) due to node availability ++ * Stop qpidd:1 (rhos4-node3) due to node availability ++ * Stop keystone:0 (Started rhos4-node4) due to unrunnable qpidd-clone running ++ * Stop keystone:1 (Started rhos4-node3) due to unrunnable qpidd-clone running ++ * Stop glance-fs:0 (Started rhos4-node4) due to required keystone-clone running ++ * Stop glance-registry:0 (Started rhos4-node4) due to required glance-fs:0 start ++ * Stop glance-api:0 (Started rhos4-node4) due to required glance-registry:0 start ++ * Stop glance-fs:1 (Started rhos4-node3) due to required keystone-clone running ++ * Stop glance-registry:1 (Started rhos4-node3) due to required glance-fs:1 start ++ * Stop glance-api:1 (Started rhos4-node3) due to required glance-registry:1 start ++ * Stop cinder-api:0 (Started rhos4-node4) due to required glance-clone running ++ * Stop cinder-scheduler:0 (Started rhos4-node4) due to required cinder-api:0 start ++ * Stop cinder-volume:0 (Started rhos4-node4) due to required cinder-scheduler:0 start ++ * Stop cinder-api:1 (Started rhos4-node3) due to required glance-clone running ++ * Stop cinder-scheduler:1 (Started rhos4-node3) due to required cinder-api:1 start ++ * Stop cinder-volume:1 (Started rhos4-node3) due to required cinder-scheduler:1 start + + Executing cluster transition: + * Pseudo action: cinder-clone_stop_0 +diff --git a/pengine/test10/cloned-group.summary b/pengine/test10/cloned-group.summary +index 7d64be4..e1456b9 100644 +--- a/pengine/test10/cloned-group.summary ++++ b/pengine/test10/cloned-group.summary +@@ -12,9 +12,9 @@ OFFLINE: [ webcluster02 ] + + Transition Summary: + * Restart apache2:0 (Started webcluster01) +- * Restart mysql-proxy:0 (Started webcluster01) +- * Stop apache2:2 (webcluster01) +- * Stop mysql-proxy:2 (webcluster01) ++ * Restart mysql-proxy:0 (Started webcluster01) due to required apache2:0 start ++ * Stop apache2:2 (webcluster01) due to node availability ++ * Stop mysql-proxy:2 (webcluster01) due to node availability + + Executing cluster transition: + * Pseudo action: apache2_clone_stop_0 +diff --git a/pengine/test10/cloned_start_one.summary b/pengine/test10/cloned_start_one.summary +index 20ac58f..5dedc18 100644 +--- a/pengine/test10/cloned_start_one.summary ++++ b/pengine/test10/cloned_start_one.summary +@@ -13,8 +13,8 @@ Online: [ c7auto1 c7auto4 ] + + Transition Summary: + * Start FAKECLONE:0 (c7auto1) +- * Stop FAKECLONE2:0 (c7auto3) +- * Stop FAKECLONE2:1 (Started c7auto4) ++ * Stop FAKECLONE2:0 (c7auto3) due to node availability ++ * Stop FAKECLONE2:1 (Started c7auto4) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory + + Executing cluster transition: + * Pseudo action: FAKECLONE-clone_start_0 +diff --git a/pengine/test10/cloned_start_two.summary b/pengine/test10/cloned_start_two.summary +index bea4609..2b47881 100644 +--- a/pengine/test10/cloned_start_two.summary ++++ b/pengine/test10/cloned_start_two.summary +@@ -13,7 +13,7 @@ Online: [ c7auto1 c7auto2 c7auto4 ] + Transition Summary: + * Start FAKECLONE:0 (c7auto2) + * Start FAKECLONE:1 (c7auto1) +- * Stop FAKECLONE2:0 (c7auto3) ++ * Stop FAKECLONE2:0 (c7auto3) due to node availability + + Executing cluster transition: + * Pseudo action: FAKECLONE-clone_start_0 +diff --git a/pengine/test10/cloned_stop_one.summary b/pengine/test10/cloned_stop_one.summary +index 1a952a2..892548d 100644 +--- a/pengine/test10/cloned_stop_one.summary ++++ b/pengine/test10/cloned_stop_one.summary +@@ -12,8 +12,8 @@ Online: [ c7auto1 c7auto2 c7auto4 ] + Stopped: [ c7auto1 c7auto2 ] + + Transition Summary: +- * Stop FAKECLONE:2 (c7auto3) +- * Stop FAKECLONE2:0 (c7auto3) ++ * Stop FAKECLONE:2 (c7auto3) due to node availability ++ * Stop FAKECLONE2:0 (c7auto3) due to node availability + + Executing cluster transition: + * Pseudo action: FAKECLONE2-clone_stop_0 +diff --git a/pengine/test10/cloned_stop_two.summary b/pengine/test10/cloned_stop_two.summary +index 531295f..8f1e039 100644 +--- a/pengine/test10/cloned_stop_two.summary ++++ b/pengine/test10/cloned_stop_two.summary +@@ -13,10 +13,10 @@ Online: [ c7auto1 c7auto4 ] + Stopped: [ c7auto1 c7auto2 ] + + Transition Summary: +- * Stop FAKECLONE:1 (c7auto2) +- * Stop FAKECLONE:2 (c7auto3) +- * Stop FAKECLONE2:0 (c7auto3) +- * Stop FAKECLONE2:1 (Started c7auto4) ++ * Stop FAKECLONE:1 (c7auto2) due to node availability ++ * Stop FAKECLONE:2 (c7auto3) due to node availability ++ * Stop FAKECLONE2:0 (c7auto3) due to node availability ++ * Stop FAKECLONE2:1 (Started c7auto4) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory + + Executing cluster transition: + * Pseudo action: FAKECLONE2-clone_stop_0 +diff --git a/pengine/test10/colocation_constraint_stops_slave.summary b/pengine/test10/colocation_constraint_stops_slave.summary +index fe9e044..a97b74b 100644 +--- a/pengine/test10/colocation_constraint_stops_slave.summary ++++ b/pengine/test10/colocation_constraint_stops_slave.summary +@@ -9,7 +9,7 @@ OFFLINE: [ fc16-builder2 ] + NATIVE_RSC_B (ocf::pacemaker:Dummy): Started fc16-builder ( disabled ) + + Transition Summary: +- * Stop NATIVE_RSC_A:0 (fc16-builder) ++ * Stop NATIVE_RSC_A:0 (fc16-builder) due to node availability + * Stop NATIVE_RSC_B (fc16-builder) + + Executing cluster transition: +diff --git a/pengine/test10/complex_enforce_colo.summary b/pengine/test10/complex_enforce_colo.summary +index dd838b2..a21d5c1 100644 +--- a/pengine/test10/complex_enforce_colo.summary ++++ b/pengine/test10/complex_enforce_colo.summary +@@ -102,17 +102,17 @@ Online: [ rhos6-node1 rhos6-node2 rhos6-node3 ] + Started: [ rhos6-node1 rhos6-node2 rhos6-node3 ] + + Transition Summary: +- * Stop keystone:0 (rhos6-node1) +- * Stop keystone:1 (rhos6-node2) +- * Stop keystone:2 (rhos6-node3) ++ * Stop keystone:0 (rhos6-node1) due to node availability ++ * Stop keystone:1 (rhos6-node2) due to node availability ++ * Stop keystone:2 (rhos6-node3) due to node availability + * Stop glance-registry:0 (rhos6-node1) + * Stop glance-registry:1 (rhos6-node2) + * Stop glance-registry:2 (rhos6-node3) + * Stop glance-api:0 (rhos6-node1) + * Stop glance-api:1 (rhos6-node2) + * Stop glance-api:2 (rhos6-node3) +- * Stop cinder-api (Started rhos6-node1) +- * Stop cinder-scheduler (Started rhos6-node1) ++ * Stop cinder-api (Started rhos6-node1) due to unrunnable keystone-clone running ++ * Stop cinder-scheduler (Started rhos6-node1) due to required cinder-api start + * Stop cinder-volume (Started rhos6-node1) + * Stop swift-account:0 (rhos6-node1) + * Stop swift-account:1 (rhos6-node2) +@@ -126,7 +126,7 @@ Transition Summary: + * Stop swift-proxy:0 (rhos6-node1) + * Stop swift-proxy:1 (rhos6-node2) + * Stop swift-proxy:2 (rhos6-node3) +- * Stop swift-object-expirer (Started rhos6-node2) ++ * Stop swift-object-expirer (Started rhos6-node2) due to required swift-proxy-clone running + * Stop neutron-server:0 (rhos6-node1) + * Stop neutron-server:1 (rhos6-node2) + * Stop neutron-server:2 (rhos6-node3) +@@ -166,35 +166,35 @@ Transition Summary: + * Stop nova-conductor:0 (rhos6-node1) + * Stop nova-conductor:1 (rhos6-node2) + * Stop nova-conductor:2 (rhos6-node3) +- * Stop ceilometer-central (Started rhos6-node3) +- * Stop ceilometer-collector:0 (Started rhos6-node1) +- * Stop ceilometer-collector:1 (Started rhos6-node2) +- * Stop ceilometer-collector:2 (Started rhos6-node3) +- * Stop ceilometer-api:0 (Started rhos6-node1) +- * Stop ceilometer-api:1 (Started rhos6-node2) +- * Stop ceilometer-api:2 (Started rhos6-node3) +- * Stop ceilometer-delay:0 (Started rhos6-node1) +- * Stop ceilometer-delay:1 (Started rhos6-node2) +- * Stop ceilometer-delay:2 (Started rhos6-node3) +- * Stop ceilometer-alarm-evaluator:0 (Started rhos6-node1) +- * Stop ceilometer-alarm-evaluator:1 (Started rhos6-node2) +- * Stop ceilometer-alarm-evaluator:2 (Started rhos6-node3) +- * Stop ceilometer-alarm-notifier:0 (Started rhos6-node1) +- * Stop ceilometer-alarm-notifier:1 (Started rhos6-node2) +- * Stop ceilometer-alarm-notifier:2 (Started rhos6-node3) +- * Stop ceilometer-notification:0 (Started rhos6-node1) +- * Stop ceilometer-notification:1 (Started rhos6-node2) +- * Stop ceilometer-notification:2 (Started rhos6-node3) +- * Stop heat-api:0 (Started rhos6-node1) +- * Stop heat-api:1 (Started rhos6-node2) +- * Stop heat-api:2 (Started rhos6-node3) +- * Stop heat-api-cfn:0 (Started rhos6-node1) +- * Stop heat-api-cfn:1 (Started rhos6-node2) +- * Stop heat-api-cfn:2 (Started rhos6-node3) +- * Stop heat-api-cloudwatch:0 (Started rhos6-node1) +- * Stop heat-api-cloudwatch:1 (Started rhos6-node2) +- * Stop heat-api-cloudwatch:2 (Started rhos6-node3) +- * Stop heat-engine (Started rhos6-node2) ++ * Stop ceilometer-central (Started rhos6-node3) due to unrunnable keystone-clone running ++ * Stop ceilometer-collector:0 (Started rhos6-node1) due to required ceilometer-central start ++ * Stop ceilometer-collector:1 (Started rhos6-node2) due to required ceilometer-central start ++ * Stop ceilometer-collector:2 (Started rhos6-node3) due to required ceilometer-central start ++ * Stop ceilometer-api:0 (Started rhos6-node1) due to required ceilometer-collector:0 start ++ * Stop ceilometer-api:1 (Started rhos6-node2) due to required ceilometer-collector:1 start ++ * Stop ceilometer-api:2 (Started rhos6-node3) due to required ceilometer-collector:2 start ++ * Stop ceilometer-delay:0 (Started rhos6-node1) due to required ceilometer-api:0 start ++ * Stop ceilometer-delay:1 (Started rhos6-node2) due to required ceilometer-api:1 start ++ * Stop ceilometer-delay:2 (Started rhos6-node3) due to required ceilometer-api:2 start ++ * Stop ceilometer-alarm-evaluator:0 (Started rhos6-node1) due to required ceilometer-delay:0 start ++ * Stop ceilometer-alarm-evaluator:1 (Started rhos6-node2) due to required ceilometer-delay:1 start ++ * Stop ceilometer-alarm-evaluator:2 (Started rhos6-node3) due to required ceilometer-delay:2 start ++ * Stop ceilometer-alarm-notifier:0 (Started rhos6-node1) due to required ceilometer-alarm-evaluator:0 start ++ * Stop ceilometer-alarm-notifier:1 (Started rhos6-node2) due to required ceilometer-alarm-evaluator:1 start ++ * Stop ceilometer-alarm-notifier:2 (Started rhos6-node3) due to required ceilometer-alarm-evaluator:2 start ++ * Stop ceilometer-notification:0 (Started rhos6-node1) due to required ceilometer-alarm-notifier:0 start ++ * Stop ceilometer-notification:1 (Started rhos6-node2) due to required ceilometer-alarm-notifier:1 start ++ * Stop ceilometer-notification:2 (Started rhos6-node3) due to required ceilometer-alarm-notifier:2 start ++ * Stop heat-api:0 (Started rhos6-node1) due to required ceilometer-notification:0 start ++ * Stop heat-api:1 (Started rhos6-node2) due to required ceilometer-notification:1 start ++ * Stop heat-api:2 (Started rhos6-node3) due to required ceilometer-notification:2 start ++ * Stop heat-api-cfn:0 (Started rhos6-node1) due to required heat-api:0 start ++ * Stop heat-api-cfn:1 (Started rhos6-node2) due to required heat-api:1 start ++ * Stop heat-api-cfn:2 (Started rhos6-node3) due to required heat-api:2 start ++ * Stop heat-api-cloudwatch:0 (Started rhos6-node1) due to required heat-api-cfn:0 start ++ * Stop heat-api-cloudwatch:1 (Started rhos6-node2) due to required heat-api-cfn:1 start ++ * Stop heat-api-cloudwatch:2 (Started rhos6-node3) due to required heat-api-cfn:2 start ++ * Stop heat-engine (Started rhos6-node2) due to required heat-api-cloudwatch-clone running + + Executing cluster transition: + * Pseudo action: glance-api-clone_stop_0 +diff --git a/pengine/test10/container-2.summary b/pengine/test10/container-2.summary +index b0c08b8..f011cd3 100644 +--- a/pengine/test10/container-2.summary ++++ b/pengine/test10/container-2.summary +@@ -9,7 +9,7 @@ Online: [ node1 node2 ] + Transition Summary: + * Restart container1 (Started node1) + * Recover rsc1 (Started node1) +- * Restart rsc2 (Started node1) ++ * Restart rsc2 (Started node1) due to required container1 start + + Executing cluster transition: + * Resource action: rsc1 stop on node1 +diff --git a/pengine/test10/container-3.summary b/pengine/test10/container-3.summary +index 194c687..f853ab2 100644 +--- a/pengine/test10/container-3.summary ++++ b/pengine/test10/container-3.summary +@@ -9,7 +9,7 @@ Online: [ node1 node2 ] + Transition Summary: + * Restart container1 (Started node1) + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) ++ * Restart rsc2 (Started node1) due to required container1 start + + Executing cluster transition: + * Resource action: rsc2 stop on node1 +diff --git a/pengine/test10/container-group-2.summary b/pengine/test10/container-group-2.summary +index 4451c63..c0dbbf8 100644 +--- a/pengine/test10/container-group-2.summary ++++ b/pengine/test10/container-group-2.summary +@@ -10,7 +10,7 @@ Online: [ node1 node2 ] + Transition Summary: + * Restart container1 (Started node1) + * Recover rsc1 (Started node1) +- * Restart rsc2 (Started node1) ++ * Restart rsc2 (Started node1) due to required rsc1 start + + Executing cluster transition: + * Pseudo action: container-group_stop_0 +diff --git a/pengine/test10/enforce-colo1.summary b/pengine/test10/enforce-colo1.summary +index 9859073..b79b8cb 100644 +--- a/pengine/test10/enforce-colo1.summary ++++ b/pengine/test10/enforce-colo1.summary +@@ -11,10 +11,10 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ] + + Transition Summary: + * Stop engine (Started rhel7-auto3) +- * Stop keystone:0 (rhel7-auto2) +- * Stop keystone:1 (rhel7-auto3) +- * Stop keystone:2 (rhel7-auto1) +- * Stop central (Started rhel7-auto3) ++ * Stop keystone:0 (rhel7-auto2) due to node availability ++ * Stop keystone:1 (rhel7-auto3) due to node availability ++ * Stop keystone:2 (rhel7-auto1) due to node availability ++ * Stop central (Started rhel7-auto3) due to unrunnable keystone-clone running + + Executing cluster transition: + * Resource action: engine stop on rhel7-auto3 +diff --git a/pengine/test10/group-fail.summary b/pengine/test10/group-fail.summary +index aa03d21..9067bf6 100644 +--- a/pengine/test10/group-fail.summary ++++ b/pengine/test10/group-fail.summary +@@ -10,9 +10,9 @@ Online: [ node1 node2 ] + + Transition Summary: + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) ++ * Restart rsc2 (Started node1) due to required rsc1 start + * Start rsc3 (node1) +- * Restart rsc4 (Started node1) ++ * Restart rsc4 (Started node1) due to required rsc3 start + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/group-unmanaged-stopped.summary b/pengine/test10/group-unmanaged-stopped.summary +index 9f542ff..de0586a 100644 +--- a/pengine/test10/group-unmanaged-stopped.summary ++++ b/pengine/test10/group-unmanaged-stopped.summary +@@ -8,7 +8,7 @@ Online: [ pcmk-1 pcmk-2 ] + r192.168.122.115 (ocf::heartbeat:IPaddr2): Started pcmk-1 + + Transition Summary: +- * Stop r192.168.122.115 (pcmk-1) ++ * Stop r192.168.122.115 (pcmk-1) due to node availability + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/group10.summary b/pengine/test10/group10.summary +index c630a21..570fd28 100644 +--- a/pengine/test10/group10.summary ++++ b/pengine/test10/group10.summary +@@ -19,8 +19,8 @@ Online: [ c001n01 c001n02 c001n03 c001n08 ] + + Transition Summary: + * Recover child_192.168.100.181 (Started c001n01) +- * Restart child_192.168.100.182 (Started c001n01) +- * Restart child_192.168.100.183 (Started c001n01) ++ * Restart child_192.168.100.182 (Started c001n01) due to required child_192.168.100.181 start ++ * Restart child_192.168.100.183 (Started c001n01) due to required child_192.168.100.182 start + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/group11.summary b/pengine/test10/group11.summary +index 9619e51..204d632 100644 +--- a/pengine/test10/group11.summary ++++ b/pengine/test10/group11.summary +@@ -9,8 +9,8 @@ Online: [ node1 ] + rsc3 (heartbeat:apache): Started node1 + + Transition Summary: +- * Stop rsc2 (node1) +- * Stop rsc3 (node1) ++ * Stop rsc2 (node1) due to node availability ++ * Stop rsc3 (node1) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/group13.summary b/pengine/test10/group13.summary +index e728b2e..5f92a4f 100644 +--- a/pengine/test10/group13.summary ++++ b/pengine/test10/group13.summary +@@ -8,7 +8,7 @@ Online: [ jamesltc ] + resource_fs (ocf::heartbeat:Filesystem): Stopped + + Transition Summary: +- * Stop resource_nfs (jamesltc) ++ * Stop resource_nfs (jamesltc) due to node availability + + Executing cluster transition: + * Pseudo action: nfs_stop_0 +diff --git a/pengine/test10/group14.summary b/pengine/test10/group14.summary +index 451aeda..69e688b 100644 +--- a/pengine/test10/group14.summary ++++ b/pengine/test10/group14.summary +@@ -33,24 +33,24 @@ OFFLINE: [ c001n02 c001n03 c001n04 c001n05 ] + ocf_msdummy:11 (ocf::heartbeat:Stateful): Stopped + + Transition Summary: +- * Start DcIPaddr (c001n06 - blocked) +- * Stop r192.168.100.181 (Started c001n06) +- * Start r192.168.100.182 (c001n07 - blocked) +- * Start r192.168.100.183 (c001n07 - blocked) +- * Start lsb_dummy (c001n06 - blocked) +- * Start migrator (c001n06 - blocked) +- * Start rsc_c001n03 (c001n06 - blocked) +- * Start rsc_c001n02 (c001n07 - blocked) +- * Start rsc_c001n04 (c001n06 - blocked) +- * Start rsc_c001n05 (c001n07 - blocked) +- * Start rsc_c001n06 (c001n06 - blocked) +- * Start rsc_c001n07 (c001n07 - blocked) ++ * Start DcIPaddr (c001n06 - blocked) due to no quorum ++ * Stop r192.168.100.181 (Started c001n06) due to no quorum ++ * Start r192.168.100.182 (c001n07 - blocked) due to no quorum ++ * Start r192.168.100.183 (c001n07 - blocked) due to no quorum ++ * Start lsb_dummy (c001n06 - blocked) due to no quorum ++ * Start migrator (c001n06 - blocked) due to no quorum ++ * Start rsc_c001n03 (c001n06 - blocked) due to no quorum ++ * Start rsc_c001n02 (c001n07 - blocked) due to no quorum ++ * Start rsc_c001n04 (c001n06 - blocked) due to no quorum ++ * Start rsc_c001n05 (c001n07 - blocked) due to no quorum ++ * Start rsc_c001n06 (c001n06 - blocked) due to no quorum ++ * Start rsc_c001n07 (c001n07 - blocked) due to no quorum + * Start child_DoFencing:0 (c001n06) + * Start child_DoFencing:1 (c001n07) +- * Start ocf_msdummy:0 (c001n06 - blocked) +- * Start ocf_msdummy:1 (c001n07 - blocked) +- * Start ocf_msdummy:2 (c001n06 - blocked) +- * Start ocf_msdummy:3 (c001n07 - blocked) ++ * Start ocf_msdummy:0 (c001n06 - blocked) due to no quorum ++ * Start ocf_msdummy:1 (c001n07 - blocked) due to no quorum ++ * Start ocf_msdummy:2 (c001n06 - blocked) due to no quorum ++ * Start ocf_msdummy:3 (c001n07 - blocked) due to no quorum + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/group9.summary b/pengine/test10/group9.summary +index 6989f1d..f6755b9 100644 +--- a/pengine/test10/group9.summary ++++ b/pengine/test10/group9.summary +@@ -15,7 +15,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Recover rsc4 (Started node1) +- * Restart rsc5 (Started node1) ++ * Restart rsc5 (Started node1) due to required rsc4 start + * Move rsc6 (Started node1 -> node2) + * Recover rsc7 (Started node1 -> node2) + * Move rsc8 (Started node1 -> node2) +diff --git a/pengine/test10/inc10.summary b/pengine/test10/inc10.summary +index 77552e7..053a9ba 100644 +--- a/pengine/test10/inc10.summary ++++ b/pengine/test10/inc10.summary +@@ -9,8 +9,8 @@ Online: [ xen-1 xen-3 xen-4 ] + Started: [ xen-1 xen-2 xen-3 xen-4 ] + + Transition Summary: +- * Stop child_DoFencing:1 (xen-2) +- * Stop ocfs2:1 (xen-2) ++ * Stop child_DoFencing:1 (xen-2) due to node availability ++ * Stop ocfs2:1 (xen-2) due to node availability + + Executing cluster transition: + * Pseudo action: DoFencing_stop_0 +diff --git a/pengine/test10/inc12.summary b/pengine/test10/inc12.summary +index 5068b7e..e950c67 100644 +--- a/pengine/test10/inc12.summary ++++ b/pengine/test10/inc12.summary +@@ -38,9 +38,9 @@ Transition Summary: + * Shutdown c001n04 + * Shutdown c001n03 + * Shutdown c001n02 +- * Stop ocf_192.168.100.181 (c001n02) +- * Stop heartbeat_192.168.100.182 (c001n02) +- * Stop ocf_192.168.100.183 (c001n02) ++ * Stop ocf_192.168.100.181 (c001n02) due to node availability ++ * Stop heartbeat_192.168.100.182 (c001n02) due to node availability ++ * Stop ocf_192.168.100.183 (c001n02) due to node availability + * Stop lsb_dummy (c001n04) + * Stop rsc_c001n03 (c001n05) + * Stop rsc_c001n02 (c001n02) +@@ -48,21 +48,21 @@ Transition Summary: + * Stop rsc_c001n05 (c001n05) + * Stop rsc_c001n06 (c001n06) + * Stop rsc_c001n07 (c001n07) +- * Stop child_DoFencing:0 (c001n02) +- * Stop child_DoFencing:1 (c001n04) +- * Stop child_DoFencing:2 (c001n05) +- * Stop child_DoFencing:3 (c001n06) +- * Stop child_DoFencing:4 (c001n07) +- * Stop ocf_msdummy:10 (c001n02) +- * Stop ocf_msdummy:11 (c001n02) +- * Stop ocf_msdummy:2 (c001n04) +- * Stop ocf_msdummy:3 (c001n04) +- * Stop ocf_msdummy:4 (c001n05) +- * Stop ocf_msdummy:5 (c001n05) +- * Stop ocf_msdummy:6 (c001n06) +- * Stop ocf_msdummy:7 (c001n06) +- * Stop ocf_msdummy:8 (c001n07) +- * Stop ocf_msdummy:9 (c001n07) ++ * Stop child_DoFencing:0 (c001n02) due to node availability ++ * Stop child_DoFencing:1 (c001n04) due to node availability ++ * Stop child_DoFencing:2 (c001n05) due to node availability ++ * Stop child_DoFencing:3 (c001n06) due to node availability ++ * Stop child_DoFencing:4 (c001n07) due to node availability ++ * Stop ocf_msdummy:10 (c001n02) due to node availability ++ * Stop ocf_msdummy:11 (c001n02) due to node availability ++ * Stop ocf_msdummy:2 (c001n04) due to node availability ++ * Stop ocf_msdummy:3 (c001n04) due to node availability ++ * Stop ocf_msdummy:4 (c001n05) due to node availability ++ * Stop ocf_msdummy:5 (c001n05) due to node availability ++ * Stop ocf_msdummy:6 (c001n06) due to node availability ++ * Stop ocf_msdummy:7 (c001n06) due to node availability ++ * Stop ocf_msdummy:8 (c001n07) due to node availability ++ * Stop ocf_msdummy:9 (c001n07) due to node availability + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/inc2.summary b/pengine/test10/inc2.summary +index 7c6a9bb..898729f 100644 +--- a/pengine/test10/inc2.summary ++++ b/pengine/test10/inc2.summary +@@ -12,7 +12,7 @@ Online: [ node1 node2 ] + Transition Summary: + * Move child_rsc1:2 (Started node1 -> node2) + * Move child_rsc1:3 (Started node1 -> node2) +- * Stop child_rsc1:4 (node1) ++ * Stop child_rsc1:4 (node1) due to node availability + + Executing cluster transition: + * Resource action: child_rsc1:0 monitor on node2 +diff --git a/pengine/test10/inc3.summary b/pengine/test10/inc3.summary +index adc59e7..955ff3b 100644 +--- a/pengine/test10/inc3.summary ++++ b/pengine/test10/inc3.summary +@@ -18,7 +18,7 @@ Online: [ node1 node2 ] + Transition Summary: + * Move child_rsc1:2 (Started node1 -> node2) + * Move child_rsc1:3 (Started node1 -> node2) +- * Stop child_rsc1:4 (node1) ++ * Stop child_rsc1:4 (node1) due to node availability + * Move child_rsc2:3 (Started node2 -> node1) + * Move child_rsc2:4 (Started node2 -> node1) + +diff --git a/pengine/test10/inc4.summary b/pengine/test10/inc4.summary +index 03947cd..e730360 100644 +--- a/pengine/test10/inc4.summary ++++ b/pengine/test10/inc4.summary +@@ -18,7 +18,7 @@ Online: [ node1 node2 ] + Transition Summary: + * Move child_rsc1:2 (Started node1 -> node2) + * Move child_rsc1:3 (Started node1 -> node2) +- * Stop child_rsc1:4 (node1) ++ * Stop child_rsc1:4 (node1) due to node availability + * Move child_rsc2:3 (Started node2 -> node1) + * Move child_rsc2:4 (Started node2 -> node1) + +diff --git a/pengine/test10/inc6.summary b/pengine/test10/inc6.summary +index cf84c1f..4c754d7 100644 +--- a/pengine/test10/inc6.summary ++++ b/pengine/test10/inc6.summary +@@ -31,8 +31,8 @@ Transition Summary: + * Move child_rsc2:1 (Started node1 -> node2) + * Move child_rsc4:1 (Started node1 -> node2) + * Move child_rsc5:1 (Started node2 -> node1) +- * Restart child_rsc6:0 (Started node1) +- * Restart child_rsc6:1 (Started node2) ++ * Restart child_rsc6:0 (Started node1) due to required rsc5 running ++ * Restart child_rsc6:1 (Started node2) due to required rsc5 running + * Move child_rsc7:1 (Started node2 -> node1) + + Executing cluster transition: +diff --git a/pengine/test10/inc9.summary b/pengine/test10/inc9.summary +index 5a7f123..3c35aee 100644 +--- a/pengine/test10/inc9.summary ++++ b/pengine/test10/inc9.summary +@@ -9,9 +9,9 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop child_rsc1:5 (node1) +- * Stop child_rsc1:6 (node1) +- * Stop child_rsc1:7 (node2) ++ * Stop child_rsc1:5 (node1) due to node availability ++ * Stop child_rsc1:6 (node1) due to node availability ++ * Stop child_rsc1:7 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: rsc1_stop_0 +diff --git a/pengine/test10/interleave-pseudo-stop.summary b/pengine/test10/interleave-pseudo-stop.summary +index 01ce0ec..ee3fa29 100644 +--- a/pengine/test10/interleave-pseudo-stop.summary ++++ b/pengine/test10/interleave-pseudo-stop.summary +@@ -18,10 +18,10 @@ Online: [ node2 ] + + Transition Summary: + * Fence (reboot) node1 'peer is no longer part of the cluster' +- * Stop stonithclone:1 (node1) +- * Stop evmsclone:1 (node1) +- * Stop imagestoreclone:1 (node1) +- * Stop configstoreclone:1 (node1) ++ * Stop stonithclone:1 (node1) due to node availability ++ * Stop evmsclone:1 (node1) due to node availability ++ * Stop imagestoreclone:1 (node1) due to node availability ++ * Stop configstoreclone:1 (node1) due to node availability + + Executing cluster transition: + * Pseudo action: evmscloneset_pre_notify_stop_0 +diff --git a/pengine/test10/interleave-restart.summary b/pengine/test10/interleave-restart.summary +index 7f1b1e4..5ac19b9 100644 +--- a/pengine/test10/interleave-restart.summary ++++ b/pengine/test10/interleave-restart.summary +@@ -14,8 +14,8 @@ Online: [ node1 node2 ] + + Transition Summary: + * Recover evmsclone:1 (Started node1) +- * Restart imagestoreclone:1 (Started node1) +- * Restart configstoreclone:1 (Started node1) ++ * Restart imagestoreclone:1 (Started node1) due to required evmsclone:1 start ++ * Restart configstoreclone:1 (Started node1) due to required evmsclone:1 start + + Executing cluster transition: + * Pseudo action: evmscloneset_pre_notify_stop_0 +diff --git a/pengine/test10/interleave-stop.summary b/pengine/test10/interleave-stop.summary +index b5a1a98..529e33a 100644 +--- a/pengine/test10/interleave-stop.summary ++++ b/pengine/test10/interleave-stop.summary +@@ -13,10 +13,10 @@ Online: [ node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop stonithclone:1 (node1) +- * Stop evmsclone:1 (node1) +- * Stop imagestoreclone:1 (node1) +- * Stop configstoreclone:1 (node1) ++ * Stop stonithclone:1 (node1) due to node availability ++ * Stop evmsclone:1 (node1) due to node availability ++ * Stop imagestoreclone:1 (node1) due to node availability ++ * Stop configstoreclone:1 (node1) due to node availability + + Executing cluster transition: + * Pseudo action: stonithcloneset_stop_0 +diff --git a/pengine/test10/master-7.summary b/pengine/test10/master-7.summary +index 4c58184..58ef275 100644 +--- a/pengine/test10/master-7.summary ++++ b/pengine/test10/master-7.summary +@@ -36,9 +36,9 @@ Transition Summary: + * Move ocf_192.168.100.183 (Started c001n03 -> c001n02) + * Move lsb_dummy (Started c001n02 -> c001n08) + * Move rsc_c001n01 (Started c001n01 -> c001n03) +- * Stop child_DoFencing:0 (c001n01) ++ * Stop child_DoFencing:0 (c001n01) due to node availability + * Demote ocf_msdummy:0 (Master -> Stopped c001n01) +- * Stop ocf_msdummy:4 (c001n01) ++ * Stop ocf_msdummy:4 (c001n01) due to node availability + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/master-8.summary b/pengine/test10/master-8.summary +index e8d90d2..c18e884 100644 +--- a/pengine/test10/master-8.summary ++++ b/pengine/test10/master-8.summary +@@ -36,7 +36,7 @@ Transition Summary: + * Move ocf_192.168.100.183 (Started c001n03 -> c001n02) + * Move lsb_dummy (Started c001n02 -> c001n08) + * Move rsc_c001n01 (Started c001n01 -> c001n03) +- * Stop child_DoFencing:0 (c001n01) ++ * Stop child_DoFencing:0 (c001n01) due to node availability + * Demote ocf_msdummy:0 (Master -> Slave c001n01 - blocked) + * Move ocf_msdummy:0 (Slave c001n01 -> c001n03) + +diff --git a/pengine/test10/master-9.summary b/pengine/test10/master-9.summary +index c1cd4a1..ef59602 100644 +--- a/pengine/test10/master-9.summary ++++ b/pengine/test10/master-9.summary +@@ -31,18 +31,18 @@ Online: [ ibm1 va1 ] + + Transition Summary: + * Shutdown ibm1 +- * Start DcIPaddr (va1 - blocked) +- * Start ocf_127.0.0.11 (va1 - blocked) +- * Start heartbeat_127.0.0.12 (va1 - blocked) +- * Start ocf_127.0.0.13 (va1 - blocked) +- * Start lsb_dummy (va1 - blocked) +- * Start rsc_sgi2 (va1 - blocked) +- * Start rsc_ibm1 (va1 - blocked) +- * Start rsc_va1 (va1 - blocked) +- * Start rsc_test02 (va1 - blocked) +- * Stop child_DoFencing:1 (ibm1) +- * Start ocf_msdummy:0 (va1 - blocked) +- * Start ocf_msdummy:1 (va1 - blocked) ++ * Start DcIPaddr (va1 - blocked) due to no quorum ++ * Start ocf_127.0.0.11 (va1 - blocked) due to no quorum ++ * Start heartbeat_127.0.0.12 (va1 - blocked) due to no quorum ++ * Start ocf_127.0.0.13 (va1 - blocked) due to no quorum ++ * Start lsb_dummy (va1 - blocked) due to no quorum ++ * Start rsc_sgi2 (va1 - blocked) due to no quorum ++ * Start rsc_ibm1 (va1 - blocked) due to no quorum ++ * Start rsc_va1 (va1 - blocked) due to no quorum ++ * Start rsc_test02 (va1 - blocked) due to no quorum ++ * Stop child_DoFencing:1 (ibm1) due to node availability ++ * Start ocf_msdummy:0 (va1 - blocked) due to no quorum ++ * Start ocf_msdummy:1 (va1 - blocked) due to no quorum + + Executing cluster transition: + * Resource action: child_DoFencing:1 monitor on va1 +diff --git a/pengine/test10/master-asymmetrical-order.summary b/pengine/test10/master-asymmetrical-order.summary +index 1ff2836..d09f62e 100644 +--- a/pengine/test10/master-asymmetrical-order.summary ++++ b/pengine/test10/master-asymmetrical-order.summary +@@ -12,7 +12,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Demote rsc1:0 (Master -> Stopped node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +diff --git a/pengine/test10/master-failed-demote-2.summary b/pengine/test10/master-failed-demote-2.summary +index 847e0a1..086d02e 100644 +--- a/pengine/test10/master-failed-demote-2.summary ++++ b/pengine/test10/master-failed-demote-2.summary +@@ -11,7 +11,7 @@ Online: [ dl380g5a dl380g5b ] + stateful-2:1 (ocf::heartbeat:Stateful): Slave dl380g5a + + Transition Summary: +- * Stop stateful-1:0 (dl380g5b) ++ * Stop stateful-1:0 (dl380g5b) due to node availability + * Promote stateful-1:1 (Slave -> Master dl380g5a) + * Promote stateful-2:1 (Slave -> Master dl380g5a) + +diff --git a/pengine/test10/master-failed-demote.summary b/pengine/test10/master-failed-demote.summary +index cc3fbee..0f6c410 100644 +--- a/pengine/test10/master-failed-demote.summary ++++ b/pengine/test10/master-failed-demote.summary +@@ -11,7 +11,7 @@ Online: [ dl380g5a dl380g5b ] + stateful-2:1 (ocf::heartbeat:Stateful): Slave dl380g5a + + Transition Summary: +- * Stop stateful-1:0 (dl380g5b) ++ * Stop stateful-1:0 (dl380g5b) due to node availability + * Promote stateful-1:1 (Slave -> Master dl380g5a) + * Promote stateful-2:1 (Slave -> Master dl380g5a) + +diff --git a/pengine/test10/master-pseudo.summary b/pengine/test10/master-pseudo.summary +index 4ac7605..2ee2d03 100644 +--- a/pengine/test10/master-pseudo.summary ++++ b/pengine/test10/master-pseudo.summary +@@ -12,7 +12,7 @@ Online: [ sambuca.linbit ] + + Transition Summary: + * Start ip_float_right (sambuca.linbit) +- * Restart drbd_float:0 (Slave sambuca.linbit) ++ * Restart drbd_float:0 (Slave sambuca.linbit) due to required ip_float_right start + * Promote drbd_float:0 (Slave -> Master sambuca.linbit) + * Start ip_nfs (sambuca.linbit) + +diff --git a/pengine/test10/master-stop.summary b/pengine/test10/master-stop.summary +index cbe6683..8b802d4 100644 +--- a/pengine/test10/master-stop.summary ++++ b/pengine/test10/master-stop.summary +@@ -6,7 +6,7 @@ Online: [ node1 node2 node3 ] + Slaves: [ node1 node2 node3 ] + + Transition Summary: +- * Stop dummy:2 (node3) ++ * Stop dummy:2 (node3) due to node availability + + Executing cluster transition: + * Pseudo action: m_stop_0 +diff --git a/pengine/test10/migrate-5.summary b/pengine/test10/migrate-5.summary +index 9a98330..a9d5771 100644 +--- a/pengine/test10/migrate-5.summary ++++ b/pengine/test10/migrate-5.summary +@@ -9,7 +9,7 @@ Online: [ dom0-01 ] + + Transition Summary: + * Migrate domU-test01 (Started dom0-02 -> dom0-01) +- * Stop dom0-iscsi1-cnx1:1 (dom0-02) ++ * Stop dom0-iscsi1-cnx1:1 (dom0-02) due to node availability + + Executing cluster transition: + * Resource action: domU-test01 migrate_to on dom0-02 +diff --git a/pengine/test10/migrate-both-vms.summary b/pengine/test10/migrate-both-vms.summary +index 531fd4c..5b60727 100644 +--- a/pengine/test10/migrate-both-vms.summary ++++ b/pengine/test10/migrate-both-vms.summary +@@ -22,16 +22,16 @@ Online: [ cvmh01 cvmh02 ] + vm-swbuildsl6 (ocf::ccni:xcatVirtualDomain): Started cvmh04 + + Transition Summary: +- * Stop fs-libvirt-VM-xcm:0 (cvmh04) +- * Stop fs-libvirt-VM-xcm:2 (cvmh03) +- * Stop p-watch-ib0:0 (cvmh04) +- * Stop p-watch-ib0:2 (cvmh03) +- * Stop p-fs-gpfs:0 (cvmh04) +- * Stop p-fs-gpfs:2 (cvmh03) +- * Stop p-libvirtd:0 (cvmh04) +- * Stop p-libvirtd:2 (cvmh03) +- * Stop fs-bind-libvirt-VM-cvmh:0 (cvmh04) +- * Stop fs-bind-libvirt-VM-cvmh:2 (cvmh03) ++ * Stop fs-libvirt-VM-xcm:0 (cvmh04) due to node availability ++ * Stop fs-libvirt-VM-xcm:2 (cvmh03) due to node availability ++ * Stop p-watch-ib0:0 (cvmh04) due to node availability ++ * Stop p-watch-ib0:2 (cvmh03) due to node availability ++ * Stop p-fs-gpfs:0 (cvmh04) due to node availability ++ * Stop p-fs-gpfs:2 (cvmh03) due to node availability ++ * Stop p-libvirtd:0 (cvmh04) due to node availability ++ * Stop p-libvirtd:2 (cvmh03) due to node availability ++ * Stop fs-bind-libvirt-VM-cvmh:0 (cvmh04) due to node availability ++ * Stop fs-bind-libvirt-VM-cvmh:2 (cvmh03) due to node availability + * Migrate vm-compute-test (Started cvmh03 -> cvmh01) + * Migrate vm-swbuildsl6 (Started cvmh04 -> cvmh02) + +diff --git a/pengine/test10/migrate-fencing.summary b/pengine/test10/migrate-fencing.summary +index 7468fe2..cfd4eb8 100644 +--- a/pengine/test10/migrate-fencing.summary ++++ b/pengine/test10/migrate-fencing.summary +@@ -23,14 +23,14 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + + Transition Summary: + * Fence (reboot) pcmk-4 'termination was requested' +- * Stop FencingChild:0 (pcmk-4) ++ * Stop FencingChild:0 (pcmk-4) due to node availability + * Move r192.168.101.181 (Started pcmk-4 -> pcmk-1) + * Move r192.168.101.182 (Started pcmk-4 -> pcmk-1) + * Move r192.168.101.183 (Started pcmk-4 -> pcmk-1) + * Move rsc_pcmk-4 (Started pcmk-4 -> pcmk-2) + * Move lsb-dummy (Started pcmk-4 -> pcmk-1) + * Migrate migrator (Started pcmk-1 -> pcmk-3) +- * Stop ping-1:0 (pcmk-4) ++ * Stop ping-1:0 (pcmk-4) due to node availability + * Demote stateful-1:0 (Master -> Stopped pcmk-4) + * Promote stateful-1:1 (Slave -> Master pcmk-1) + +diff --git a/pengine/test10/migrate-shutdown.summary b/pengine/test10/migrate-shutdown.summary +index 630d58d..b9aa5b2 100644 +--- a/pengine/test10/migrate-shutdown.summary ++++ b/pengine/test10/migrate-shutdown.summary +@@ -27,20 +27,20 @@ Transition Summary: + * Shutdown pcmk-2 + * Shutdown pcmk-1 + * Stop Fencing (pcmk-1) +- * Stop r192.168.122.105 (pcmk-2) +- * Stop r192.168.122.106 (pcmk-2) +- * Stop r192.168.122.107 (pcmk-2) ++ * Stop r192.168.122.105 (pcmk-2) due to node availability ++ * Stop r192.168.122.106 (pcmk-2) due to node availability ++ * Stop r192.168.122.107 (pcmk-2) due to node availability + * Stop rsc_pcmk-1 (pcmk-1) + * Stop rsc_pcmk-2 (pcmk-2) + * Stop rsc_pcmk-4 (pcmk-4) + * Stop lsb-dummy (pcmk-2) + * Stop migrator (pcmk-1) +- * Stop ping-1:0 (pcmk-1) +- * Stop ping-1:1 (pcmk-2) +- * Stop ping-1:2 (pcmk-4) +- * Stop stateful-1:0 (pcmk-1) ++ * Stop ping-1:0 (pcmk-1) due to node availability ++ * Stop ping-1:1 (pcmk-2) due to node availability ++ * Stop ping-1:2 (pcmk-4) due to node availability ++ * Stop stateful-1:0 (pcmk-1) due to node availability + * Demote stateful-1:1 (Master -> Stopped pcmk-2) +- * Stop stateful-1:2 (pcmk-4) ++ * Stop stateful-1:2 (pcmk-4) due to node availability + + Executing cluster transition: + * Resource action: Fencing stop on pcmk-1 +diff --git a/pengine/test10/migrate-stop-complex.summary b/pengine/test10/migrate-stop-complex.summary +index e8f148c..04dce1a 100644 +--- a/pengine/test10/migrate-stop-complex.summary ++++ b/pengine/test10/migrate-stop-complex.summary +@@ -13,8 +13,8 @@ Online: [ dom0-01 ] + Transition Summary: + * Move top (Started dom0-02 -> dom0-01) + * Migrate domU-test01 (Started dom0-02 -> dom0-01) +- * Stop dom0-iscsi1-cnx1:1 (dom0-02) +- * Stop bottom:1 (dom0-02) ++ * Stop dom0-iscsi1-cnx1:1 (dom0-02) due to node availability ++ * Stop bottom:1 (dom0-02) due to node availability + + Executing cluster transition: + * Resource action: top stop on dom0-02 +diff --git a/pengine/test10/migrate-stop-start-complex.summary b/pengine/test10/migrate-stop-start-complex.summary +index 162f73b..5408bb6 100644 +--- a/pengine/test10/migrate-stop-start-complex.summary ++++ b/pengine/test10/migrate-stop-start-complex.summary +@@ -13,7 +13,7 @@ Online: [ dom0-01 ] + + Transition Summary: + * Migrate domU-test01 (Started dom0-02 -> dom0-01) +- * Stop dom0-iscsi1-cnx1:1 (dom0-02) ++ * Stop dom0-iscsi1-cnx1:1 (dom0-02) due to node availability + * Move bottom:0 (Started dom0-02 -> dom0-01) + + Executing cluster transition: +diff --git a/pengine/test10/migrate-stop.summary b/pengine/test10/migrate-stop.summary +index 9a98330..a9d5771 100644 +--- a/pengine/test10/migrate-stop.summary ++++ b/pengine/test10/migrate-stop.summary +@@ -9,7 +9,7 @@ Online: [ dom0-01 ] + + Transition Summary: + * Migrate domU-test01 (Started dom0-02 -> dom0-01) +- * Stop dom0-iscsi1-cnx1:1 (dom0-02) ++ * Stop dom0-iscsi1-cnx1:1 (dom0-02) due to node availability + + Executing cluster transition: + * Resource action: domU-test01 migrate_to on dom0-02 +diff --git a/pengine/test10/monitor-recovery.summary b/pengine/test10/monitor-recovery.summary +index cae6d07..d8a13fe 100644 +--- a/pengine/test10/monitor-recovery.summary ++++ b/pengine/test10/monitor-recovery.summary +@@ -11,7 +11,7 @@ OFFLINE: [ CSE-2 ] + Stopped: [ CSE-2 ] + + Transition Summary: +- * Stop d_tomcat:0 (CSE-1) ++ * Stop d_tomcat:0 (CSE-1) due to node availability + + Executing cluster transition: + * Pseudo action: cl_tomcat_stop_0 +diff --git a/pengine/test10/notify-0.summary b/pengine/test10/notify-0.summary +index 2d95b26..69112da 100644 +--- a/pengine/test10/notify-0.summary ++++ b/pengine/test10/notify-0.summary +@@ -12,7 +12,7 @@ OFFLINE: [ node2 ] + + Transition Summary: + * Start child_rsc1:1 (node1) +- * Stop child_rsc2:0 (node1) ++ * Stop child_rsc2:0 (node1) due to node availability + + Executing cluster transition: + * Resource action: child_rsc1:1 monitor on node1 +diff --git a/pengine/test10/notify-1.summary b/pengine/test10/notify-1.summary +index d670be7..0a734a6 100644 +--- a/pengine/test10/notify-1.summary ++++ b/pengine/test10/notify-1.summary +@@ -12,7 +12,7 @@ OFFLINE: [ node2 ] + + Transition Summary: + * Start child_rsc1:1 (node1) +- * Stop child_rsc2:0 (node1) ++ * Stop child_rsc2:0 (node1) due to node availability + + Executing cluster transition: + * Resource action: child_rsc1:1 monitor on node1 +diff --git a/pengine/test10/notify-2.summary b/pengine/test10/notify-2.summary +index d670be7..0a734a6 100644 +--- a/pengine/test10/notify-2.summary ++++ b/pengine/test10/notify-2.summary +@@ -12,7 +12,7 @@ OFFLINE: [ node2 ] + + Transition Summary: + * Start child_rsc1:1 (node1) +- * Stop child_rsc2:0 (node1) ++ * Stop child_rsc2:0 (node1) due to node availability + + Executing cluster transition: + * Resource action: child_rsc1:1 monitor on node1 +diff --git a/pengine/test10/notify-3.summary b/pengine/test10/notify-3.summary +index efc3a07..03a3d4e 100644 +--- a/pengine/test10/notify-3.summary ++++ b/pengine/test10/notify-3.summary +@@ -11,7 +11,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Move child_rsc1:1 (Started node2 -> node1) +- * Stop child_rsc2:0 (node1) ++ * Stop child_rsc2:0 (node1) due to node availability + + Executing cluster transition: + * Resource action: child_rsc1:0 monitor on node2 +diff --git a/pengine/test10/novell-252693.summary b/pengine/test10/novell-252693.summary +index 23f0632..0682119 100644 +--- a/pengine/test10/novell-252693.summary ++++ b/pengine/test10/novell-252693.summary +@@ -16,11 +16,11 @@ Online: [ node1 node2 ] + + Transition Summary: + * Shutdown node1 +- * Stop stonithclone:1 (node1) +- * Stop evmsdclone:1 (node1) +- * Stop evmsclone:1 (node1) +- * Stop imagestoreclone:1 (node1) +- * Stop configstoreclone:1 (node1) ++ * Stop stonithclone:1 (node1) due to node availability ++ * Stop evmsdclone:1 (node1) due to node availability ++ * Stop evmsclone:1 (node1) due to node availability ++ * Stop imagestoreclone:1 (node1) due to node availability ++ * Stop configstoreclone:1 (node1) due to node availability + * Migrate sles10 (Started node1 -> node2) + + Executing cluster transition: +diff --git a/pengine/test10/one-or-more-1.summary b/pengine/test10/one-or-more-1.summary +index f26919c..af31a46 100644 +--- a/pengine/test10/one-or-more-1.summary ++++ b/pengine/test10/one-or-more-1.summary +@@ -10,9 +10,9 @@ OFFLINE: [ fc16-builder2 ] + D (ocf::pacemaker:Dummy): Stopped + + Transition Summary: +- * Start B (fc16-builder - blocked) +- * Start C (fc16-builder - blocked) +- * Start D (fc16-builder - blocked) ++ * Start B (fc16-builder - blocked) due to unrunnable A start ++ * Start C (fc16-builder - blocked) due to unrunnable A start ++ * Start D (fc16-builder - blocked) due to unrunnable one-or-more:require-all-set-1 + + Executing cluster transition: + * Resource action: A monitor on fc16-builder +diff --git a/pengine/test10/one-or-more-3.summary b/pengine/test10/one-or-more-3.summary +index f0ee084..abf4081 100644 +--- a/pengine/test10/one-or-more-3.summary ++++ b/pengine/test10/one-or-more-3.summary +@@ -11,7 +11,7 @@ OFFLINE: [ fc16-builder2 ] + + Transition Summary: + * Start A (fc16-builder) +- * Start D (fc16-builder - blocked) ++ * Start D (fc16-builder - blocked) due to unrunnable one-or-more:require-all-set-1 + + Executing cluster transition: + * Resource action: A monitor on fc16-builder +diff --git a/pengine/test10/one-or-more-6.summary b/pengine/test10/one-or-more-6.summary +index 18cab2e..665ac52 100644 +--- a/pengine/test10/one-or-more-6.summary ++++ b/pengine/test10/one-or-more-6.summary +@@ -9,7 +9,7 @@ OFFLINE: [ fc16-builder2 ] + C (ocf::pacemaker:Dummy): Started fc16-builder + + Transition Summary: +- * Stop B (fc16-builder) ++ * Stop B (fc16-builder) due to node availability + + Executing cluster transition: + * Resource action: B stop on fc16-builder +diff --git a/pengine/test10/one-or-more-7.summary b/pengine/test10/one-or-more-7.summary +index 4322db3..89b3416 100644 +--- a/pengine/test10/one-or-more-7.summary ++++ b/pengine/test10/one-or-more-7.summary +@@ -9,7 +9,7 @@ OFFLINE: [ fc16-builder2 ] + C (ocf::pacemaker:Dummy): Started fc16-builder ( disabled ) + + Transition Summary: +- * Stop C (fc16-builder) ++ * Stop C (fc16-builder) due to node availability + + Executing cluster transition: + * Resource action: C stop on fc16-builder +diff --git a/pengine/test10/order-mandatory.summary b/pengine/test10/order-mandatory.summary +index a1d2b4b..021d8fe 100644 +--- a/pengine/test10/order-mandatory.summary ++++ b/pengine/test10/order-mandatory.summary +@@ -9,8 +9,8 @@ Online: [ node1 ] + + Transition Summary: + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) +- * Stop rsc4 (Started node1) ++ * Restart rsc2 (Started node1) due to required rsc1 start ++ * Stop rsc4 (Started node1) due to unrunnable rsc3 start + + Executing cluster transition: + * Resource action: rsc1 start on node1 +diff --git a/pengine/test10/order-required.summary b/pengine/test10/order-required.summary +index a1d2b4b..021d8fe 100644 +--- a/pengine/test10/order-required.summary ++++ b/pengine/test10/order-required.summary +@@ -9,8 +9,8 @@ Online: [ node1 ] + + Transition Summary: + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) +- * Stop rsc4 (Started node1) ++ * Restart rsc2 (Started node1) due to required rsc1 start ++ * Stop rsc4 (Started node1) due to unrunnable rsc3 start + + Executing cluster transition: + * Resource action: rsc1 start on node1 +diff --git a/pengine/test10/order-wrong-kind.summary b/pengine/test10/order-wrong-kind.summary +index 8525b52..c498caf 100644 +--- a/pengine/test10/order-wrong-kind.summary ++++ b/pengine/test10/order-wrong-kind.summary +@@ -10,7 +10,7 @@ Online: [ node1 ] + + Transition Summary: + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) ++ * Restart rsc2 (Started node1) due to required rsc1 start + + Executing cluster transition: + * Resource action: rsc1 start on node1 +diff --git a/pengine/test10/order7.summary b/pengine/test10/order7.summary +index 74c4b60..6648bff 100644 +--- a/pengine/test10/order7.summary ++++ b/pengine/test10/order7.summary +@@ -13,7 +13,7 @@ Transition Summary: + * Start rsc2 (node1) + * Start rsc3 (node1) + * Start rscB (node1) +- * Start rscC (node1 - blocked) ++ * Start rscC (node1 - blocked) due to unrunnable rscA start + + Executing cluster transition: + * Resource action: rsc2 monitor on node1 +diff --git a/pengine/test10/order_constraint_stops_master.summary b/pengine/test10/order_constraint_stops_master.summary +index f1125a1..d3d8891 100644 +--- a/pengine/test10/order_constraint_stops_master.summary ++++ b/pengine/test10/order_constraint_stops_master.summary +@@ -8,8 +8,8 @@ Online: [ fc16-builder fc16-builder2 ] + NATIVE_RSC_B (ocf::pacemaker:Dummy): Started fc16-builder2 ( disabled ) + + Transition Summary: +- * Stop NATIVE_RSC_A:0 (Master fc16-builder) +- * Stop NATIVE_RSC_B (fc16-builder2) ++ * Stop NATIVE_RSC_A:0 (Master fc16-builder) due to required NATIVE_RSC_B start ++ * Stop NATIVE_RSC_B (fc16-builder2) due to node availability + + Executing cluster transition: + * Pseudo action: MASTER_RSC_A_pre_notify_demote_0 +diff --git a/pengine/test10/order_constraint_stops_slave.summary b/pengine/test10/order_constraint_stops_slave.summary +index 843d3b6..896c9c3 100644 +--- a/pengine/test10/order_constraint_stops_slave.summary ++++ b/pengine/test10/order_constraint_stops_slave.summary +@@ -9,8 +9,8 @@ OFFLINE: [ fc16-builder2 ] + NATIVE_RSC_B (ocf::pacemaker:Dummy): Started fc16-builder ( disabled ) + + Transition Summary: +- * Stop NATIVE_RSC_A:0 (Slave fc16-builder) +- * Stop NATIVE_RSC_B (fc16-builder) ++ * Stop NATIVE_RSC_A:0 (Slave fc16-builder) due to required NATIVE_RSC_B start ++ * Stop NATIVE_RSC_B (fc16-builder) due to node availability + + Executing cluster transition: + * Pseudo action: MASTER_RSC_A_pre_notify_stop_0 +diff --git a/pengine/test10/ordered-set-basic-startup.summary b/pengine/test10/ordered-set-basic-startup.summary +index 182dd08..cfa8f8b 100644 +--- a/pengine/test10/ordered-set-basic-startup.summary ++++ b/pengine/test10/ordered-set-basic-startup.summary +@@ -12,10 +12,10 @@ OFFLINE: [ fc16-builder2 ] + F (ocf::pacemaker:Dummy): Stopped + + Transition Summary: +- * Start A (fc16-builder - blocked) ++ * Start A (fc16-builder - blocked) due to unrunnable C start + * Start B (fc16-builder) +- * Start E (fc16-builder - blocked) +- * Start F (fc16-builder - blocked) ++ * Start E (fc16-builder - blocked) due to unrunnable A start ++ * Start F (fc16-builder - blocked) due to unrunnable D start + + Executing cluster transition: + * Resource action: A monitor on fc16-builder +diff --git a/pengine/test10/ordered-set-natural.summary b/pengine/test10/ordered-set-natural.summary +index a3ba453..1888d66 100644 +--- a/pengine/test10/ordered-set-natural.summary ++++ b/pengine/test10/ordered-set-natural.summary +@@ -21,12 +21,12 @@ Online: [ node1 node2 ] + dummy2-5 (ocf::heartbeat:Dummy): Stopped + + Transition Summary: +- * Start dummy1-1 (node1 - blocked) +- * Start dummy1-2 (node1 - blocked) +- * Start dummy2-1 (node2 - blocked) +- * Start dummy2-2 (node2 - blocked) +- * Start dummy3-4 (node1 - blocked) +- * Start dummy3-5 (node1 - blocked) ++ * Start dummy1-1 (node1 - blocked) due to no quorum ++ * Start dummy1-2 (node1 - blocked) due to no quorum ++ * Start dummy2-1 (node2 - blocked) due to no quorum ++ * Start dummy2-2 (node2 - blocked) due to no quorum ++ * Start dummy3-4 (node1 - blocked) due to no quorum ++ * Start dummy3-5 (node1 - blocked) due to no quorum + + Executing cluster transition: + +diff --git a/pengine/test10/probe-2.summary b/pengine/test10/probe-2.summary +index e8a2269..05588c8 100644 +--- a/pengine/test10/probe-2.summary ++++ b/pengine/test10/probe-2.summary +@@ -36,13 +36,13 @@ Transition Summary: + * Move fs_mysql (Started wc02 -> wc01) + * Move intip_sql (Started wc02 -> wc01) + * Move mysql-server (Started wc02 -> wc01) +- * Stop drbd_www:1 (wc02) +- * Stop nfs-common:1 (wc02) +- * Stop mysql-proxy:1 (wc02) +- * Stop fs_www:1 (wc02) +- * Stop apache2:1 (wc02) ++ * Stop drbd_www:1 (wc02) due to node availability ++ * Stop nfs-common:1 (wc02) due to node availability ++ * Stop mysql-proxy:1 (wc02) due to node availability ++ * Stop fs_www:1 (wc02) due to node availability ++ * Stop apache2:1 (wc02) due to node availability + * Restart stonith_rackpdu:0 (Started wc01) +- * Stop stonith_rackpdu:1 (wc02) ++ * Stop stonith_rackpdu:1 (wc02) due to node availability + + Executing cluster transition: + * Resource action: drbd_mysql:0 cancel=10000 on wc01 +diff --git a/pengine/test10/quorum-2.summary b/pengine/test10/quorum-2.summary +index 9854fda..1dce96c 100644 +--- a/pengine/test10/quorum-2.summary ++++ b/pengine/test10/quorum-2.summary +@@ -8,7 +8,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Move rsc2 (Started node1 -> node2) +- * Start rsc3 (node1 - blocked) ++ * Start rsc3 (node1 - blocked) due to quorum freeze + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/quorum-3.summary b/pengine/test10/quorum-3.summary +index 2ff7561..e8e4bf3 100644 +--- a/pengine/test10/quorum-3.summary ++++ b/pengine/test10/quorum-3.summary +@@ -7,9 +7,9 @@ Online: [ node1 node2 ] + rsc3 (heartbeat:apache): Stopped + + Transition Summary: +- * Stop rsc1 (Started node1) +- * Stop rsc2 (Started node1) +- * Start rsc3 (node1 - blocked) ++ * Stop rsc1 (Started node1) due to no quorum ++ * Stop rsc2 (Started node1) due to no quorum ++ * Start rsc3 (node1 - blocked) due to no quorum + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/rec-node-10.summary b/pengine/test10/rec-node-10.summary +index 6daec5c..503dd0d 100644 +--- a/pengine/test10/rec-node-10.summary ++++ b/pengine/test10/rec-node-10.summary +@@ -8,9 +8,9 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + + Transition Summary: +- * Start stonith-1 (node2 - blocked) +- * Stop rsc1 (Started node1 - blocked) +- * Stop rsc2 (Started node1 - blocked) ++ * Start stonith-1 (node2 - blocked) due to no quorum ++ * Stop rsc1 (Started node1 - blocked) due to no quorum ++ * Stop rsc2 (Started node1 - blocked) due to no quorum + + Executing cluster transition: + * Resource action: stonith-1 monitor on node2 +diff --git a/pengine/test10/rec-node-13.summary b/pengine/test10/rec-node-13.summary +index 9873757..de2fa28 100644 +--- a/pengine/test10/rec-node-13.summary ++++ b/pengine/test10/rec-node-13.summary +@@ -35,7 +35,7 @@ OFFLINE: [ c001n03 c001n05 ] + + Transition Summary: + * Fence (reboot) c001n04 'ocf_msdummy:6 failed there' +- * Stop ocf_msdummy:6 (c001n04) ++ * Stop ocf_msdummy:6 (c001n04) due to node availability + + Executing cluster transition: + * Fencing c001n04 (reboot) +diff --git a/pengine/test10/rec-node-8.summary b/pengine/test10/rec-node-8.summary +index 4cd6724..1a7232e 100644 +--- a/pengine/test10/rec-node-8.summary ++++ b/pengine/test10/rec-node-8.summary +@@ -9,10 +9,10 @@ Online: [ node2 ] + rsc3 (heartbeat:apache): Stopped + + Transition Summary: +- * Start stonith-1 (node2 - blocked) ++ * Start stonith-1 (node2 - blocked) due to quorum freeze + * Stop rsc1 (Started node1 - blocked) + * Stop rsc2 (Started node1 - blocked) +- * Start rsc3 (node2 - blocked) ++ * Start rsc3 (node2 - blocked) due to quorum freeze + + Executing cluster transition: + * Resource action: stonith-1 monitor on node2 +diff --git a/pengine/test10/rec-node-9.summary b/pengine/test10/rec-node-9.summary +index 09ca805..7f6d8ae 100644 +--- a/pengine/test10/rec-node-9.summary ++++ b/pengine/test10/rec-node-9.summary +@@ -7,8 +7,8 @@ OFFLINE: [ node1 ] + rsc2 (heartbeat:apache): Stopped + + Transition Summary: +- * Start rsc1 (node2 - blocked) +- * Start rsc2 (node2 - blocked) ++ * Start rsc1 (node2 - blocked) due to no quorum ++ * Start rsc2 (node2 - blocked) due to no quorum + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/rec-rsc-9.summary b/pengine/test10/rec-rsc-9.summary +index 1c06087..02a2fe5 100644 +--- a/pengine/test10/rec-rsc-9.summary ++++ b/pengine/test10/rec-rsc-9.summary +@@ -9,8 +9,8 @@ Online: [ node1 ] + rsc3 (heartbeat:apache): FAILED node1 + + Transition Summary: +- * Restart rsc1 (Started node1) +- * Restart rsc2 (Started node1) ++ * Restart rsc1 (Started node1) due to required bar running ++ * Restart rsc2 (Started node1) due to required bar running + * Recover rsc3 (Started node1) + + Executing cluster transition: +diff --git a/pengine/test10/remote-disable.summary b/pengine/test10/remote-disable.summary +index 69b4b62..2ec1748 100644 +--- a/pengine/test10/remote-disable.summary ++++ b/pengine/test10/remote-disable.summary +@@ -12,7 +12,7 @@ RemoteOnline: [ remote1 ] + FAKE4 (ocf::heartbeat:Dummy): Started 18node1 + + Transition Summary: +- * Stop remote1 (18builder) ++ * Stop remote1 (18builder) due to node availability + * Stop FAKE2 (remote1) + + Executing cluster transition: +diff --git a/pengine/test10/remote-orphaned.summary b/pengine/test10/remote-orphaned.summary +index 84762b2..2b9add7 100644 +--- a/pengine/test10/remote-orphaned.summary ++++ b/pengine/test10/remote-orphaned.summary +@@ -26,8 +26,8 @@ RemoteOnline: [ remote1 ] + + Transition Summary: + * Move rsc_18node2 (Started remote1 -> 18node1) +- * Stop ping-1:2 (remote1) +- * Stop remote1 (18node1) ++ * Stop ping-1:2 (remote1) due to node availability ++ * Stop remote1 (18node1) due to node availability + + Executing cluster transition: + * Resource action: rsc_18node2 stop on remote1 +diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary +index 35f3d58..a8dac38 100644 +--- a/pengine/test10/remote-recover-all.summary ++++ b/pengine/test10/remote-recover-all.summary +@@ -41,16 +41,16 @@ Transition Summary: + * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' + * Fence (reboot) galera-2 'resources are active and the connection is unrecoverable' + * Fence (reboot) controller-1 'peer is no longer part of the cluster' +- * Stop messaging-1 (controller-1) ++ * Stop messaging-1 (controller-1) due to node availability + * Move galera-0 (Started controller-1 -> controller-2) +- * Stop galera-2 (controller-1) +- * Stop rabbitmq:2 (messaging-1) ++ * Stop galera-2 (controller-1) due to node availability ++ * Stop rabbitmq:2 (messaging-1) due to node availability + * Demote galera:1 (Master -> Stopped galera-2) +- * Stop redis:0 (controller-1) ++ * Stop redis:0 (controller-1) due to node availability + * Move ip-172.17.1.14 (Started controller-1 -> controller-2) + * Move ip-172.17.1.17 (Started controller-1 -> controller-2) + * Move ip-172.17.4.11 (Started controller-1 -> controller-2) +- * Stop haproxy:0 (controller-1) ++ * Stop haproxy:0 (controller-1) due to node availability + * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) + * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) + * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-recover-connection.summary b/pengine/test10/remote-recover-connection.summary +index e0a9d0e..7b5b5fc 100644 +--- a/pengine/test10/remote-recover-connection.summary ++++ b/pengine/test10/remote-recover-connection.summary +@@ -42,11 +42,11 @@ Transition Summary: + * Move messaging-1 (Started controller-1 -> controller-2) + * Move galera-0 (Started controller-1 -> controller-2) + * Move galera-2 (Started controller-1 -> controller-2) +- * Stop redis:0 (controller-1) ++ * Stop redis:0 (controller-1) due to node availability + * Move ip-172.17.1.14 (Started controller-1 -> controller-2) + * Move ip-172.17.1.17 (Started controller-1 -> controller-2) + * Move ip-172.17.4.11 (Started controller-1 -> controller-2) +- * Stop haproxy:0 (controller-1) ++ * Stop haproxy:0 (controller-1) due to node availability + * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) + * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) + * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary +index 19b5181..4383e81 100644 +--- a/pengine/test10/remote-recover-no-resources.summary ++++ b/pengine/test10/remote-recover-no-resources.summary +@@ -40,15 +40,15 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + Transition Summary: + * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' + * Fence (reboot) controller-1 'peer is no longer part of the cluster' +- * Stop messaging-1 (controller-1) ++ * Stop messaging-1 (controller-1) due to node availability + * Move galera-0 (Started controller-1 -> controller-2) + * Stop galera-2 (controller-1) +- * Stop rabbitmq:2 (messaging-1) +- * Stop redis:0 (controller-1) ++ * Stop rabbitmq:2 (messaging-1) due to node availability ++ * Stop redis:0 (controller-1) due to node availability + * Move ip-172.17.1.14 (Started controller-1 -> controller-2) + * Move ip-172.17.1.17 (Started controller-1 -> controller-2) + * Move ip-172.17.4.11 (Started controller-1 -> controller-2) +- * Stop haproxy:0 (controller-1) ++ * Stop haproxy:0 (controller-1) due to node availability + * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) + * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) + * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary +index 772a699..3ffb5f4 100644 +--- a/pengine/test10/remote-recover-unknown.summary ++++ b/pengine/test10/remote-recover-unknown.summary +@@ -41,15 +41,15 @@ Transition Summary: + * Fence (reboot) galera-2 'resources are in an unknown state and the connection is unrecoverable' + * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' + * Fence (reboot) controller-1 'peer is no longer part of the cluster' +- * Stop messaging-1 (controller-1) ++ * Stop messaging-1 (controller-1) due to node availability + * Move galera-0 (Started controller-1 -> controller-2) +- * Stop galera-2 (controller-1) +- * Stop rabbitmq:2 (messaging-1) +- * Stop redis:0 (controller-1) ++ * Stop galera-2 (controller-1) due to node availability ++ * Stop rabbitmq:2 (messaging-1) due to node availability ++ * Stop redis:0 (controller-1) due to node availability + * Move ip-172.17.1.14 (Started controller-1 -> controller-2) + * Move ip-172.17.1.17 (Started controller-1 -> controller-2) + * Move ip-172.17.4.11 (Started controller-1 -> controller-2) +- * Stop haproxy:0 (controller-1) ++ * Stop haproxy:0 (controller-1) due to node availability + * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) + * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) + * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/remote-recovery.summary b/pengine/test10/remote-recovery.summary +index e0a9d0e..7b5b5fc 100644 +--- a/pengine/test10/remote-recovery.summary ++++ b/pengine/test10/remote-recovery.summary +@@ -42,11 +42,11 @@ Transition Summary: + * Move messaging-1 (Started controller-1 -> controller-2) + * Move galera-0 (Started controller-1 -> controller-2) + * Move galera-2 (Started controller-1 -> controller-2) +- * Stop redis:0 (controller-1) ++ * Stop redis:0 (controller-1) due to node availability + * Move ip-172.17.1.14 (Started controller-1 -> controller-2) + * Move ip-172.17.1.17 (Started controller-1 -> controller-2) + * Move ip-172.17.4.11 (Started controller-1 -> controller-2) +- * Stop haproxy:0 (controller-1) ++ * Stop haproxy:0 (controller-1) due to node availability + * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) + * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) + * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) +diff --git a/pengine/test10/rsc-sets-clone.summary b/pengine/test10/rsc-sets-clone.summary +index 7ee23a2..70570e6 100644 +--- a/pengine/test10/rsc-sets-clone.summary ++++ b/pengine/test10/rsc-sets-clone.summary +@@ -12,7 +12,7 @@ Online: [ node2 ] + Transition Summary: + * Move rsc1 (Started node1 -> node2) + * Move rsc3 (Started node1 -> node2) +- * Stop rsc:0 (node1) ++ * Stop rsc:0 (node1) due to node availability + + Executing cluster transition: + * Resource action: rsc1 stop on node1 +diff --git a/pengine/test10/stop-failure-no-quorum.summary b/pengine/test10/stop-failure-no-quorum.summary +index 4e337ab..75945b1 100644 +--- a/pengine/test10/stop-failure-no-quorum.summary ++++ b/pengine/test10/stop-failure-no-quorum.summary +@@ -16,11 +16,11 @@ Online: [ pcmk-1 ] + + Transition Summary: + * Fence (reboot) pcmk-2 'clvm:0 failed there' +- * Start dlm:0 (pcmk-1 - blocked) +- * Stop clvm:0 (pcmk-2) +- * Start clvm:2 (pcmk-1 - blocked) +- * Start ClusterIP (pcmk-1 - blocked) +- * Start Fencing (pcmk-1 - blocked) ++ * Start dlm:0 (pcmk-1 - blocked) due to no quorum ++ * Stop clvm:0 (pcmk-2) due to node availability ++ * Start clvm:2 (pcmk-1 - blocked) due to no quorum ++ * Start ClusterIP (pcmk-1 - blocked) due to no quorum ++ * Start Fencing (pcmk-1 - blocked) due to no quorum + + Executing cluster transition: + * Fencing pcmk-2 (reboot) +diff --git a/pengine/test10/stop-failure-with-fencing.summary b/pengine/test10/stop-failure-with-fencing.summary +index cdb39b0..09680c8 100644 +--- a/pengine/test10/stop-failure-with-fencing.summary ++++ b/pengine/test10/stop-failure-with-fencing.summary +@@ -15,11 +15,11 @@ Online: [ pcmk-1 ] + + Transition Summary: + * Fence (reboot) pcmk-2 'clvm:0 failed there' +- * Start dlm:0 (pcmk-1 - blocked) +- * Stop clvm:0 (pcmk-2) +- * Start clvm:1 (pcmk-1 - blocked) +- * Start ClusterIP (pcmk-1 - blocked) +- * Start Fencing (pcmk-1 - blocked) ++ * Start dlm:0 (pcmk-1 - blocked) due to no quorum ++ * Stop clvm:0 (pcmk-2) due to node availability ++ * Start clvm:1 (pcmk-1 - blocked) due to no quorum ++ * Start ClusterIP (pcmk-1 - blocked) due to no quorum ++ * Start Fencing (pcmk-1 - blocked) due to no quorum + + Executing cluster transition: + * Resource action: Fencing monitor on pcmk-1 +diff --git a/pengine/test10/ticket-clone-14.summary b/pengine/test10/ticket-clone-14.summary +index c807656..ceb76bf 100644 +--- a/pengine/test10/ticket-clone-14.summary ++++ b/pengine/test10/ticket-clone-14.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: clone1_stop_0 +diff --git a/pengine/test10/ticket-clone-15.summary b/pengine/test10/ticket-clone-15.summary +index c807656..ceb76bf 100644 +--- a/pengine/test10/ticket-clone-15.summary ++++ b/pengine/test10/ticket-clone-15.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: clone1_stop_0 +diff --git a/pengine/test10/ticket-clone-17.summary b/pengine/test10/ticket-clone-17.summary +index c807656..ceb76bf 100644 +--- a/pengine/test10/ticket-clone-17.summary ++++ b/pengine/test10/ticket-clone-17.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: clone1_stop_0 +diff --git a/pengine/test10/ticket-clone-18.summary b/pengine/test10/ticket-clone-18.summary +index c807656..ceb76bf 100644 +--- a/pengine/test10/ticket-clone-18.summary ++++ b/pengine/test10/ticket-clone-18.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: clone1_stop_0 +diff --git a/pengine/test10/ticket-clone-20.summary b/pengine/test10/ticket-clone-20.summary +index c807656..ceb76bf 100644 +--- a/pengine/test10/ticket-clone-20.summary ++++ b/pengine/test10/ticket-clone-20.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: clone1_stop_0 +diff --git a/pengine/test10/ticket-clone-21.summary b/pengine/test10/ticket-clone-21.summary +index ce5335f..6b18cfa 100644 +--- a/pengine/test10/ticket-clone-21.summary ++++ b/pengine/test10/ticket-clone-21.summary +@@ -10,8 +10,8 @@ Transition Summary: + * Fence (reboot) node2 'deadman ticket was lost' + * Fence (reboot) node1 'deadman ticket was lost' + * Stop rsc_stonith (node1) +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Fencing node1 (reboot) +diff --git a/pengine/test10/ticket-clone-23.summary b/pengine/test10/ticket-clone-23.summary +index c807656..ceb76bf 100644 +--- a/pengine/test10/ticket-clone-23.summary ++++ b/pengine/test10/ticket-clone-23.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: clone1_stop_0 +diff --git a/pengine/test10/ticket-clone-3.summary b/pengine/test10/ticket-clone-3.summary +index c807656..ceb76bf 100644 +--- a/pengine/test10/ticket-clone-3.summary ++++ b/pengine/test10/ticket-clone-3.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: clone1_stop_0 +diff --git a/pengine/test10/ticket-clone-6.summary b/pengine/test10/ticket-clone-6.summary +index c807656..ceb76bf 100644 +--- a/pengine/test10/ticket-clone-6.summary ++++ b/pengine/test10/ticket-clone-6.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: clone1_stop_0 +diff --git a/pengine/test10/ticket-clone-9.summary b/pengine/test10/ticket-clone-9.summary +index ce5335f..6b18cfa 100644 +--- a/pengine/test10/ticket-clone-9.summary ++++ b/pengine/test10/ticket-clone-9.summary +@@ -10,8 +10,8 @@ Transition Summary: + * Fence (reboot) node2 'deadman ticket was lost' + * Fence (reboot) node1 'deadman ticket was lost' + * Stop rsc_stonith (node1) +- * Stop rsc1:0 (node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:0 (node1) due to node availability ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Fencing node1 (reboot) +diff --git a/pengine/test10/ticket-group-14.summary b/pengine/test10/ticket-group-14.summary +index c0aaf74..824bb70 100644 +--- a/pengine/test10/ticket-group-14.summary ++++ b/pengine/test10/ticket-group-14.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/ticket-group-15.summary b/pengine/test10/ticket-group-15.summary +index c0aaf74..824bb70 100644 +--- a/pengine/test10/ticket-group-15.summary ++++ b/pengine/test10/ticket-group-15.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/ticket-group-17.summary b/pengine/test10/ticket-group-17.summary +index c0aaf74..824bb70 100644 +--- a/pengine/test10/ticket-group-17.summary ++++ b/pengine/test10/ticket-group-17.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/ticket-group-18.summary b/pengine/test10/ticket-group-18.summary +index c0aaf74..824bb70 100644 +--- a/pengine/test10/ticket-group-18.summary ++++ b/pengine/test10/ticket-group-18.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/ticket-group-20.summary b/pengine/test10/ticket-group-20.summary +index c0aaf74..824bb70 100644 +--- a/pengine/test10/ticket-group-20.summary ++++ b/pengine/test10/ticket-group-20.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/ticket-group-21.summary b/pengine/test10/ticket-group-21.summary +index 385e44f..e397a42 100644 +--- a/pengine/test10/ticket-group-21.summary ++++ b/pengine/test10/ticket-group-21.summary +@@ -9,8 +9,8 @@ Online: [ node1 node2 ] + + Transition Summary: + * Fence (reboot) node2 'deadman ticket was lost' +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Fencing node2 (reboot) +diff --git a/pengine/test10/ticket-group-23.summary b/pengine/test10/ticket-group-23.summary +index c0aaf74..824bb70 100644 +--- a/pengine/test10/ticket-group-23.summary ++++ b/pengine/test10/ticket-group-23.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/ticket-group-3.summary b/pengine/test10/ticket-group-3.summary +index c0aaf74..824bb70 100644 +--- a/pengine/test10/ticket-group-3.summary ++++ b/pengine/test10/ticket-group-3.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/ticket-group-6.summary b/pengine/test10/ticket-group-6.summary +index c0aaf74..824bb70 100644 +--- a/pengine/test10/ticket-group-6.summary ++++ b/pengine/test10/ticket-group-6.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/ticket-group-9.summary b/pengine/test10/ticket-group-9.summary +index 385e44f..e397a42 100644 +--- a/pengine/test10/ticket-group-9.summary ++++ b/pengine/test10/ticket-group-9.summary +@@ -9,8 +9,8 @@ Online: [ node1 node2 ] + + Transition Summary: + * Fence (reboot) node2 'deadman ticket was lost' +- * Stop rsc1 (node2) +- * Stop rsc2 (node2) ++ * Stop rsc1 (node2) due to node availability ++ * Stop rsc2 (node2) due to node availability + + Executing cluster transition: + * Fencing node2 (reboot) +diff --git a/pengine/test10/ticket-master-14.summary b/pengine/test10/ticket-master-14.summary +index 9f7b89d..9a42d78 100644 +--- a/pengine/test10/ticket-master-14.summary ++++ b/pengine/test10/ticket-master-14.summary +@@ -9,7 +9,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Demote rsc1:0 (Master -> Stopped node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +diff --git a/pengine/test10/ticket-master-15.summary b/pengine/test10/ticket-master-15.summary +index 9f7b89d..9a42d78 100644 +--- a/pengine/test10/ticket-master-15.summary ++++ b/pengine/test10/ticket-master-15.summary +@@ -9,7 +9,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Demote rsc1:0 (Master -> Stopped node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +diff --git a/pengine/test10/ticket-master-3.summary b/pengine/test10/ticket-master-3.summary +index 9f7b89d..9a42d78 100644 +--- a/pengine/test10/ticket-master-3.summary ++++ b/pengine/test10/ticket-master-3.summary +@@ -9,7 +9,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Demote rsc1:0 (Master -> Stopped node1) +- * Stop rsc1:1 (node2) ++ * Stop rsc1:1 (node2) due to node availability + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +diff --git a/pengine/test10/ticket-rsc-sets-10.summary b/pengine/test10/ticket-rsc-sets-10.summary +index 66f807a..8ef03e0 100644 +--- a/pengine/test10/ticket-rsc-sets-10.summary ++++ b/pengine/test10/ticket-rsc-sets-10.summary +@@ -15,10 +15,10 @@ Online: [ node1 node2 ] + + Transition Summary: + * Stop rsc1 (node2) +- * Stop rsc2 (node1) +- * Stop rsc3 (node1) +- * Stop rsc4:0 (node1) +- * Stop rsc4:1 (node2) ++ * Stop rsc2 (node1) due to node availability ++ * Stop rsc3 (node1) due to node availability ++ * Stop rsc4:0 (node1) due to node availability ++ * Stop rsc4:1 (node2) due to node availability + * Demote rsc5:0 (Master -> Slave node1) + + Executing cluster transition: +diff --git a/pengine/test10/ticket-rsc-sets-12.summary b/pengine/test10/ticket-rsc-sets-12.summary +index 6801c64..616a2a2 100644 +--- a/pengine/test10/ticket-rsc-sets-12.summary ++++ b/pengine/test10/ticket-rsc-sets-12.summary +@@ -14,8 +14,8 @@ Online: [ node1 node2 ] + + Transition Summary: + * Stop rsc1 (node2) +- * Stop rsc2 (node1) +- * Stop rsc3 (node1) ++ * Stop rsc2 (node1) due to node availability ++ * Stop rsc3 (node1) due to node availability + + Executing cluster transition: + * Resource action: rsc1 stop on node2 +diff --git a/pengine/test10/ticket-rsc-sets-13.summary b/pengine/test10/ticket-rsc-sets-13.summary +index 66f807a..8ef03e0 100644 +--- a/pengine/test10/ticket-rsc-sets-13.summary ++++ b/pengine/test10/ticket-rsc-sets-13.summary +@@ -15,10 +15,10 @@ Online: [ node1 node2 ] + + Transition Summary: + * Stop rsc1 (node2) +- * Stop rsc2 (node1) +- * Stop rsc3 (node1) +- * Stop rsc4:0 (node1) +- * Stop rsc4:1 (node2) ++ * Stop rsc2 (node1) due to node availability ++ * Stop rsc3 (node1) due to node availability ++ * Stop rsc4:0 (node1) due to node availability ++ * Stop rsc4:1 (node2) due to node availability + * Demote rsc5:0 (Master -> Slave node1) + + Executing cluster transition: +diff --git a/pengine/test10/ticket-rsc-sets-14.summary b/pengine/test10/ticket-rsc-sets-14.summary +index 66f807a..8ef03e0 100644 +--- a/pengine/test10/ticket-rsc-sets-14.summary ++++ b/pengine/test10/ticket-rsc-sets-14.summary +@@ -15,10 +15,10 @@ Online: [ node1 node2 ] + + Transition Summary: + * Stop rsc1 (node2) +- * Stop rsc2 (node1) +- * Stop rsc3 (node1) +- * Stop rsc4:0 (node1) +- * Stop rsc4:1 (node2) ++ * Stop rsc2 (node1) due to node availability ++ * Stop rsc3 (node1) due to node availability ++ * Stop rsc4:0 (node1) due to node availability ++ * Stop rsc4:1 (node2) due to node availability + * Demote rsc5:0 (Master -> Slave node1) + + Executing cluster transition: +diff --git a/pengine/test10/ticket-rsc-sets-3.summary b/pengine/test10/ticket-rsc-sets-3.summary +index 66f807a..8ef03e0 100644 +--- a/pengine/test10/ticket-rsc-sets-3.summary ++++ b/pengine/test10/ticket-rsc-sets-3.summary +@@ -15,10 +15,10 @@ Online: [ node1 node2 ] + + Transition Summary: + * Stop rsc1 (node2) +- * Stop rsc2 (node1) +- * Stop rsc3 (node1) +- * Stop rsc4:0 (node1) +- * Stop rsc4:1 (node2) ++ * Stop rsc2 (node1) due to node availability ++ * Stop rsc3 (node1) due to node availability ++ * Stop rsc4:0 (node1) due to node availability ++ * Stop rsc4:1 (node2) due to node availability + * Demote rsc5:0 (Master -> Slave node1) + + Executing cluster transition: +diff --git a/pengine/test10/ticket-rsc-sets-7.summary b/pengine/test10/ticket-rsc-sets-7.summary +index 66f807a..8ef03e0 100644 +--- a/pengine/test10/ticket-rsc-sets-7.summary ++++ b/pengine/test10/ticket-rsc-sets-7.summary +@@ -15,10 +15,10 @@ Online: [ node1 node2 ] + + Transition Summary: + * Stop rsc1 (node2) +- * Stop rsc2 (node1) +- * Stop rsc3 (node1) +- * Stop rsc4:0 (node1) +- * Stop rsc4:1 (node2) ++ * Stop rsc2 (node1) due to node availability ++ * Stop rsc3 (node1) due to node availability ++ * Stop rsc4:0 (node1) due to node availability ++ * Stop rsc4:1 (node2) due to node availability + * Demote rsc5:0 (Master -> Slave node1) + + Executing cluster transition: +diff --git a/pengine/test10/ticket-rsc-sets-9.summary b/pengine/test10/ticket-rsc-sets-9.summary +index 66f807a..8ef03e0 100644 +--- a/pengine/test10/ticket-rsc-sets-9.summary ++++ b/pengine/test10/ticket-rsc-sets-9.summary +@@ -15,10 +15,10 @@ Online: [ node1 node2 ] + + Transition Summary: + * Stop rsc1 (node2) +- * Stop rsc2 (node1) +- * Stop rsc3 (node1) +- * Stop rsc4:0 (node1) +- * Stop rsc4:1 (node2) ++ * Stop rsc2 (node1) due to node availability ++ * Stop rsc3 (node1) due to node availability ++ * Stop rsc4:0 (node1) due to node availability ++ * Stop rsc4:1 (node2) due to node availability + * Demote rsc5:0 (Master -> Slave node1) + + Executing cluster transition: +diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary +index d776228..f728e39 100644 +--- a/pengine/test10/unfence-definition.summary ++++ b/pengine/test10/unfence-definition.summary +@@ -16,9 +16,9 @@ Transition Summary: + * Fence (on) virt-3 'Required by dlm:2' + * Fence (on) virt-1 'Device definition changed' + * Restart fencing (Started virt-1) +- * Restart dlm:0 (Started virt-1) ++ * Restart dlm:0 (Started virt-1) due to required stonith + * Start dlm:2 (virt-3) +- * Restart clvmd:0 (Started virt-1) ++ * Restart clvmd:0 (Started virt-1) due to required stonith + * Start clvmd:1 (virt-2) + * Start clvmd:2 (virt-3) + +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +index e71eafc..e8b15ec 100644 +--- a/pengine/test10/unfence-parameters.summary ++++ b/pengine/test10/unfence-parameters.summary +@@ -17,10 +17,10 @@ Transition Summary: + * Fence (on) virt-2 'Device parameters changed (reload)' + * Fence (on) virt-1 'Device parameters changed (reload)' + * Restart fencing (Started virt-1) +- * Restart dlm:0 (Started virt-1) +- * Restart dlm:1 (Started virt-2) ++ * Restart dlm:0 (Started virt-1) due to required stonith ++ * Restart dlm:1 (Started virt-2) due to required stonith + * Start dlm:2 (virt-3) +- * Restart clvmd:0 (Started virt-1) ++ * Restart clvmd:0 (Started virt-1) due to required stonith + * Start clvmd:1 (virt-2) + * Start clvmd:2 (virt-3) + +diff --git a/pengine/test10/unmanaged-block-restart.summary b/pengine/test10/unmanaged-block-restart.summary +index 0b70481..01dffeb 100644 +--- a/pengine/test10/unmanaged-block-restart.summary ++++ b/pengine/test10/unmanaged-block-restart.summary +@@ -10,8 +10,8 @@ Online: [ yingying.site ] + + Transition Summary: + * Start rsc1 (yingying.site) +- * Stop rsc2 (Started yingying.site - blocked) +- * Stop rsc3 (Started yingying.site - blocked) ++ * Stop rsc2 (Started yingying.site - blocked) due to required rsc1 start ++ * Stop rsc3 (Started yingying.site - blocked) due to required rsc2 start + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/unmanaged-stop-1.summary b/pengine/test10/unmanaged-stop-1.summary +index 978ddc1..94e0908 100644 +--- a/pengine/test10/unmanaged-stop-1.summary ++++ b/pengine/test10/unmanaged-stop-1.summary +@@ -7,7 +7,7 @@ Online: [ yingying.site ] + rsc2 (ocf::pacemaker:Dummy): FAILED yingying.site ( blocked ) + + Transition Summary: +- * Stop rsc1 (yingying.site - blocked) ++ * Stop rsc1 (yingying.site - blocked) due to node availability + + Executing cluster transition: + +diff --git a/pengine/test10/unmanaged-stop-2.summary b/pengine/test10/unmanaged-stop-2.summary +index 978ddc1..94e0908 100644 +--- a/pengine/test10/unmanaged-stop-2.summary ++++ b/pengine/test10/unmanaged-stop-2.summary +@@ -7,7 +7,7 @@ Online: [ yingying.site ] + rsc2 (ocf::pacemaker:Dummy): FAILED yingying.site ( blocked ) + + Transition Summary: +- * Stop rsc1 (yingying.site - blocked) ++ * Stop rsc1 (yingying.site - blocked) due to node availability + + Executing cluster transition: + +diff --git a/pengine/test10/unmanaged-stop-3.summary b/pengine/test10/unmanaged-stop-3.summary +index 74829a1..c72d451 100644 +--- a/pengine/test10/unmanaged-stop-3.summary ++++ b/pengine/test10/unmanaged-stop-3.summary +@@ -8,7 +8,7 @@ Online: [ yingying.site ] + rsc2 (ocf::pacemaker:Dummy): FAILED yingying.site ( disabled, blocked ) + + Transition Summary: +- * Stop rsc1 (yingying.site - blocked) ++ * Stop rsc1 (yingying.site - blocked) unrunnable rsc2 stop due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/unmanaged-stop-4.summary b/pengine/test10/unmanaged-stop-4.summary +index 457e41f..2704e98 100644 +--- a/pengine/test10/unmanaged-stop-4.summary ++++ b/pengine/test10/unmanaged-stop-4.summary +@@ -9,7 +9,7 @@ Online: [ yingying.site ] + rsc3 (ocf::heartbeat:Dummy): Stopped ( disabled ) + + Transition Summary: +- * Stop rsc1 (yingying.site - blocked) ++ * Stop rsc1 (yingying.site - blocked) unrunnable rsc2 stop due to node availability + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/unrunnable-1.summary b/pengine/test10/unrunnable-1.summary +index 92f4031..8c27e45 100644 +--- a/pengine/test10/unrunnable-1.summary ++++ b/pengine/test10/unrunnable-1.summary +@@ -19,15 +19,15 @@ Online: [ c001n03 ] + child_DoFencing:3 (stonith:ssh): Stopped + + Transition Summary: +- * Start DcIPaddr (c001n03 - blocked) +- * Start child_192.168.100.181 (c001n03 - blocked) +- * Start child_192.168.100.182 (c001n03 - blocked) +- * Start child_192.168.100.183 (c001n03 - blocked) +- * Start rsc_c001n08 (c001n03 - blocked) +- * Start rsc_c001n02 (c001n03 - blocked) +- * Start rsc_c001n03 (c001n03 - blocked) +- * Start rsc_c001n01 (c001n03 - blocked) +- * Stop child_DoFencing:1 (c001n02 - blocked) ++ * Start DcIPaddr (c001n03 - blocked) due to no quorum ++ * Start child_192.168.100.181 (c001n03 - blocked) due to no quorum ++ * Start child_192.168.100.182 (c001n03 - blocked) due to no quorum ++ * Start child_192.168.100.183 (c001n03 - blocked) due to no quorum ++ * Start rsc_c001n08 (c001n03 - blocked) due to no quorum ++ * Start rsc_c001n02 (c001n03 - blocked) due to no quorum ++ * Start rsc_c001n03 (c001n03 - blocked) due to no quorum ++ * Start rsc_c001n01 (c001n03 - blocked) due to no quorum ++ * Stop child_DoFencing:1 (c001n02 - blocked) due to node availability + + Executing cluster transition: + * Resource action: DcIPaddr monitor on c001n03 +diff --git a/pengine/test10/unrunnable-2.summary b/pengine/test10/unrunnable-2.summary +index 0c0c765..b67f7cb 100644 +--- a/pengine/test10/unrunnable-2.summary ++++ b/pengine/test10/unrunnable-2.summary +@@ -85,7 +85,7 @@ Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] + Stopped: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] + + Transition Summary: +- * Start openstack-cinder-volume (overcloud-controller-2 - blocked) ++ * Start openstack-cinder-volume (overcloud-controller-2 - blocked) due to unrunnable openstack-cinder-scheduler-clone running + + Executing cluster transition: + +diff --git a/pengine/test10/utilization-order2.summary b/pengine/test10/utilization-order2.summary +index 7871579..db65a16 100644 +--- a/pengine/test10/utilization-order2.summary ++++ b/pengine/test10/utilization-order2.summary +@@ -11,7 +11,7 @@ Online: [ node1 node2 ] + Transition Summary: + * Start rsc4 (node1) + * Move rsc3 (Started node1 -> node2) +- * Stop rsc2:0 (node1) ++ * Stop rsc2:0 (node1) due to node availability + * Stop rsc1 (node2) + + Executing cluster transition: +diff --git a/pengine/test10/utilization-order4.summary b/pengine/test10/utilization-order4.summary +index efb5958..09a8c31 100644 +--- a/pengine/test10/utilization-order4.summary ++++ b/pengine/test10/utilization-order4.summary +@@ -17,10 +17,10 @@ Online: [ deglxen001 ] + Transition Summary: + * Migrate degllx62-vm (Started deglxen002 -> deglxen001) + * Stop degllx61-vm (deglxen001) +- * Stop nfs-xen_config:1 (deglxen002) +- * Stop nfs-xen_swapfiles:1 (deglxen002) +- * Stop nfs-xen_images:1 (deglxen002) +- * Stop prim-ping:1 (deglxen002) ++ * Stop nfs-xen_config:1 (deglxen002) due to node availability ++ * Stop nfs-xen_swapfiles:1 (deglxen002) due to node availability ++ * Stop nfs-xen_images:1 (deglxen002) due to node availability ++ * Stop prim-ping:1 (deglxen002) due to node availability + + Executing cluster transition: + * Resource action: degllx61-vm stop on deglxen001 +diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary +index 14b1ee2..1d06552 100644 +--- a/pengine/test10/whitebox-fail1.summary ++++ b/pengine/test10/whitebox-fail1.summary +@@ -18,7 +18,7 @@ Transition Summary: + * Recover container1 (Started 18node2) + * Recover M:4 (Started lxc1) + * Recover B (Started lxc1) +- * Restart lxc1 (Started 18node2) ++ * Restart lxc1 (Started 18node2) due to required container1 start + + Executing cluster transition: + * Resource action: lxc1 stop on 18node2 +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary +index 9dab410..bea75a2 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.summary ++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary +@@ -29,9 +29,9 @@ Transition Summary: + * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) 'guest is unclean' + * Fence (reboot) kiff-01 'peer is no longer part of the cluster' + * Move fence-kiff-02 (Started kiff-01 -> kiff-02) +- * Stop dlm:0 (kiff-01) +- * Stop clvmd:0 (kiff-01) +- * Stop shared0:0 (kiff-01) ++ * Stop dlm:0 (kiff-01) due to node availability ++ * Stop clvmd:0 (kiff-01) due to node availability ++ * Stop shared0:0 (kiff-01) due to node availability + * Recover R-lxc-01_kiff-01 (Started kiff-01 -> kiff-02) + * Move R-lxc-02_kiff-01 (Started kiff-01 -> kiff-02) + * Recover vm-fs (Started lxc-01_kiff-01) +diff --git a/pengine/test10/whitebox-move.summary b/pengine/test10/whitebox-move.summary +index 3422ac7..dbf0780 100644 +--- a/pengine/test10/whitebox-move.summary ++++ b/pengine/test10/whitebox-move.summary +@@ -12,8 +12,8 @@ Containers: [ lxc1:container1 lxc2:container2 ] + + Transition Summary: + * Move container1 (Started 18node1 -> 18node2) +- * Restart M:3 (Started lxc1) +- * Restart A (Started lxc1) ++ * Restart M:3 (Started lxc1) due to required container1 start ++ * Restart A (Started lxc1) due to required container1 start + * Move lxc1 (Started 18node1 -> 18node2) + + Executing cluster transition: +diff --git a/pengine/test10/whitebox-ms-ordering-move.summary b/pengine/test10/whitebox-ms-ordering-move.summary +index d5f133b..b48d988 100644 +--- a/pengine/test10/whitebox-ms-ordering-move.summary ++++ b/pengine/test10/whitebox-ms-ordering-move.summary +@@ -31,7 +31,7 @@ Containers: [ lxc1:container1 lxc2:container2 ] + + Transition Summary: + * Move container1 (Started rhel7-1 -> rhel7-2) +- * Restart lxc-ms:0 (Master lxc1) ++ * Restart lxc-ms:0 (Master lxc1) due to required container1 start + * Move lxc1 (Started rhel7-1 -> rhel7-2) + + Executing cluster transition: +diff --git a/pengine/test10/whitebox-orphan-ms.summary b/pengine/test10/whitebox-orphan-ms.summary +index 30c6a3b..3efa6bd 100644 +--- a/pengine/test10/whitebox-orphan-ms.summary ++++ b/pengine/test10/whitebox-orphan-ms.summary +@@ -28,11 +28,11 @@ Containers: [ lxc1:container1 lxc2:container2 ] + + Transition Summary: + * Move FencingFail (Started 18node3 -> 18node1) +- * Stop container2 (18node1) +- * Stop lxc1 (18node1) ++ * Stop container2 (18node1) due to node availability ++ * Stop lxc1 (18node1) due to node availability + * Demote lxc-ms (Master -> Stopped lxc1) +- * Stop lxc2 (18node1) +- * Stop container1 (18node1) ++ * Stop lxc2 (18node1) due to node availability ++ * Stop container1 (18node1) due to node availability + + Executing cluster transition: + * Resource action: FencingFail stop on 18node3 +diff --git a/pengine/test10/whitebox-orphaned.summary b/pengine/test10/whitebox-orphaned.summary +index 7be8453..52b54aa 100644 +--- a/pengine/test10/whitebox-orphaned.summary ++++ b/pengine/test10/whitebox-orphaned.summary +@@ -16,10 +16,10 @@ Containers: [ lxc1:container1 lxc2:container2 ] + lxc1 (ocf::pacemaker:remote): ORPHANED Started 18node2 + + Transition Summary: +- * Stop M:4 (lxc1) ++ * Stop M:4 (lxc1) due to node availability + * Move B (Started lxc1 -> lxc2) +- * Stop container1 (18node2) +- * Stop lxc1 (18node2) ++ * Stop container1 (18node2) due to node availability ++ * Stop lxc1 (18node2) due to node availability + + Executing cluster transition: + * Pseudo action: M-clone_stop_0 +diff --git a/pengine/test10/whitebox-stop.summary b/pengine/test10/whitebox-stop.summary +index b6a2954..89094da 100644 +--- a/pengine/test10/whitebox-stop.summary ++++ b/pengine/test10/whitebox-stop.summary +@@ -15,10 +15,10 @@ Containers: [ lxc1:container1 lxc2:container2 ] + D (ocf::pacemaker:Dummy): Started 18node1 + + Transition Summary: +- * Stop container1 (18node2) +- * Stop M:4 (lxc1) ++ * Stop container1 (18node2) due to node availability ++ * Stop M:4 (lxc1) due to node availability + * Move B (Started lxc1 -> lxc2) +- * Stop lxc1 (18node2) ++ * Stop lxc1 (18node2) due to node availability + + Executing cluster transition: + * Pseudo action: M-clone_stop_0 +-- +1.8.3.1 + + +From ab87380e8800a1b1f867450e61049e9becbaec21 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 1 Aug 2017 16:18:06 +1000 +Subject: [PATCH 06/12] Log: PE: Output resource changes in a friendlier format + +--- + lib/pengine/utils.c | 1 + + pengine/native.c | 235 ++++++++++++--------- + pengine/test10/1-a-then-bm-move-b.summary | 2 +- + pengine/test10/10-a-then-bm-b-move-a-clone.summary | 2 +- + .../11-a-then-bm-b-move-a-clone-starting.summary | 4 +- + pengine/test10/1360.summary | 2 +- + pengine/test10/2-am-then-b-move-a.summary | 2 +- + pengine/test10/3-am-then-bm-both-migrate.summary | 4 +- + .../test10/4-am-then-bm-b-not-migratable.summary | 4 +- + .../test10/5-am-then-bm-a-not-migratable.summary | 4 +- + pengine/test10/594.summary | 4 +- + pengine/test10/6-migrate-group.summary | 6 +- + pengine/test10/662.summary | 2 +- + pengine/test10/696.summary | 2 +- + .../7-migrate-group-one-unmigratable.summary | 6 +- + pengine/test10/726.summary | 2 +- + pengine/test10/735.summary | 2 +- + pengine/test10/764.summary | 10 +- + pengine/test10/797.summary | 12 +- + .../8-am-then-bm-a-migrating-b-stopping.summary | 2 +- + pengine/test10/829.summary | 2 +- + .../9-am-then-bm-b-migrating-a-stopping.summary | 2 +- + pengine/test10/994-2.summary | 4 +- + pengine/test10/994.summary | 2 +- + pengine/test10/a-demote-then-b-migrate.summary | 4 +- + pengine/test10/a-promote-then-b-migrate.summary | 2 +- + pengine/test10/anti-colocation-master.summary | 4 +- + pengine/test10/anti-colocation-order.summary | 4 +- + pengine/test10/anti-colocation-slave.summary | 4 +- + pengine/test10/asymmetrical-order-move.summary | 2 +- + pengine/test10/bug-1572-1.summary | 8 +- + pengine/test10/bug-1572-2.summary | 2 +- + pengine/test10/bug-1718.summary | 4 +- + pengine/test10/bug-1820-1.summary | 4 +- + pengine/test10/bug-1820.summary | 4 +- + pengine/test10/bug-1822.summary | 4 +- + .../test10/bug-5007-masterslave_colocation.summary | 4 +- + .../test10/bug-5014-A-stopped-B-stopped.summary | 2 +- + .../test10/bug-5014-CthenAthenB-C-stopped.summary | 4 +- + .../bug-5014-GROUP-A-stopped-B-stopped.summary | 2 +- + .../bug-5014-ordered-set-symmetrical-true.summary | 2 +- + pengine/test10/bug-5025-1.summary | 2 +- + pengine/test10/bug-5025-3.summary | 2 +- + pengine/test10/bug-5028.summary | 2 +- + pengine/test10/bug-5059.summary | 1 - + pengine/test10/bug-5186-partial-migrate.summary | 8 +- + pengine/test10/bug-cl-5212.summary | 14 +- + pengine/test10/bug-cl-5247.summary | 10 +- + pengine/test10/bug-lf-2106.summary | 4 +- + pengine/test10/bug-lf-2153.summary | 2 +- + pengine/test10/bug-lf-2171.summary | 4 +- + pengine/test10/bug-lf-2361.summary | 4 +- + pengine/test10/bug-lf-2435.summary | 2 +- + pengine/test10/bug-lf-2445.summary | 2 +- + pengine/test10/bug-lf-2453.summary | 6 +- + pengine/test10/bug-lf-2508.summary | 2 +- + pengine/test10/bug-lf-2551.summary | 4 +- + pengine/test10/bug-lf-2574.summary | 2 +- + pengine/test10/bug-lf-2606.summary | 4 +- + pengine/test10/bug-lf-2613.summary | 12 +- + pengine/test10/bug-lf-2619.summary | 12 +- + pengine/test10/bug-n-385265-2.summary | 4 +- + pengine/test10/bug-n-387749.summary | 4 +- + pengine/test10/bug-pm-11.summary | 1 - + pengine/test10/bug-rh-1097457.summary | 12 +- + pengine/test10/bug-rh-880249.summary | 3 +- + pengine/test10/bug-suse-707150.summary | 2 +- + pengine/test10/bundle-nested-colocation.summary | 6 +- + .../test10/bundle-order-partial-start-2.summary | 4 +- + pengine/test10/bundle-order-partial-stop.summary | 8 +- + .../test10/bundle-order-startup-clone-2.summary | 3 - + pengine/test10/bundle-order-startup-clone.summary | 12 +- + pengine/test10/bundle-order-stop-clone.summary | 2 +- + pengine/test10/bundle-order-stop.summary | 8 +- + pengine/test10/clone-anon-failcount.summary | 12 +- + pengine/test10/clone-fail-block-colocation.summary | 8 +- + pengine/test10/clone-interleave-2.summary | 2 +- + pengine/test10/clone-interleave-3.summary | 6 +- + pengine/test10/clone-no-shuffle.summary | 2 +- + pengine/test10/clone-require-all-2.summary | 6 +- + pengine/test10/clone-require-all-3.summary | 6 +- + pengine/test10/clone-require-all-4.summary | 2 +- + .../clone-require-all-no-interleave-3.summary | 6 +- + .../test10/clone_min_interleave_start_one.summary | 12 +- + .../test10/clone_min_interleave_stop_two.summary | 12 +- + pengine/test10/clone_min_start_one.summary | 4 +- + pengine/test10/clone_min_stop_all.summary | 4 +- + pengine/test10/clone_min_stop_two.summary | 4 +- + pengine/test10/cloned-group-stop.summary | 28 +-- + pengine/test10/cloned-group.summary | 4 +- + pengine/test10/cloned_start_one.summary | 2 +- + pengine/test10/cloned_stop_two.summary | 2 +- + pengine/test10/colo_master_w_native.summary | 2 +- + pengine/test10/colo_slave_w_native.summary | 4 +- + pengine/test10/coloc-clone-stays-active.summary | 2 +- + pengine/test10/coloc-intra-set.summary | 4 +- + pengine/test10/coloc_fp_logic.summary | 2 +- + .../colocation_constraint_stops_master.summary | 2 +- + .../colocation_constraint_stops_slave.summary | 2 +- + pengine/test10/complex_enforce_colo.summary | 66 +++--- + pengine/test10/container-2.summary | 6 +- + pengine/test10/container-3.summary | 4 +- + pengine/test10/container-4.summary | 6 +- + pengine/test10/container-group-2.summary | 6 +- + pengine/test10/container-group-3.summary | 2 +- + pengine/test10/container-group-4.summary | 6 +- + pengine/test10/enforce-colo1.summary | 4 +- + pengine/test10/group-dependents.summary | 28 +-- + pengine/test10/group-fail.summary | 4 +- + pengine/test10/group10.summary | 6 +- + pengine/test10/group14.summary | 32 +-- + pengine/test10/group5.summary | 10 +- + pengine/test10/group6.summary | 12 +- + pengine/test10/group9.summary | 10 +- + pengine/test10/guest-node-host-dies.summary | 12 +- + pengine/test10/inc11.summary | 3 +- + pengine/test10/inc12.summary | 20 +- + pengine/test10/inc2.summary | 4 +- + pengine/test10/inc3.summary | 8 +- + pengine/test10/inc4.summary | 8 +- + pengine/test10/inc5.summary | 8 +- + pengine/test10/inc6.summary | 12 +- + pengine/test10/interleave-restart.summary | 6 +- + pengine/test10/isolation-restart-all.summary | 22 +- + pengine/test10/load-stopped-loop-2.summary | 16 +- + pengine/test10/load-stopped-loop.summary | 20 +- + pengine/test10/master-1.summary | 1 - + pengine/test10/master-10.summary | 1 - + pengine/test10/master-11.summary | 1 - + pengine/test10/master-2.summary | 1 - + pengine/test10/master-3.summary | 1 - + pengine/test10/master-7.summary | 16 +- + pengine/test10/master-8.summary | 15 +- + pengine/test10/master-9.summary | 22 +- + pengine/test10/master-asymmetrical-order.summary | 4 +- + pengine/test10/master-demote.summary | 2 +- + pengine/test10/master-failed-demote-2.summary | 2 +- + pengine/test10/master-failed-demote.summary | 2 +- + pengine/test10/master-move.summary | 4 +- + .../test10/master-partially-demoted-group.summary | 12 +- + pengine/test10/master-probed-score.summary | 6 +- + pengine/test10/master-pseudo.summary | 3 +- + pengine/test10/master-stop.summary | 2 +- + pengine/test10/migrate-1.summary | 2 +- + pengine/test10/migrate-3.summary | 2 +- + pengine/test10/migrate-4.summary | 2 +- + pengine/test10/migrate-5.summary | 2 +- + pengine/test10/migrate-begin.summary | 2 +- + pengine/test10/migrate-both-vms.summary | 4 +- + pengine/test10/migrate-fail-2.summary | 2 +- + pengine/test10/migrate-fail-3.summary | 2 +- + pengine/test10/migrate-fail-4.summary | 2 +- + pengine/test10/migrate-fail-6.summary | 2 +- + pengine/test10/migrate-fail-7.summary | 2 +- + pengine/test10/migrate-fail-8.summary | 2 +- + pengine/test10/migrate-fail-9.summary | 2 +- + pengine/test10/migrate-fencing.summary | 14 +- + pengine/test10/migrate-partial-2.summary | 2 +- + pengine/test10/migrate-partial-3.summary | 2 +- + pengine/test10/migrate-partial-4.summary | 4 +- + pengine/test10/migrate-shutdown.summary | 6 +- + pengine/test10/migrate-start-complex.summary | 4 +- + pengine/test10/migrate-start.summary | 2 +- + pengine/test10/migrate-stop-complex.summary | 4 +- + pengine/test10/migrate-stop-start-complex.summary | 4 +- + pengine/test10/migrate-stop.summary | 2 +- + pengine/test10/migrate-stop_start.summary | 4 +- + pengine/test10/mon-rsc-2.summary | 2 +- + pengine/test10/mon-rsc-4.summary | 2 +- + pengine/test10/monitor-onfail-restart.summary | 2 +- + pengine/test10/multi1.summary | 2 +- + pengine/test10/multiple-monitor-one-failed.summary | 2 +- + pengine/test10/not-installed-agent.summary | 4 +- + pengine/test10/not-installed-tools.summary | 2 +- + .../test10/not-reschedule-unneeded-monitor.summary | 2 +- + pengine/test10/notify-3.summary | 2 +- + pengine/test10/novell-239082.summary | 4 +- + pengine/test10/novell-252693-2.summary | 2 +- + pengine/test10/novell-252693-3.summary | 4 +- + pengine/test10/novell-252693.summary | 2 +- + pengine/test10/one-or-more-1.summary | 6 +- + pengine/test10/one-or-more-3.summary | 2 +- + pengine/test10/order-mandatory.summary | 4 +- + pengine/test10/order-required.summary | 4 +- + pengine/test10/order-serialize-set.summary | 12 +- + pengine/test10/order-serialize.summary | 12 +- + pengine/test10/order-sets.summary | 8 +- + pengine/test10/order-wrong-kind.summary | 2 +- + pengine/test10/order3.summary | 8 +- + pengine/test10/order5.summary | 8 +- + pengine/test10/order6.summary | 8 +- + pengine/test10/order7.summary | 2 +- + pengine/test10/ordered-set-basic-startup.summary | 6 +- + pengine/test10/ordered-set-natural.summary | 12 +- + pengine/test10/params-1.summary | 2 +- + pengine/test10/params-2.summary | 2 +- + pengine/test10/params-4.summary | 2 +- + pengine/test10/params-5.summary | 2 +- + pengine/test10/params-6.summary | 2 +- + pengine/test10/probe-2.summary | 12 +- + pengine/test10/probe-4.summary | 2 +- + pengine/test10/quorum-1.summary | 2 +- + pengine/test10/quorum-2.summary | 4 +- + pengine/test10/quorum-3.summary | 6 +- + pengine/test10/rec-node-10.summary | 6 +- + pengine/test10/rec-node-11.summary | 6 +- + pengine/test10/rec-node-13.summary | 2 +- + pengine/test10/rec-node-15.summary | 6 +- + pengine/test10/rec-node-4.summary | 4 +- + pengine/test10/rec-node-6.summary | 4 +- + pengine/test10/rec-node-7.summary | 4 +- + pengine/test10/rec-node-8.summary | 8 +- + pengine/test10/rec-node-9.summary | 4 +- + pengine/test10/rec-rsc-1.summary | 2 +- + pengine/test10/rec-rsc-2.summary | 2 +- + pengine/test10/rec-rsc-5.summary | 4 +- + pengine/test10/rec-rsc-6.summary | 2 +- + pengine/test10/rec-rsc-9.summary | 6 +- + .../test10/remote-fence-before-reconnect.summary | 2 +- + pengine/test10/remote-fence-unclean.summary | 8 +- + pengine/test10/remote-move.summary | 4 +- + pengine/test10/remote-orphaned.summary | 2 +- + pengine/test10/remote-partial-migrate.summary | 14 +- + pengine/test10/remote-partial-migrate2.summary | 34 +-- + pengine/test10/remote-recover-all.summary | 18 +- + pengine/test10/remote-recover-connection.summary | 20 +- + pengine/test10/remote-recover-fail.summary | 6 +- + pengine/test10/remote-recover-no-resources.summary | 16 +- + pengine/test10/remote-recover-unknown.summary | 16 +- + pengine/test10/remote-recovery.summary | 20 +- + pengine/test10/remote-start-fail.summary | 2 +- + pengine/test10/remote-startup-probes.summary | 4 +- + pengine/test10/remote-startup.summary | 2 +- + pengine/test10/remote-unclean2.summary | 2 +- + pengine/test10/rsc-discovery-per-node.summary | 6 +- + pengine/test10/rsc-sets-clone-1.summary | 2 +- + pengine/test10/rsc-sets-clone.summary | 4 +- + pengine/test10/rsc-sets-master.summary | 8 +- + pengine/test10/rsc-sets-seq-false.summary | 12 +- + pengine/test10/rsc-sets-seq-true.summary | 12 +- + pengine/test10/rsc_dep4.summary | 2 +- + pengine/test10/standby.summary | 16 +- + .../test10/start-then-stop-with-unfence.summary | 2 +- + pengine/test10/stonith-0.summary | 10 +- + pengine/test10/stonith-1.summary | 10 +- + pengine/test10/stonith-4.summary | 2 +- + pengine/test10/stop-failure-no-quorum.summary | 8 +- + pengine/test10/stop-failure-with-fencing.summary | 8 +- + pengine/test10/stopped-monitor-01.summary | 2 +- + pengine/test10/stopped-monitor-02.summary | 2 +- + pengine/test10/stopped-monitor-08.summary | 2 +- + pengine/test10/ticket-master-14.summary | 4 +- + pengine/test10/ticket-master-15.summary | 4 +- + pengine/test10/ticket-master-2.summary | 1 - + pengine/test10/ticket-master-21.summary | 4 +- + pengine/test10/ticket-master-3.summary | 4 +- + pengine/test10/ticket-master-9.summary | 4 +- + pengine/test10/unfence-definition.summary | 6 +- + pengine/test10/unfence-parameters.summary | 8 +- + pengine/test10/unmanaged-block-restart.summary | 4 +- + pengine/test10/unmanaged-stop-1.summary | 2 +- + pengine/test10/unmanaged-stop-2.summary | 2 +- + pengine/test10/unmanaged-stop-3.summary | 2 +- + pengine/test10/unmanaged-stop-4.summary | 2 +- + pengine/test10/unrunnable-1.summary | 18 +- + pengine/test10/unrunnable-2.summary | 2 +- + pengine/test10/utilization-order2.summary | 2 +- + pengine/test10/utilization-order3.summary | 2 +- + pengine/test10/utilization-order4.summary | 2 +- + pengine/test10/whitebox-fail1.summary | 8 +- + pengine/test10/whitebox-fail2.summary | 8 +- + pengine/test10/whitebox-fail3.summary | 2 +- + .../test10/whitebox-imply-stop-on-fence.summary | 12 +- + pengine/test10/whitebox-migrate1.summary | 8 +- + pengine/test10/whitebox-move.summary | 8 +- + pengine/test10/whitebox-ms-ordering-move.summary | 4 +- + pengine/test10/whitebox-orphan-ms.summary | 5 +- + pengine/test10/whitebox-orphaned.summary | 2 +- + pengine/test10/whitebox-start.summary | 4 +- + pengine/test10/whitebox-stop.summary | 2 +- + .../test10/whitebox-unexpectedly-running.summary | 2 +- + 280 files changed, 924 insertions(+), 915 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index bc34323..222a3e2 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -2253,6 +2253,7 @@ void pe_action_set_reason(pe_action_t *action, const char *reason, bool overwrit + if(action->reason == NULL || overwrite) { + free(action->reason); + if(reason) { ++ crm_trace("Set %s reason to '%s'", action->uuid, reason); + action->reason = strdup(reason); + } else { + action->reason = NULL; +diff --git a/pengine/native.c b/pengine/native.c +index 63666fd..5281f36 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -2155,6 +2155,108 @@ native_expand(resource_t * rsc, pe_working_set_t * data_set) + } \ + } while(0) + ++static int rsc_width = 5; ++static int detail_width = 5; ++static void ++LogAction(const char *change, resource_t *rsc, pe_node_t *origin, pe_node_t *destination, pe_action_t *action, pe_action_t *source, gboolean terminal) ++{ ++ int len = 0; ++ char *reason = NULL; ++ char *details = NULL; ++ bool same_host = FALSE; ++ bool same_role = FALSE; ++ bool need_role = FALSE; ++ ++ CRM_ASSERT(action); ++ CRM_ASSERT(destination != NULL || origin != NULL); ++ ++ if(source == NULL) { ++ source = action; ++ } ++ ++ len = strlen(rsc->id); ++ if(len > rsc_width) { ++ rsc_width = len + 2; ++ } ++ ++ if(rsc->role > RSC_ROLE_STARTED || rsc->next_role > RSC_ROLE_SLAVE) { ++ need_role = TRUE; ++ } ++ ++ if(origin != NULL && destination != NULL && origin->details == destination->details) { ++ same_host = TRUE; ++ } ++ ++ if(rsc->role == rsc->next_role) { ++ same_role = TRUE; ++ } ++ ++ if(need_role && origin == NULL) { ++ /* Promoting from Stopped */ ++ details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), destination->details->uname); ++ ++ } else if(need_role && destination == NULL) { ++ /* Demoting a Master or Stopping a Slave */ ++ details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname); ++ ++ } else if(origin == NULL || destination == NULL) { ++ /* Starting or stopping a resource */ ++ details = crm_strdup_printf("%s", origin?origin->details->uname:destination->details->uname); ++ ++ } else if(need_role && same_role && same_host) { ++ /* Recovering or Restarting a Master/Slave resource */ ++ details = crm_strdup_printf("%s %s", role2text(rsc->role), origin->details->uname); ++ ++ } else if(same_role && same_host) { ++ /* Recovering or Restarting a normal resource */ ++ details = crm_strdup_printf("%s", origin->details->uname); ++ ++ } else if(same_role && need_role) { ++ /* Moving a Master/Slave resource */ ++ details = crm_strdup_printf("%s -> %s %s", origin->details->uname, destination->details->uname, role2text(rsc->role)); ++ ++ } else if(same_role) { ++ /* Moving a normal resource */ ++ details = crm_strdup_printf("%s -> %s", origin->details->uname, destination->details->uname); ++ ++ } else if(same_host) { ++ /* Promoting or Demoting a Master/Slave resource */ ++ details = crm_strdup_printf("%s -> %s %s", role2text(rsc->role), role2text(rsc->next_role), origin->details->uname); ++ ++ } else { ++ /* Moving and promoting/demoting */ ++ details = crm_strdup_printf("%s %s -> %s %s", role2text(rsc->role), origin->details->uname, role2text(rsc->next_role), destination->details->uname); ++ } ++ ++ len = strlen(details); ++ if(len > detail_width) { ++ detail_width = len; ++ } ++ ++ if(source->reason && is_not_set(action->flags, pe_action_runnable)) { ++ reason = crm_strdup_printf(" due to %s (blocked)", source->reason); ++ ++ } else if(source->reason) { ++ reason = crm_strdup_printf(" due to %s", source->reason); ++ ++ } else if(is_not_set(action->flags, pe_action_runnable)) { ++ reason = strdup(" blocked"); ++ ++ } else { ++ reason = strdup(""); ++ } ++ ++ if(terminal) { ++ printf(" * %-8s %-*s ( %*s ) %s\n", change, rsc_width, rsc->id, detail_width, details, reason); ++ } else { ++ crm_notice(" * %-8s %-*s ( %*s ) %s", change, rsc_width, rsc->id, detail_width, details, reason); ++ } ++ ++ free(details); ++ free(reason); ++} ++ ++ + void + LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + { +@@ -2234,11 +2336,11 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + } else { + possible_matches = find_actions(rsc->actions, key, current); + } +- free(key); + if (possible_matches) { + stop = possible_matches->data; + g_list_free(possible_matches); + } ++ free(key); + + key = promote_key(rsc); + possible_matches = find_actions(rsc->actions, key, next); +@@ -2269,34 +2371,29 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + CRM_CHECK(next != NULL,); + if (next == NULL) { + } else if (migrate_to && is_set(migrate_to->flags, pe_action_runnable) && current) { +- log_change(start, "Migrate %s\t(%s %s -> %s)", +- rsc->id, role2text(rsc->role), current->details->uname, +- next->details->uname); ++ LogAction("Migrate", rsc, current, next, start, NULL, terminal); + + } else if (is_set(rsc->flags, pe_rsc_reload)) { +- log_change(start, "Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); ++ LogAction("Reload", rsc, current, next, start, NULL, terminal); + + } else if (start == NULL || is_set(start->flags, pe_action_optional)) { + pe_rsc_info(rsc, "Leave %s\t(%s %s)", rsc->id, role2text(rsc->role), + next->details->uname); + + } else if (start && is_set(start->flags, pe_action_runnable) == FALSE) { +- log_change(start, "Stop %s\t(%s %s%s)", rsc->id, role2text(rsc->role), current?current->details->uname:"N/A", +- stop && is_not_set(stop->flags, pe_action_runnable) ? " - blocked" : ""); ++ LogAction("Stop", rsc, current, NULL, stop, start, terminal); + STOP_SANITY_ASSERT(__LINE__); + + } else if (moving && current) { +- log_change(stop, "%s %s\t(%s %s -> %s)", +- is_set(rsc->flags, pe_rsc_failed) ? "Recover" : "Move ", +- rsc->id, role2text(rsc->role), +- current->details->uname, next->details->uname); ++ LogAction(is_set(rsc->flags, pe_rsc_failed) ? "Recover" : "Move", ++ rsc, current, next, stop, NULL, terminal); + + } else if (is_set(rsc->flags, pe_rsc_failed)) { +- log_change(stop, "Recover %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); ++ LogAction("Recover", rsc, current, NULL, stop, NULL, terminal); + STOP_SANITY_ASSERT(__LINE__); + + } else { +- log_change(start, "Restart %s\t(%s %s)", rsc->id, role2text(rsc->role), next->details->uname); ++ LogAction("Restart", rsc, current, next, start, NULL, terminal); + /* STOP_SANITY_ASSERT(__LINE__); False positive for migrate-fail-7 */ + } + +@@ -2304,50 +2401,16 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + return; + } + +- if (rsc->role > RSC_ROLE_SLAVE && rsc->role > rsc->next_role) { +- CRM_CHECK(current != NULL,); +- if (current != NULL) { +- gboolean allowed = FALSE; +- +- if (demote != NULL && (demote->flags & pe_action_runnable)) { +- allowed = TRUE; +- } +- +- log_change(demote, "Demote %s\t(%s -> %s %s%s)", +- rsc->id, +- role2text(rsc->role), +- role2text(rsc->next_role), +- current->details->uname, allowed ? "" : " - blocked"); +- +- if (stop != NULL && is_not_set(stop->flags, pe_action_optional) +- && rsc->next_role > RSC_ROLE_STOPPED && moving == FALSE) { +- if (is_set(rsc->flags, pe_rsc_failed)) { +- log_change(stop, "Recover %s\t(%s %s)", +- rsc->id, role2text(rsc->role), next->details->uname); +- STOP_SANITY_ASSERT(__LINE__); +- +- } else if (is_set(rsc->flags, pe_rsc_reload)) { +- log_change(start, "Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), +- next->details->uname); +- +- } else { +- log_change(start, "Restart %s\t(%s %s)", +- rsc->id, role2text(rsc->next_role), next->details->uname); +- STOP_SANITY_ASSERT(__LINE__); +- } +- } +- } ++ if(stop ++ && (rsc->next_role == RSC_ROLE_STOPPED ++ || (start && is_not_set(start->flags, pe_action_runnable)))) { + +- } else if (rsc->next_role == RSC_ROLE_STOPPED) { + GListPtr gIter = NULL; + +- CRM_CHECK(current != NULL,); +- + key = stop_key(rsc); + for (gIter = rsc->running_on; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + action_t *stop_op = NULL; +- gboolean allowed = FALSE; + + possible_matches = find_actions(rsc->actions, key, node); + if (possible_matches) { +@@ -2357,73 +2420,39 @@ LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal) + + if (stop_op && (stop_op->flags & pe_action_runnable)) { + STOP_SANITY_ASSERT(__LINE__); +- allowed = TRUE; + } + +- log_change(start, "Stop %s\t(%s%s) %s", rsc->id, node->details->uname, +- allowed ? "" : " - blocked", stop->reason?stop->reason:""); ++ LogAction("Stop", rsc, node, NULL, stop_op, start, terminal); + } + + free(key); +- } + +- if (moving) { +- log_change(stop, "Move %s\t(%s %s -> %s)", +- rsc->id, role2text(rsc->next_role), current->details->uname, +- next->details->uname); ++ } else if (stop && is_set(rsc->flags, pe_rsc_failed)) { ++ /* 'stop' may be NULL if the failure was ignored */ ++ LogAction("Recover", rsc, current, next, stop, start, terminal); + STOP_SANITY_ASSERT(__LINE__); +- } + +- if (rsc->role == RSC_ROLE_STOPPED) { +- gboolean allowed = FALSE; ++ } else if (moving) { ++ LogAction("Move", rsc, current, next, stop, NULL, terminal); ++ STOP_SANITY_ASSERT(__LINE__); + +- if (start && (start->flags & pe_action_runnable)) { +- allowed = TRUE; +- } ++ } else if (is_set(rsc->flags, pe_rsc_reload)) { ++ LogAction("Reload", rsc, current, next, start, NULL, terminal); + +- CRM_CHECK(next != NULL,); +- if (next != NULL) { +- log_change(start, "Start %s\t(%s%s)", rsc->id, next->details->uname, +- allowed ? "" : " - blocked"); +- } +- if (allowed == FALSE) { +- return; +- } +- } ++ } else if (stop != NULL && is_not_set(stop->flags, pe_action_optional)) { ++ LogAction("Restart", rsc, current, next, start, NULL, terminal); ++ STOP_SANITY_ASSERT(__LINE__); + +- if (rsc->next_role > RSC_ROLE_SLAVE && rsc->role < rsc->next_role) { +- gboolean allowed = FALSE; ++ } else if (rsc->role == RSC_ROLE_MASTER) { ++ CRM_LOG_ASSERT(current != NULL); ++ LogAction("Demote", rsc, current, next, demote, NULL, terminal); + ++ } else if(rsc->next_role == RSC_ROLE_MASTER) { + CRM_LOG_ASSERT(next); +- if (stop != NULL && is_not_set(stop->flags, pe_action_optional) +- && rsc->role > RSC_ROLE_STOPPED) { +- if (is_set(rsc->flags, pe_rsc_failed)) { +- log_change(stop, "Recover %s\t(%s %s)", +- rsc->id, role2text(rsc->role), next?next->details->uname:NULL); +- STOP_SANITY_ASSERT(__LINE__); +- +- } else if (is_set(rsc->flags, pe_rsc_reload)) { +- log_change(start, "Reload %s\t(%s %s)", rsc->id, role2text(rsc->role), +- next?next->details->uname:NULL); +- STOP_SANITY_ASSERT(__LINE__); +- +- } else { +- log_change(start, "Restart %s\t(%s %s)", +- rsc->id, role2text(rsc->role), next?next->details->uname:NULL); +- STOP_SANITY_ASSERT(__LINE__); +- } +- } +- +- if (promote && (promote->flags & pe_action_runnable)) { +- allowed = TRUE; +- } ++ LogAction("Promote", rsc, current, next, promote, NULL, terminal); + +- log_change(promote, "Promote %s\t(%s -> %s %s%s)", +- rsc->id, +- role2text(rsc->role), +- role2text(rsc->next_role), +- next?next->details->uname:NULL, +- allowed ? "" : " - blocked"); ++ } else if (rsc->role == RSC_ROLE_STOPPED && rsc->next_role > RSC_ROLE_STOPPED) { ++ LogAction("Start", rsc, current, next, start, NULL, terminal); + } + } + +diff --git a/pengine/test10/1-a-then-bm-move-b.summary b/pengine/test10/1-a-then-bm-move-b.summary +index 571feda..f4d6207 100644 +--- a/pengine/test10/1-a-then-bm-move-b.summary ++++ b/pengine/test10/1-a-then-bm-move-b.summary +@@ -6,7 +6,7 @@ Online: [ 18node1 18node2 18node3 ] + B (ocf::heartbeat:Dummy): Started 18node2 + + Transition Summary: +- * Migrate B (Started 18node2 -> 18node1) ++ * Migrate B ( 18node2 -> 18node1 ) + + Executing cluster transition: + * Resource action: B migrate_to on 18node2 +diff --git a/pengine/test10/10-a-then-bm-b-move-a-clone.summary b/pengine/test10/10-a-then-bm-b-move-a-clone.summary +index b39963f..ee7697f 100644 +--- a/pengine/test10/10-a-then-bm-b-move-a-clone.summary ++++ b/pengine/test10/10-a-then-bm-b-move-a-clone.summary +@@ -9,7 +9,7 @@ Online: [ f20node2 ] + + Transition Summary: + * Stop myclone:1 (f20node1) due to node availability +- * Migrate vm (Started f20node1 -> f20node2) ++ * Migrate vm ( f20node1 -> f20node2 ) + + Executing cluster transition: + * Resource action: vm migrate_to on f20node1 +diff --git a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.summary b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.summary +index 498cd2b..9138c81 100644 +--- a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.summary ++++ b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.summary +@@ -9,8 +9,8 @@ Online: [ f20node2 ] + vm (ocf::heartbeat:Dummy): Started f20node1 + + Transition Summary: +- * Move myclone:0 (Started f20node1 -> f20node2) +- * Move vm (Started f20node1 -> f20node2) due to unrunnable myclone-clone stop ++ * Move myclone:0 ( f20node1 -> f20node2 ) ++ * Move vm ( f20node1 -> f20node2 ) due to unrunnable myclone-clone stop + + Executing cluster transition: + * Resource action: myclone monitor on f20node2 +diff --git a/pengine/test10/1360.summary b/pengine/test10/1360.summary +index 04ec941..acff1b8 100644 +--- a/pengine/test10/1360.summary ++++ b/pengine/test10/1360.summary +@@ -8,7 +8,7 @@ Online: [ ssgtest1a ssgtest1b ] + Started: [ ssgtest1a ] + + Transition Summary: +- * Move dollies:0 (Started ssgtest1a -> ssgtest1b) ++ * Move dollies:0 ( ssgtest1a -> ssgtest1b ) + + Executing cluster transition: + * Pseudo action: dolly_stop_0 +diff --git a/pengine/test10/2-am-then-b-move-a.summary b/pengine/test10/2-am-then-b-move-a.summary +index 1d74081..a238cf6 100644 +--- a/pengine/test10/2-am-then-b-move-a.summary ++++ b/pengine/test10/2-am-then-b-move-a.summary +@@ -6,7 +6,7 @@ Online: [ 18node1 18node2 18node3 ] + B (ocf::heartbeat:Dummy): Started 18node2 + + Transition Summary: +- * Migrate A (Started 18node1 -> 18node2) ++ * Migrate A ( 18node1 -> 18node2 ) + + Executing cluster transition: + * Resource action: A migrate_to on 18node1 +diff --git a/pengine/test10/3-am-then-bm-both-migrate.summary b/pengine/test10/3-am-then-bm-both-migrate.summary +index 6e1e33a..d938449 100644 +--- a/pengine/test10/3-am-then-bm-both-migrate.summary ++++ b/pengine/test10/3-am-then-bm-both-migrate.summary +@@ -6,8 +6,8 @@ Online: [ 18node1 18node2 18node3 ] + B (ocf::heartbeat:Dummy): Started 18node2 + + Transition Summary: +- * Migrate A (Started 18node1 -> 18node2) +- * Migrate B (Started 18node2 -> 18node1) ++ * Migrate A ( 18node1 -> 18node2 ) ++ * Migrate B ( 18node2 -> 18node1 ) + + Executing cluster transition: + * Resource action: A migrate_to on 18node1 +diff --git a/pengine/test10/4-am-then-bm-b-not-migratable.summary b/pengine/test10/4-am-then-bm-b-not-migratable.summary +index 2283c8b..16ce149 100644 +--- a/pengine/test10/4-am-then-bm-b-not-migratable.summary ++++ b/pengine/test10/4-am-then-bm-b-not-migratable.summary +@@ -6,8 +6,8 @@ Online: [ 18node1 18node2 18node3 ] + B (ocf::heartbeat:Dummy): Started 18node2 + + Transition Summary: +- * Migrate A (Started 18node1 -> 18node2) +- * Move B (Started 18node2 -> 18node1) ++ * Migrate A ( 18node1 -> 18node2 ) ++ * Move B ( 18node2 -> 18node1 ) + + Executing cluster transition: + * Resource action: B stop on 18node2 +diff --git a/pengine/test10/5-am-then-bm-a-not-migratable.summary b/pengine/test10/5-am-then-bm-a-not-migratable.summary +index 7e95dbb..fa1dc33 100644 +--- a/pengine/test10/5-am-then-bm-a-not-migratable.summary ++++ b/pengine/test10/5-am-then-bm-a-not-migratable.summary +@@ -6,8 +6,8 @@ Online: [ 18node1 18node2 18node3 ] + B (ocf::heartbeat:Dummy): Started 18node2 + + Transition Summary: +- * Move A (Started 18node1 -> 18node2) +- * Move B (Started 18node2 -> 18node1) due to unrunnable A stop ++ * Move A ( 18node1 -> 18node2 ) ++ * Move B ( 18node2 -> 18node1 ) due to unrunnable A stop + + Executing cluster transition: + * Resource action: B stop on 18node2 +diff --git a/pengine/test10/594.summary b/pengine/test10/594.summary +index de1d179..4a36789 100644 +--- a/pengine/test10/594.summary ++++ b/pengine/test10/594.summary +@@ -15,8 +15,8 @@ Online: [ hadev1 hadev2 ] + Transition Summary: + * Shutdown hadev2 + * Fence (reboot) hadev3 'peer is no longer part of the cluster' +- * Move DcIPaddr (Started hadev2 -> hadev1) +- * Move rsc_hadev2 (Started hadev2 -> hadev1) ++ * Move DcIPaddr ( hadev2 -> hadev1 ) ++ * Move rsc_hadev2 ( hadev2 -> hadev1 ) + * Stop child_DoFencing:0 (hadev2) due to node availability + * Stop child_DoFencing:2 (hadev1) due to node availability + +diff --git a/pengine/test10/6-migrate-group.summary b/pengine/test10/6-migrate-group.summary +index 3c4e7c6..c7c9f6d 100644 +--- a/pengine/test10/6-migrate-group.summary ++++ b/pengine/test10/6-migrate-group.summary +@@ -8,9 +8,9 @@ Online: [ 18node1 18node2 18node3 ] + C (ocf::heartbeat:Dummy): Started 18node1 + + Transition Summary: +- * Migrate A (Started 18node1 -> 18node2) +- * Migrate B (Started 18node1 -> 18node2) +- * Migrate C (Started 18node1 -> 18node2) ++ * Migrate A ( 18node1 -> 18node2 ) ++ * Migrate B ( 18node1 -> 18node2 ) ++ * Migrate C ( 18node1 -> 18node2 ) + + Executing cluster transition: + * Pseudo action: thegroup_stop_0 +diff --git a/pengine/test10/662.summary b/pengine/test10/662.summary +index 4a9d911..a19f71e 100644 +--- a/pengine/test10/662.summary ++++ b/pengine/test10/662.summary +@@ -15,7 +15,7 @@ Online: [ c001n02 c001n03 c001n04 c001n09 ] + + Transition Summary: + * Shutdown c001n02 +- * Move rsc_c001n02 (Started c001n02 -> c001n03) ++ * Move rsc_c001n02 ( c001n02 -> c001n03 ) + * Stop child_DoFencing:0 (c001n02) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/696.summary b/pengine/test10/696.summary +index af16c93..78e40d1 100644 +--- a/pengine/test10/696.summary ++++ b/pengine/test10/696.summary +@@ -12,7 +12,7 @@ Online: [ hadev1 hadev2 hadev3 ] + child_DoFencing:2 (stonith:ssh): Stopped + + Transition Summary: +- * Move rsc_hadev1 (Started hadev3 -> hadev1) ++ * Move rsc_hadev1 ( hadev3 -> hadev1 ) + * Start child_DoFencing:2 (hadev1) + + Executing cluster transition: +diff --git a/pengine/test10/7-migrate-group-one-unmigratable.summary b/pengine/test10/7-migrate-group-one-unmigratable.summary +index cf1b370..6c6b127 100644 +--- a/pengine/test10/7-migrate-group-one-unmigratable.summary ++++ b/pengine/test10/7-migrate-group-one-unmigratable.summary +@@ -8,9 +8,9 @@ Online: [ 18node1 18node2 18node3 ] + C (ocf::heartbeat:Dummy): Started 18node1 + + Transition Summary: +- * Migrate A (Started 18node1 -> 18node2) +- * Move B (Started 18node1 -> 18node2) +- * Move C (Started 18node1 -> 18node2) due to unrunnable B stop ++ * Migrate A ( 18node1 -> 18node2 ) ++ * Move B ( 18node1 -> 18node2 ) ++ * Move C ( 18node1 -> 18node2 ) due to unrunnable B stop + + Executing cluster transition: + * Pseudo action: thegroup_stop_0 +diff --git a/pengine/test10/726.summary b/pengine/test10/726.summary +index 920aefe..b3cd4e4 100644 +--- a/pengine/test10/726.summary ++++ b/pengine/test10/726.summary +@@ -15,7 +15,7 @@ Online: [ ibm1 sgi2 test02 test03 ] + + Transition Summary: + * Start rsc_sgi2 (sgi2) +- * Move rsc_ibm1 (Started test03 -> ibm1) ++ * Move rsc_ibm1 ( test03 -> ibm1 ) + * Start rsc_test02 (test02) + * Start child_DoFencing:2 (ibm1) + * Start child_DoFencing:3 (sgi2) +diff --git a/pengine/test10/735.summary b/pengine/test10/735.summary +index 69538de..2db520a 100644 +--- a/pengine/test10/735.summary ++++ b/pengine/test10/735.summary +@@ -13,7 +13,7 @@ OFFLINE: [ hadev1 ] + child_DoFencing:2 (stonith:ssh): Stopped + + Transition Summary: +- * Move rsc_hadev1 (Started hadev2 -> hadev3) ++ * Move rsc_hadev1 ( hadev2 -> hadev3 ) + * Start rsc_hadev3 (hadev3) + * Start child_DoFencing:0 (hadev2) + * Start child_DoFencing:1 (hadev3) +diff --git a/pengine/test10/764.summary b/pengine/test10/764.summary +index 0d5c612..ea62931 100644 +--- a/pengine/test10/764.summary ++++ b/pengine/test10/764.summary +@@ -15,11 +15,11 @@ OFFLINE: [ posic042 posic044 ] + child_DoFencing:3 (stonith:ssh): Stopped + + Transition Summary: +- * Stop DcIPaddr (Started posic043) due to no quorum +- * Stop rsc_posic041 (Started posic041) due to no quorum +- * Stop rsc_posic042 (Started posic041) due to no quorum +- * Stop rsc_posic043 (Started posic043) due to no quorum +- * Stop rsc_posic044 (Started posic041) due to no quorum ++ * Stop DcIPaddr ( posic043 ) due to no quorum ++ * Stop rsc_posic041 ( posic041 ) due to no quorum ++ * Stop rsc_posic042 ( posic041 ) due to no quorum ++ * Stop rsc_posic043 ( posic043 ) due to no quorum ++ * Stop rsc_posic044 ( posic041 ) due to no quorum + + Executing cluster transition: + * Resource action: DcIPaddr monitor on posic041 +diff --git a/pengine/test10/797.summary b/pengine/test10/797.summary +index 9e94003..61afbf1 100644 +--- a/pengine/test10/797.summary ++++ b/pengine/test10/797.summary +@@ -16,12 +16,12 @@ Online: [ c001n01 c001n02 c001n03 ] + + Transition Summary: + * Shutdown c001n02 +- * Stop DcIPaddr (Started c001n03) due to no quorum +- * Stop rsc_c001n08 (Started c001n02) due to no quorum +- * Stop rsc_c001n02 (Started c001n02) due to no quorum +- * Stop rsc_c001n03 (Started c001n03) due to no quorum +- * Stop rsc_c001n01 (Started c001n01) due to no quorum +- * Restart child_DoFencing:0 (Started c001n01) ++ * Stop DcIPaddr ( c001n03 ) due to no quorum ++ * Stop rsc_c001n08 ( c001n02 ) due to no quorum ++ * Stop rsc_c001n02 ( c001n02 ) due to no quorum ++ * Stop rsc_c001n03 ( c001n03 ) due to no quorum ++ * Stop rsc_c001n01 ( c001n01 ) due to no quorum ++ * Restart child_DoFencing:0 ( c001n01 ) + * Stop child_DoFencing:1 (c001n02) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/8-am-then-bm-a-migrating-b-stopping.summary b/pengine/test10/8-am-then-bm-a-migrating-b-stopping.summary +index 2aa3f07..06a1356 100644 +--- a/pengine/test10/8-am-then-bm-a-migrating-b-stopping.summary ++++ b/pengine/test10/8-am-then-bm-a-migrating-b-stopping.summary +@@ -7,7 +7,7 @@ Online: [ 18node1 18node2 18node3 ] + B (ocf::heartbeat:Dummy): Started 18node2 ( disabled ) + + Transition Summary: +- * Migrate A (Started 18node1 -> 18node2) ++ * Migrate A ( 18node1 -> 18node2 ) + * Stop B (18node2) + + Executing cluster transition: +diff --git a/pengine/test10/829.summary b/pengine/test10/829.summary +index feca908..8cc27dc 100644 +--- a/pengine/test10/829.summary ++++ b/pengine/test10/829.summary +@@ -16,7 +16,7 @@ Online: [ c001n01 c001n03 c001n08 ] + + Transition Summary: + * Fence (reboot) c001n02 'peer is no longer part of the cluster' +- * Move rsc_c001n02 (Started c001n02 -> c001n01) ++ * Move rsc_c001n02 ( c001n02 -> c001n01 ) + * Stop child_DoFencing:0 (c001n02) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/9-am-then-bm-b-migrating-a-stopping.summary b/pengine/test10/9-am-then-bm-b-migrating-a-stopping.summary +index 44fc1a2..250d888 100644 +--- a/pengine/test10/9-am-then-bm-b-migrating-a-stopping.summary ++++ b/pengine/test10/9-am-then-bm-b-migrating-a-stopping.summary +@@ -8,7 +8,7 @@ Online: [ 18node1 18node2 18node3 ] + + Transition Summary: + * Stop A (18node1) due to node availability +- * Stop B (Started 18node2) due to unrunnable A start ++ * Stop B ( 18node2 ) due to unrunnable A start + + Executing cluster transition: + * Resource action: B stop on 18node2 +diff --git a/pengine/test10/994-2.summary b/pengine/test10/994-2.summary +index a1d477f..0f7def7 100644 +--- a/pengine/test10/994-2.summary ++++ b/pengine/test10/994-2.summary +@@ -10,8 +10,8 @@ Online: [ paul ] + depends (lsb:postfix): Started paul + + Transition Summary: +- * Recover postfix_9 (Started paul) +- * Restart depends (Started paul) due to required group_1 running ++ * Recover postfix_9 ( paul ) ++ * Restart depends ( paul ) due to required group_1 running + + Executing cluster transition: + * Resource action: depends stop on paul +diff --git a/pengine/test10/994.summary b/pengine/test10/994.summary +index 6e8e4a2..4f0c42f 100644 +--- a/pengine/test10/994.summary ++++ b/pengine/test10/994.summary +@@ -9,7 +9,7 @@ Online: [ paul ] + postfix_9 (lsb:postfix): FAILED paul + + Transition Summary: +- * Recover postfix_9 (Started paul) ++ * Recover postfix_9 ( paul ) + + Executing cluster transition: + * Pseudo action: group_1_stop_0 +diff --git a/pengine/test10/a-demote-then-b-migrate.summary b/pengine/test10/a-demote-then-b-migrate.summary +index 84b250e..9e461e8 100644 +--- a/pengine/test10/a-demote-then-b-migrate.summary ++++ b/pengine/test10/a-demote-then-b-migrate.summary +@@ -8,9 +8,9 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Demote rsc1:0 (Master -> Slave node1) ++ * Demote rsc1:0 ( Master -> Slave node1 ) + * Promote rsc1:1 (Slave -> Master node2) +- * Migrate rsc2 (Started node1 -> node2) ++ * Migrate rsc2 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc1:1 cancel=5000 on node1 +diff --git a/pengine/test10/a-promote-then-b-migrate.summary b/pengine/test10/a-promote-then-b-migrate.summary +index c7d791b..166b7b0 100644 +--- a/pengine/test10/a-promote-then-b-migrate.summary ++++ b/pengine/test10/a-promote-then-b-migrate.summary +@@ -9,7 +9,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Promote rsc1:1 (Slave -> Master node2) +- * Migrate rsc2 (Started node1 -> node2) ++ * Migrate rsc2 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc1:1 cancel=10000 on node2 +diff --git a/pengine/test10/anti-colocation-master.summary b/pengine/test10/anti-colocation-master.summary +index 31fd635..df4c4ed 100644 +--- a/pengine/test10/anti-colocation-master.summary ++++ b/pengine/test10/anti-colocation-master.summary +@@ -10,9 +10,9 @@ Online: [ sle12sp2-1 sle12sp2-2 ] + Slaves: [ sle12sp2-2 ] + + Transition Summary: +- * Move dummy1 (Started sle12sp2-2 -> sle12sp2-1) ++ * Move dummy1 ( sle12sp2-2 -> sle12sp2-1 ) + * Promote state1:0 (Slave -> Master sle12sp2-2) +- * Demote state1:1 (Master -> Slave sle12sp2-1) ++ * Demote state1:1 ( Master -> Slave sle12sp2-1 ) + + Executing cluster transition: + * Resource action: dummy1 stop on sle12sp2-2 +diff --git a/pengine/test10/anti-colocation-order.summary b/pengine/test10/anti-colocation-order.summary +index 052043a..4f03a68 100644 +--- a/pengine/test10/anti-colocation-order.summary ++++ b/pengine/test10/anti-colocation-order.summary +@@ -11,8 +11,8 @@ Online: [ node2 ] + rsc4 (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Move rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) + * Stop rsc3 (node2) due to node availability + * Stop rsc4 (node2) due to node availability + +diff --git a/pengine/test10/anti-colocation-slave.summary b/pengine/test10/anti-colocation-slave.summary +index 9179e9f..0d77064 100644 +--- a/pengine/test10/anti-colocation-slave.summary ++++ b/pengine/test10/anti-colocation-slave.summary +@@ -9,9 +9,9 @@ Online: [ sle12sp2-1 sle12sp2-2 ] + dummy1 (ocf::pacemaker:Dummy): Started sle12sp2-1 + + Transition Summary: +- * Demote state1:0 (Master -> Slave sle12sp2-1) ++ * Demote state1:0 ( Master -> Slave sle12sp2-1 ) + * Promote state1:1 (Slave -> Master sle12sp2-2) +- * Move dummy1 (Started sle12sp2-1 -> sle12sp2-2) ++ * Move dummy1 ( sle12sp2-1 -> sle12sp2-2 ) + + Executing cluster transition: + * Resource action: dummy1 stop on sle12sp2-1 +diff --git a/pengine/test10/asymmetrical-order-move.summary b/pengine/test10/asymmetrical-order-move.summary +index 503813c..34869d5 100644 +--- a/pengine/test10/asymmetrical-order-move.summary ++++ b/pengine/test10/asymmetrical-order-move.summary +@@ -9,7 +9,7 @@ Online: [ sle12sp2-1 sle12sp2-2 ] + dummy2 (ocf::pacemaker:Dummy): Started sle12sp2-1 + + Transition Summary: +- * Stop dummy2 (Started sle12sp2-1) due to unrunnable dummy1 start ++ * Stop dummy2 ( sle12sp2-1 ) due to unrunnable dummy1 start + + Executing cluster transition: + * Resource action: dummy2 stop on sle12sp2-1 +diff --git a/pengine/test10/bug-1572-1.summary b/pengine/test10/bug-1572-1.summary +index 6a24614..7715980 100644 +--- a/pengine/test10/bug-1572-1.summary ++++ b/pengine/test10/bug-1572-1.summary +@@ -12,11 +12,11 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ] + + Transition Summary: + * Shutdown arc-dknightlx +- * Stop rsc_drbd_7788:0 (arc-dknightlx) due to node availability ++ * Stop rsc_drbd_7788:0 ( Slave arc-dknightlx ) due to node availability + * Restart rsc_drbd_7788:1 (Master arc-tkincaidlx.wsicorp.com) +- * Restart fs_mirror (Started arc-tkincaidlx.wsicorp.com) due to required ms_drbd_7788 notified +- * Restart pgsql_5555 (Started arc-tkincaidlx.wsicorp.com) due to required fs_mirror start +- * Restart IPaddr_147_81_84_133 (Started arc-tkincaidlx.wsicorp.com) due to required pgsql_5555 start ++ * Restart fs_mirror ( arc-tkincaidlx.wsicorp.com ) due to required ms_drbd_7788 notified ++ * Restart pgsql_5555 ( arc-tkincaidlx.wsicorp.com ) due to required fs_mirror start ++ * Restart IPaddr_147_81_84_133 ( arc-tkincaidlx.wsicorp.com ) due to required pgsql_5555 start + + Executing cluster transition: + * Pseudo action: ms_drbd_7788_pre_notify_demote_0 +diff --git a/pengine/test10/bug-1572-2.summary b/pengine/test10/bug-1572-2.summary +index 96574cf..9d2b885 100644 +--- a/pengine/test10/bug-1572-2.summary ++++ b/pengine/test10/bug-1572-2.summary +@@ -12,7 +12,7 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ] + + Transition Summary: + * Shutdown arc-dknightlx +- * Stop rsc_drbd_7788:0 (arc-dknightlx) due to node availability ++ * Stop rsc_drbd_7788:0 ( Slave arc-dknightlx ) due to node availability + * Demote rsc_drbd_7788:1 (Master -> Slave arc-tkincaidlx.wsicorp.com) + * Stop fs_mirror (arc-tkincaidlx.wsicorp.com) due to node availability + * Stop pgsql_5555 (arc-tkincaidlx.wsicorp.com) due to node availability +diff --git a/pengine/test10/bug-1718.summary b/pengine/test10/bug-1718.summary +index b539e4e..aec182e 100644 +--- a/pengine/test10/bug-1718.summary ++++ b/pengine/test10/bug-1718.summary +@@ -13,8 +13,8 @@ OFFLINE: [ defiant.ds9 warbird.ds9 ] + resource_dummy (ocf::heartbeat:Dummy): Started ops.ds9 + + Transition Summary: +- * Stop resource_IP3 (Started ops.ds9) due to unrunnable Web_Group running +- * Stop resource_dummy (Started ops.ds9) due to required resource_IP3 start ++ * Stop resource_IP3 ( ops.ds9 ) due to unrunnable Web_Group running ++ * Stop resource_dummy ( ops.ds9 ) due to required resource_IP3 start + + Executing cluster transition: + * Pseudo action: group_fUN_stop_0 +diff --git a/pengine/test10/bug-1820-1.summary b/pengine/test10/bug-1820-1.summary +index db41ac4..2b0673a 100644 +--- a/pengine/test10/bug-1820-1.summary ++++ b/pengine/test10/bug-1820-1.summary +@@ -10,8 +10,8 @@ Online: [ star world ] + Transition Summary: + * Shutdown star + * Start p1 (world) +- * Migrate test1 (Started star -> world) +- * Migrate test2 (Started star -> world) ++ * Migrate test1 ( star -> world ) ++ * Migrate test2 ( star -> world ) + + Executing cluster transition: + * Resource action: p1 monitor on world +diff --git a/pengine/test10/bug-1820.summary b/pengine/test10/bug-1820.summary +index 6d9c021..e13f8ad 100644 +--- a/pengine/test10/bug-1820.summary ++++ b/pengine/test10/bug-1820.summary +@@ -8,8 +8,8 @@ Online: [ star world ] + + Transition Summary: + * Shutdown star +- * Migrate test1 (Started star -> world) +- * Migrate test2 (Started star -> world) ++ * Migrate test1 ( star -> world ) ++ * Migrate test2 ( star -> world ) + + Executing cluster transition: + * Pseudo action: gr1_stop_0 +diff --git a/pengine/test10/bug-1822.summary b/pengine/test10/bug-1822.summary +index 325e408..5bf91b9 100644 +--- a/pengine/test10/bug-1822.summary ++++ b/pengine/test10/bug-1822.summary +@@ -12,8 +12,8 @@ Online: [ process1a process2b ] + + Transition Summary: + * Shutdown process1a +- * Demote master_slave_Stateful:1 (Master -> Stopped process1a) +- * Demote master_slave_procdctl:1 (Master -> Stopped process1a) ++ * Stop master_slave_Stateful:1 ( Master process1a ) due to node availability ++ * Stop master_slave_procdctl:1 ( Master process1a ) due to node availability + + Executing cluster transition: + * Pseudo action: ms-sf_demote_0 +diff --git a/pengine/test10/bug-5007-masterslave_colocation.summary b/pengine/test10/bug-5007-masterslave_colocation.summary +index 06a81e1..adbc1f1 100644 +--- a/pengine/test10/bug-5007-masterslave_colocation.summary ++++ b/pengine/test10/bug-5007-masterslave_colocation.summary +@@ -9,8 +9,8 @@ Online: [ fc16-builder fc16-builder2 ] + MASTER_IP (ocf::pacemaker:Dummy): Started fc16-builder2 + + Transition Summary: +- * Move SLAVE_IP (Started fc16-builder -> fc16-builder2) +- * Move MASTER_IP (Started fc16-builder2 -> fc16-builder) ++ * Move SLAVE_IP ( fc16-builder -> fc16-builder2 ) ++ * Move MASTER_IP ( fc16-builder2 -> fc16-builder ) + + Executing cluster transition: + * Resource action: SLAVE_IP stop on fc16-builder +diff --git a/pengine/test10/bug-5014-A-stopped-B-stopped.summary b/pengine/test10/bug-5014-A-stopped-B-stopped.summary +index 95e5b60..ba0a5e1 100644 +--- a/pengine/test10/bug-5014-A-stopped-B-stopped.summary ++++ b/pengine/test10/bug-5014-A-stopped-B-stopped.summary +@@ -7,7 +7,7 @@ Online: [ fc16-builder ] + ClusterIP2 (ocf::heartbeat:IPaddr2): Stopped + + Transition Summary: +- * Start ClusterIP2 (fc16-builder - blocked) due to unrunnable ClusterIP start ++ * Start ClusterIP2 ( fc16-builder ) due to unrunnable ClusterIP start (blocked) + + Executing cluster transition: + * Resource action: ClusterIP monitor on fc16-builder +diff --git a/pengine/test10/bug-5014-CthenAthenB-C-stopped.summary b/pengine/test10/bug-5014-CthenAthenB-C-stopped.summary +index fe12fe6..d15d1b2 100644 +--- a/pengine/test10/bug-5014-CthenAthenB-C-stopped.summary ++++ b/pengine/test10/bug-5014-CthenAthenB-C-stopped.summary +@@ -8,8 +8,8 @@ Online: [ fc16-builder ] + ClusterIP3 (ocf::heartbeat:IPaddr2): Stopped ( disabled ) + + Transition Summary: +- * Start ClusterIP (fc16-builder - blocked) due to unrunnable ClusterIP3 start +- * Start ClusterIP2 (fc16-builder - blocked) due to unrunnable ClusterIP start ++ * Start ClusterIP ( fc16-builder ) due to unrunnable ClusterIP3 start (blocked) ++ * Start ClusterIP2 ( fc16-builder ) due to unrunnable ClusterIP start (blocked) + + Executing cluster transition: + * Resource action: ClusterIP monitor on fc16-builder +diff --git a/pengine/test10/bug-5014-GROUP-A-stopped-B-stopped.summary b/pengine/test10/bug-5014-GROUP-A-stopped-B-stopped.summary +index f9b6448..aa4b699 100644 +--- a/pengine/test10/bug-5014-GROUP-A-stopped-B-stopped.summary ++++ b/pengine/test10/bug-5014-GROUP-A-stopped-B-stopped.summary +@@ -9,7 +9,7 @@ Online: [ fc16-builder ] + ClusterIP2 (ocf::heartbeat:IPaddr2): Stopped + + Transition Summary: +- * Start ClusterIP2 (fc16-builder - blocked) due to unrunnable group1 running ++ * Start ClusterIP2 ( fc16-builder ) due to unrunnable group1 running (blocked) + + Executing cluster transition: + +diff --git a/pengine/test10/bug-5014-ordered-set-symmetrical-true.summary b/pengine/test10/bug-5014-ordered-set-symmetrical-true.summary +index 7d94d60..7c28340 100644 +--- a/pengine/test10/bug-5014-ordered-set-symmetrical-true.summary ++++ b/pengine/test10/bug-5014-ordered-set-symmetrical-true.summary +@@ -9,7 +9,7 @@ OFFLINE: [ fc16-builder2 ] + C (ocf::pacemaker:Dummy): Started fc16-builder ( disabled ) + + Transition Summary: +- * Stop A (Started fc16-builder) due to required C start ++ * Stop A ( fc16-builder ) due to required C start + * Stop C (fc16-builder) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/bug-5025-1.summary b/pengine/test10/bug-5025-1.summary +index 9f9baa2..34f7b34 100644 +--- a/pengine/test10/bug-5025-1.summary ++++ b/pengine/test10/bug-5025-1.summary +@@ -7,7 +7,7 @@ OFFLINE: [ fc16-builder2 fc16-builder3 ] + A (ocf::pacemaker:Dummy): Started fc16-builder + + Transition Summary: +- * Reload A (Started fc16-builder) ++ * Reload A ( fc16-builder ) + + Executing cluster transition: + * Cluster action: clear_failcount for A on fc16-builder +diff --git a/pengine/test10/bug-5025-3.summary b/pengine/test10/bug-5025-3.summary +index 0d843d2..9072771 100644 +--- a/pengine/test10/bug-5025-3.summary ++++ b/pengine/test10/bug-5025-3.summary +@@ -8,7 +8,7 @@ OFFLINE: [ fc16-builder2 fc16-builder3 ] + B (ocf::pacemaker:Dummy): Started fc16-builder + + Transition Summary: +- * Restart A (Started fc16-builder) ++ * Restart A ( fc16-builder ) + + Executing cluster transition: + * Resource action: A stop on fc16-builder +diff --git a/pengine/test10/bug-5028.summary b/pengine/test10/bug-5028.summary +index a85f75b..f1c6f63 100644 +--- a/pengine/test10/bug-5028.summary ++++ b/pengine/test10/bug-5028.summary +@@ -8,7 +8,7 @@ Online: [ bl460g6a bl460g6b ] + + Transition Summary: + * Shutdown bl460g6a +- * Stop dummy01 (Started bl460g6a - blocked) ++ * Stop dummy01 ( bl460g6a ) blocked + + Executing cluster transition: + * Pseudo action: dummy-g_stop_0 +diff --git a/pengine/test10/bug-5059.summary b/pengine/test10/bug-5059.summary +index 36a5c67..3122cf9 100644 +--- a/pengine/test10/bug-5059.summary ++++ b/pengine/test10/bug-5059.summary +@@ -17,7 +17,6 @@ OFFLINE: [ gluster04.h ] + + Transition Summary: + * Promote p_stateful1:0 (Slave -> Master gluster01.h) +- * Start p_stateful2:0 (gluster01.h) + * Promote p_stateful2:0 (Stopped -> Master gluster01.h) + * Start p_stateful2:1 (gluster02.h) + +diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary +index cd01706..4d7f0bf 100644 +--- a/pengine/test10/bug-5186-partial-migrate.summary ++++ b/pengine/test10/bug-5186-partial-migrate.summary +@@ -26,10 +26,10 @@ Online: [ bl460g1n6 bl460g1n8 ] + + Transition Summary: + * Fence (reboot) bl460g1n7 'prmDummy is thought to be active there' +- * Move prmDummy (Started bl460g1n7 -> bl460g1n6) +- * Move prmVM2 (Started bl460g1n7 -> bl460g1n8) +- * Move prmStonith8-1 (Started bl460g1n7 -> bl460g1n6) +- * Move prmStonith8-2 (Started bl460g1n7 -> bl460g1n6) ++ * Move prmDummy ( bl460g1n7 -> bl460g1n6 ) ++ * Move prmVM2 ( bl460g1n7 -> bl460g1n8 ) ++ * Move prmStonith8-1 ( bl460g1n7 -> bl460g1n6 ) ++ * Move prmStonith8-2 ( bl460g1n7 -> bl460g1n6 ) + * Stop prmDiskd1:0 (bl460g1n7) due to node availability + * Stop prmDiskd2:0 (bl460g1n7) due to node availability + * Stop prmPing:0 (bl460g1n7) due to node availability +diff --git a/pengine/test10/bug-cl-5212.summary b/pengine/test10/bug-cl-5212.summary +index 92d3af3..1800f06 100644 +--- a/pengine/test10/bug-cl-5212.summary ++++ b/pengine/test10/bug-cl-5212.summary +@@ -20,13 +20,13 @@ Online: [ srv03 ] + Started: [ srv03 ] + + Transition Summary: +- * Stop prmStonith1-1 (Started srv02 - blocked) +- * Stop prmStonith2-1 (Started srv01 - blocked) +- * Stop prmStonith3-1 (srv01 - blocked) due to node availability +- * Stop pgsql:0 (srv02 - blocked) due to node availability +- * Demote pgsql:1 (Master -> Stopped srv01 - blocked) +- * Stop prmPingd:0 (srv02 - blocked) due to node availability +- * Stop prmPingd:1 (srv01 - blocked) due to node availability ++ * Stop prmStonith1-1 ( srv02 ) blocked ++ * Stop prmStonith2-1 ( srv01 ) blocked ++ * Stop prmStonith3-1 ( srv01 ) due to node availability (blocked) ++ * Stop pgsql:0 ( Slave srv02 ) due to node availability (blocked) ++ * Stop pgsql:1 ( Master srv01 ) due to node availability (blocked) ++ * Stop prmPingd:0 ( srv02 ) due to node availability (blocked) ++ * Stop prmPingd:1 ( srv01 ) due to node availability (blocked) + + Executing cluster transition: + * Pseudo action: grpStonith1_stop_0 +diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary +index 0f328cc..3e5511d 100644 +--- a/pengine/test10/bug-cl-5247.summary ++++ b/pengine/test10/bug-cl-5247.summary +@@ -20,11 +20,11 @@ Containers: [ pgsr01:prmDB1 ] + Transition Summary: + * Fence (off) pgsr02 (resource: prmDB2) 'guest is unclean' + * Stop prmDB2 (bl460g8n4) due to node availability +- * Restart prmStonith1-2 (Started bl460g8n4) +- * Restart prmStonith2-2 (Started bl460g8n3) +- * Recover vip-master (Started pgsr02 -> pgsr01) +- * Recover vip-rep (Started pgsr02 -> pgsr01) +- * Demote pgsql:0 (Master -> Stopped pgsr02) ++ * Restart prmStonith1-2 ( bl460g8n4 ) ++ * Restart prmStonith2-2 ( bl460g8n3 ) ++ * Recover vip-master ( pgsr02 -> pgsr01 ) ++ * Recover vip-rep ( pgsr02 -> pgsr01 ) ++ * Stop pgsql:0 ( Master pgsr02 ) due to node availability + * Stop pgsr02 (bl460g8n4) + + Executing cluster transition: +diff --git a/pengine/test10/bug-lf-2106.summary b/pengine/test10/bug-lf-2106.summary +index 9a71125..1cea829 100644 +--- a/pengine/test10/bug-lf-2106.summary ++++ b/pengine/test10/bug-lf-2106.summary +@@ -35,8 +35,8 @@ Online: [ cl-virt-1 cl-virt-2 ] + Slaves: [ cl-virt-1 ] + + Transition Summary: +- * Restart pingd:0 (Started cl-virt-1) +- * Restart pingd:1 (Started cl-virt-2) ++ * Restart pingd:0 ( cl-virt-1 ) ++ * Restart pingd:1 ( cl-virt-2 ) + + Executing cluster transition: + * Cluster action: clear_failcount for pingd on cl-virt-1 +diff --git a/pengine/test10/bug-lf-2153.summary b/pengine/test10/bug-lf-2153.summary +index e95713e..01567b5 100644 +--- a/pengine/test10/bug-lf-2153.summary ++++ b/pengine/test10/bug-lf-2153.summary +@@ -18,7 +18,7 @@ Online: [ alice ] + res_portblock_iscsivg01_unblock (ocf::heartbeat:portblock): Started alice + + Transition Summary: +- * Stop res_drbd_iscsivg01:0 (bob) due to node availability ++ * Stop res_drbd_iscsivg01:0 ( Slave bob ) due to node availability + * Stop res_tgtd:0 (bob) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/bug-lf-2171.summary b/pengine/test10/bug-lf-2171.summary +index 8c1d8a4..41a7b94 100644 +--- a/pengine/test10/bug-lf-2171.summary ++++ b/pengine/test10/bug-lf-2171.summary +@@ -12,8 +12,8 @@ Online: [ xenserver1 xenserver2 ] + Transition Summary: + * Stop res_Dummy1:0 (xenserver1) due to node availability + * Stop res_Dummy1:1 (xenserver2) due to node availability +- * Stop res_Dummy2 (Started xenserver1) due to unrunnable cl_res_Dummy1 running +- * Stop res_Dummy3 (Started xenserver1) due to unrunnable cl_res_Dummy1 running ++ * Stop res_Dummy2 ( xenserver1 ) due to unrunnable cl_res_Dummy1 running ++ * Stop res_Dummy3 ( xenserver1 ) due to unrunnable cl_res_Dummy1 running + + Executing cluster transition: + * Pseudo action: gr_Dummy_stop_0 +diff --git a/pengine/test10/bug-lf-2361.summary b/pengine/test10/bug-lf-2361.summary +index 3089e04..b88cd90 100644 +--- a/pengine/test10/bug-lf-2361.summary ++++ b/pengine/test10/bug-lf-2361.summary +@@ -11,8 +11,8 @@ Online: [ alice.demo bob.demo ] + Transition Summary: + * Start stateful:0 (alice.demo) + * Start stateful:1 (bob.demo) +- * Start dummy2:0 (alice.demo - blocked) due to unrunnable dummy1 start +- * Start dummy2:1 (bob.demo - blocked) due to unrunnable dummy1 start ++ * Start dummy2:0 ( alice.demo ) due to unrunnable dummy1 start (blocked) ++ * Start dummy2:1 ( bob.demo ) due to unrunnable dummy1 start (blocked) + + Executing cluster transition: + * Pseudo action: ms_stateful_pre_notify_start_0 +diff --git a/pengine/test10/bug-lf-2435.summary b/pengine/test10/bug-lf-2435.summary +index a538342..bb07089 100644 +--- a/pengine/test10/bug-lf-2435.summary ++++ b/pengine/test10/bug-lf-2435.summary +@@ -9,7 +9,7 @@ Online: [ c19.chepkov.lan c21.chepkov.lan ] + dummy3 (ocf::pacemaker:Dummy): Started c21.chepkov.lan + + Transition Summary: +- * Move dummy2 (Started c20.chepkov.lan -> c21.chepkov.lan) ++ * Move dummy2 ( c20.chepkov.lan -> c21.chepkov.lan ) + * Stop dummy3 (c21.chepkov.lan) + + Executing cluster transition: +diff --git a/pengine/test10/bug-lf-2445.summary b/pengine/test10/bug-lf-2445.summary +index 4523d9b..12f5031 100644 +--- a/pengine/test10/bug-lf-2445.summary ++++ b/pengine/test10/bug-lf-2445.summary +@@ -7,7 +7,7 @@ Online: [ node1 node2 ] + P:1 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Move P:1 (Started node1 -> node2) ++ * Move P:1 ( node1 -> node2 ) + + Executing cluster transition: + * Pseudo action: C_stop_0 +diff --git a/pengine/test10/bug-lf-2453.summary b/pengine/test10/bug-lf-2453.summary +index 398868b..f038cbb 100644 +--- a/pengine/test10/bug-lf-2453.summary ++++ b/pengine/test10/bug-lf-2453.summary +@@ -10,11 +10,11 @@ Online: [ domu1 domu2 ] + Started: [ domu1 domu2 ] + + Transition Summary: +- * Stop PrimitiveResource1 (Started domu1) due to required CloneResource2 running ++ * Stop PrimitiveResource1 ( domu1 ) due to required CloneResource2 running + * Stop apache:0 (domu1) due to node availability + * Stop apache:1 (domu2) due to node availability +- * Stop DummyResource:0 (Started domu1) due to unrunnable CloneResource1 running +- * Stop DummyResource:1 (Started domu2) due to unrunnable CloneResource1 running ++ * Stop DummyResource:0 ( domu1 ) due to unrunnable CloneResource1 running ++ * Stop DummyResource:1 ( domu2 ) due to unrunnable CloneResource1 running + + Executing cluster transition: + * Resource action: PrimitiveResource1 stop on domu1 +diff --git a/pengine/test10/bug-lf-2508.summary b/pengine/test10/bug-lf-2508.summary +index 5d0d90c..d4e548a 100644 +--- a/pengine/test10/bug-lf-2508.summary ++++ b/pengine/test10/bug-lf-2508.summary +@@ -37,7 +37,7 @@ Online: [ srv01 srv03 srv04 ] + Transition Summary: + * Fence (reboot) srv02 'peer is no longer part of the cluster' + * Start Dummy01 (srv01) +- * Move Dummy02 (Started srv02 -> srv04) ++ * Move Dummy02 ( srv02 -> srv04 ) + * Stop prmStonith1-1:1 (srv02) due to node availability + * Stop prmStonith1-3:1 (srv02) due to node availability + * Stop prmStonith3-1:0 (srv02) due to node availability +diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary +index 9537277..1cd5108 100644 +--- a/pengine/test10/bug-lf-2551.summary ++++ b/pengine/test10/bug-lf-2551.summary +@@ -82,8 +82,8 @@ Online: [ hex-0 hex-7 hex-8 ] + + Transition Summary: + * Fence (reboot) hex-9 'peer is no longer part of the cluster' +- * Move fencing-sbd (Started hex-9 -> hex-0) +- * Move dummy1 (Started hex-9 -> hex-0) ++ * Move fencing-sbd ( hex-9 -> hex-0 ) ++ * Move dummy1 ( hex-9 -> hex-0 ) + * Stop dlm:3 (hex-9) due to node availability + * Stop o2cb:3 (hex-9) due to node availability + * Stop clvm:3 (hex-9) due to node availability +diff --git a/pengine/test10/bug-lf-2574.summary b/pengine/test10/bug-lf-2574.summary +index 800453c..8033616 100644 +--- a/pengine/test10/bug-lf-2574.summary ++++ b/pengine/test10/bug-lf-2574.summary +@@ -11,7 +11,7 @@ Online: [ srv01 srv02 srv03 ] + Started: [ srv01 srv02 srv03 ] + + Transition Summary: +- * Move main_rsc (Started srv01 -> srv03) ++ * Move main_rsc ( srv01 -> srv03 ) + * Stop prmPingd:0 (srv01) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/bug-lf-2606.summary b/pengine/test10/bug-lf-2606.summary +index 7e889c9..d3fdb24 100644 +--- a/pengine/test10/bug-lf-2606.summary ++++ b/pengine/test10/bug-lf-2606.summary +@@ -14,8 +14,8 @@ Online: [ node1 ] + Transition Summary: + * Fence (reboot) node2 'rsc1 failed there' + * Stop rsc1 (node2) +- * Move rsc2 (Started node2 -> node1) +- * Demote rsc3:1 (Master -> Stopped node2) ++ * Move rsc2 ( node2 -> node1 ) ++ * Stop rsc3:1 ( Master node2 ) due to node availability + + Executing cluster transition: + * Pseudo action: ms3_demote_0 +diff --git a/pengine/test10/bug-lf-2613.summary b/pengine/test10/bug-lf-2613.summary +index 4312c51..51078c0 100644 +--- a/pengine/test10/bug-lf-2613.summary ++++ b/pengine/test10/bug-lf-2613.summary +@@ -25,12 +25,12 @@ Online: [ act1 act2 act3 sby1 sby2 ] + prmApPostgreSQLDB3 (ocf::pacemaker:Dummy): Started act3 + + Transition Summary: +- * Move prmExPostgreSQLDB1 (Started act1 -> sby1) +- * Move prmFsPostgreSQLDB1-1 (Started act1 -> sby1) +- * Move prmFsPostgreSQLDB1-2 (Started act1 -> sby1) +- * Move prmFsPostgreSQLDB1-3 (Started act1 -> sby1) +- * Move prmIpPostgreSQLDB1 (Started act1 -> sby1) +- * Recover prmApPostgreSQLDB1 (Started act1 -> sby1) ++ * Move prmExPostgreSQLDB1 ( act1 -> sby1 ) ++ * Move prmFsPostgreSQLDB1-1 ( act1 -> sby1 ) ++ * Move prmFsPostgreSQLDB1-2 ( act1 -> sby1 ) ++ * Move prmFsPostgreSQLDB1-3 ( act1 -> sby1 ) ++ * Move prmIpPostgreSQLDB1 ( act1 -> sby1 ) ++ * Recover prmApPostgreSQLDB1 ( act1 -> sby1 ) + + Executing cluster transition: + * Pseudo action: grpPostgreSQLDB1_stop_0 +diff --git a/pengine/test10/bug-lf-2619.summary b/pengine/test10/bug-lf-2619.summary +index 9a2213d..a704be8 100644 +--- a/pengine/test10/bug-lf-2619.summary ++++ b/pengine/test10/bug-lf-2619.summary +@@ -28,12 +28,12 @@ Online: [ act1 act2 act3 sby1 sby2 ] + Started: [ act2 act3 sby1 sby2 ] + + Transition Summary: +- * Move prmExPostgreSQLDB1 (Started act1 -> sby1) +- * Move prmFsPostgreSQLDB1-1 (Started act1 -> sby1) +- * Move prmFsPostgreSQLDB1-2 (Started act1 -> sby1) +- * Move prmFsPostgreSQLDB1-3 (Started act1 -> sby1) +- * Move prmIpPostgreSQLDB1 (Started act1 -> sby1) +- * Move prmApPostgreSQLDB1 (Started act1 -> sby1) ++ * Move prmExPostgreSQLDB1 ( act1 -> sby1 ) ++ * Move prmFsPostgreSQLDB1-1 ( act1 -> sby1 ) ++ * Move prmFsPostgreSQLDB1-2 ( act1 -> sby1 ) ++ * Move prmFsPostgreSQLDB1-3 ( act1 -> sby1 ) ++ * Move prmIpPostgreSQLDB1 ( act1 -> sby1 ) ++ * Move prmApPostgreSQLDB1 ( act1 -> sby1 ) + * Stop prmPingd:0 (act1) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/bug-n-385265-2.summary b/pengine/test10/bug-n-385265-2.summary +index f02554d..2e0f7c1 100644 +--- a/pengine/test10/bug-n-385265-2.summary ++++ b/pengine/test10/bug-n-385265-2.summary +@@ -7,8 +7,8 @@ Online: [ ih01 ih02 ] + resource_idvscommon (ocf::dfs:idvs): Started ih02 + + Transition Summary: +- * Recover resource_ip_common (Started ih02 -> ih01) +- * Move resource_idvscommon (Started ih02 -> ih01) ++ * Recover resource_ip_common ( ih02 -> ih01 ) ++ * Move resource_idvscommon ( ih02 -> ih01 ) + + Executing cluster transition: + * Pseudo action: group_common_stop_0 +diff --git a/pengine/test10/bug-n-387749.summary b/pengine/test10/bug-n-387749.summary +index 2532d35..9ef7e70 100644 +--- a/pengine/test10/bug-n-387749.summary ++++ b/pengine/test10/bug-n-387749.summary +@@ -13,8 +13,8 @@ OFFLINE: [ power720-4 ] + + Transition Summary: + * Start export_home_ocfs2:0 (power720-1) +- * Move resource_ipaddr1_single (Started power720-2 -> power720-1) +- * Move resource_nfsserver_single (Started power720-2 -> power720-1) ++ * Move resource_ipaddr1_single ( power720-2 -> power720-1 ) ++ * Move resource_nfsserver_single ( power720-2 -> power720-1 ) + + Executing cluster transition: + * Resource action: export_home_ocfs2:0 monitor on power720-1 +diff --git a/pengine/test10/bug-pm-11.summary b/pengine/test10/bug-pm-11.summary +index 07f2d9a..dc26a2e 100644 +--- a/pengine/test10/bug-pm-11.summary ++++ b/pengine/test10/bug-pm-11.summary +@@ -12,7 +12,6 @@ Online: [ node-a node-b ] + + Transition Summary: + * Start stateful-2:0 (node-b) +- * Start stateful-2:1 (node-a) + * Promote stateful-2:1 (Stopped -> Master node-a) + + Executing cluster transition: +diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary +index ee21df8..c50df5f 100644 +--- a/pengine/test10/bug-rh-1097457.summary ++++ b/pengine/test10/bug-rh-1097457.summary +@@ -33,12 +33,12 @@ Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ] + + Transition Summary: + * Fence (reboot) lamaVM2 (resource: VM2) 'guest is unclean' +- * Recover VM2 (Started lama3) +- * Recover FSlun3 (Started lamaVM2 -> lama2) +- * Restart FAKE4 (Started lamaVM2) due to required VM2 start +- * Restart FAKE4-IP (Started lamaVM2) due to required VM2 start +- * Restart FAKE6:2 (Started lamaVM2) due to required VM2 start +- * Restart lamaVM2 (Started lama3) due to required VM2 start ++ * Recover VM2 ( lama3 ) ++ * Recover FSlun3 ( lamaVM2 -> lama2 ) ++ * Restart FAKE4 ( lamaVM2 ) due to required VM2 start ++ * Restart FAKE4-IP ( lamaVM2 ) due to required VM2 start ++ * Restart FAKE6:2 ( lamaVM2 ) due to required VM2 start ++ * Restart lamaVM2 ( lama3 ) due to required VM2 start + + Executing cluster transition: + * Resource action: lamaVM2 stop on lama3 +diff --git a/pengine/test10/bug-rh-880249.summary b/pengine/test10/bug-rh-880249.summary +index d4f5333..766d326 100644 +--- a/pengine/test10/bug-rh-880249.summary ++++ b/pengine/test10/bug-rh-880249.summary +@@ -7,8 +7,7 @@ Online: [ 18node1 18node2 18node3 ] + dummystateful (ocf::pacemaker:Stateful): Master [ 18node2 18node1 18node3 ] + + Transition Summary: +- * Demote dummystateful (Master -> Started 18node2) +- * Move dummystateful (Started 18node2 -> 18node3) ++ * Move dummystateful ( Master 18node2 -> Started 18node3 ) + + Executing cluster transition: + * Resource action: dummystateful demote on 18node3 +diff --git a/pengine/test10/bug-suse-707150.summary b/pengine/test10/bug-suse-707150.summary +index d6922ab..6e5a025 100644 +--- a/pengine/test10/bug-suse-707150.summary ++++ b/pengine/test10/bug-suse-707150.summary +@@ -26,7 +26,7 @@ Transition Summary: + * Start o2cb:1 (hex-9) + * Start clvm:1 (hex-9) + * Start cmirrord:1 (hex-9) +- * Start vm-01 (hex-9 - blocked) due to unrunnable base-clone running ++ * Start vm-01 ( hex-9 ) due to unrunnable base-clone running (blocked) + + Executing cluster transition: + * Resource action: vg1:1 monitor on hex-9 +diff --git a/pengine/test10/bundle-nested-colocation.summary b/pengine/test10/bundle-nested-colocation.summary +index 9755e5f..0e2a68d 100644 +--- a/pengine/test10/bundle-nested-colocation.summary ++++ b/pengine/test10/bundle-nested-colocation.summary +@@ -17,13 +17,13 @@ RemoteOnline: [ overcloud-rabbit-0 overcloud-rabbit-1 overcloud-rabbit-2 ] + galera-bundle-docker-2 (ocf::heartbeat:docker): Started overcloud-galera-2 + + Transition Summary: +- * Restart rabbitmq-bundle-docker-0 (Started overcloud-rabbit-0) ++ * Restart rabbitmq-bundle-docker-0 ( overcloud-rabbit-0 ) + * Start rabbitmq-bundle-0 (overcloud-controller-0) + * Start rabbitmq:0 (rabbitmq-bundle-0) +- * Restart rabbitmq-bundle-docker-1 (Started overcloud-rabbit-1) ++ * Restart rabbitmq-bundle-docker-1 ( overcloud-rabbit-1 ) + * Start rabbitmq-bundle-1 (overcloud-controller-1) + * Start rabbitmq:1 (rabbitmq-bundle-1) +- * Restart rabbitmq-bundle-docker-2 (Started overcloud-rabbit-2) ++ * Restart rabbitmq-bundle-docker-2 ( overcloud-rabbit-2 ) + * Start rabbitmq-bundle-2 (overcloud-controller-2) + * Start rabbitmq:2 (rabbitmq-bundle-2) + +diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary +index d67f8fc..790bd17 100644 +--- a/pengine/test10/bundle-order-partial-start-2.summary ++++ b/pengine/test10/bundle-order-partial-start-2.summary +@@ -22,8 +22,8 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 rabbitmq-bundle-0:rabbitmq- + + Transition Summary: + * Start rabbitmq:0 (rabbitmq-bundle-0) +- * Restart galera-bundle-docker-0 (Started undercloud) due to required haproxy-bundle running +- * Restart galera-bundle-0 (Started undercloud) due to required galera-bundle-docker-0 start ++ * Restart galera-bundle-docker-0 ( undercloud ) due to required haproxy-bundle running ++ * Restart galera-bundle-0 ( undercloud ) due to required galera-bundle-docker-0 start + * Start galera:0 (galera-bundle-0) + * Promote redis:0 (Slave -> Master redis-bundle-0) + * Start haproxy-bundle-docker-0 (undercloud) +diff --git a/pengine/test10/bundle-order-partial-stop.summary b/pengine/test10/bundle-order-partial-stop.summary +index e7bac73..7bc24d2 100644 +--- a/pengine/test10/bundle-order-partial-stop.summary ++++ b/pengine/test10/bundle-order-partial-stop.summary +@@ -24,15 +24,13 @@ Transition Summary: + * Shutdown undercloud + * Stop rabbitmq-bundle-docker-0 (undercloud) due to node availability + * Stop rabbitmq-bundle-0 (undercloud) due to node availability +- * Stop rabbitmq:0 (Started rabbitmq-bundle-0) due to unrunnable rabbitmq-bundle-0 start ++ * Stop rabbitmq:0 ( rabbitmq-bundle-0 ) due to unrunnable rabbitmq-bundle-0 start + * Stop galera-bundle-docker-0 (undercloud) due to node availability + * Stop galera-bundle-0 (undercloud) due to node availability +- * Demote galera:0 (Master -> Slave galera-bundle-0) +- * Restart galera:0 (Slave galera-bundle-0) due to unrunnable galera-bundle-0 start ++ * Stop galera:0 ( Master galera-bundle-0 ) due to unrunnable galera-bundle-0 start + * Stop redis-bundle-docker-0 (undercloud) due to node availability + * Stop redis-bundle-0 (undercloud) due to node availability +- * Demote redis:0 (Master -> Slave redis-bundle-0) +- * Restart redis:0 (Slave redis-bundle-0) due to unrunnable redis-bundle-0 start ++ * Stop redis:0 ( Master redis-bundle-0 ) due to unrunnable redis-bundle-0 start + * Stop ip-192.168.122.254 (undercloud) due to node availability + * Stop ip-192.168.122.250 (undercloud) due to node availability + * Stop ip-192.168.122.249 (undercloud) due to node availability +diff --git a/pengine/test10/bundle-order-startup-clone-2.summary b/pengine/test10/bundle-order-startup-clone-2.summary +index e23d933..9b0eeaa 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.summary ++++ b/pengine/test10/bundle-order-startup-clone-2.summary +@@ -36,15 +36,12 @@ Transition Summary: + * Start haproxy-bundle-docker-2 (metal-3) + * Start redis-bundle-docker-0 (metal-1) + * Start redis-bundle-0 (metal-1) +- * Start redis:0 (redis-bundle-0) + * Promote redis:0 (Stopped -> Master redis-bundle-0) + * Start redis-bundle-docker-1 (metal-2) + * Start redis-bundle-1 (metal-2) +- * Start redis:1 (redis-bundle-1) + * Promote redis:1 (Stopped -> Master redis-bundle-1) + * Start redis-bundle-docker-2 (metal-3) + * Start redis-bundle-2 (metal-3) +- * Start redis:2 (redis-bundle-2) + * Promote redis:2 (Stopped -> Master redis-bundle-2) + + Executing cluster transition: +diff --git a/pengine/test10/bundle-order-startup-clone.summary b/pengine/test10/bundle-order-startup-clone.summary +index 0acfd1e..5e826dc 100644 +--- a/pengine/test10/bundle-order-startup-clone.summary ++++ b/pengine/test10/bundle-order-startup-clone.summary +@@ -13,12 +13,12 @@ RemoteOFFLINE: [ rabbitmq-bundle-0 ] + redis-bundle-0 (ocf::heartbeat:redis): Stopped + + Transition Summary: +- * Start storage:0 (metal-1 - blocked) due to unrunnable redis-bundle promoted +- * Start storage:1 (metal-2 - blocked) due to unrunnable redis-bundle promoted +- * Start storage:2 (metal-3 - blocked) due to unrunnable redis-bundle promoted +- * Start galera-bundle-docker-0 (metal-1 - blocked) due to unrunnable storage-clone notified +- * Start galera-bundle-0 (metal-1 - blocked) due to unrunnable galera-bundle-docker-0 start +- * Start galera:0 (galera-bundle-0 - blocked) due to unrunnable galera-bundle-docker-0 start ++ * Start storage:0 ( metal-1 ) due to unrunnable redis-bundle promoted (blocked) ++ * Start storage:1 ( metal-2 ) due to unrunnable redis-bundle promoted (blocked) ++ * Start storage:2 ( metal-3 ) due to unrunnable redis-bundle promoted (blocked) ++ * Start galera-bundle-docker-0 ( metal-1 ) due to unrunnable storage-clone notified (blocked) ++ * Start galera-bundle-0 ( metal-1 ) due to unrunnable galera-bundle-docker-0 start (blocked) ++ * Start galera:0 ( galera-bundle-0 ) due to unrunnable galera-bundle-docker-0 start (blocked) + * Start haproxy-bundle-docker-0 (metal-2) + * Start redis-bundle-docker-0 (metal-2) + * Start redis-bundle-0 (metal-2) +diff --git a/pengine/test10/bundle-order-stop-clone.summary b/pengine/test10/bundle-order-stop-clone.summary +index 9a6b0f2..66cb82a 100644 +--- a/pengine/test10/bundle-order-stop-clone.summary ++++ b/pengine/test10/bundle-order-stop-clone.summary +@@ -23,7 +23,7 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund + Transition Summary: + * Stop storage:0 (metal-1) due to node availability + * Stop galera-bundle-docker-0 (metal-1) due to node availability +- * Stop galera-bundle-0 (Started metal-1) due to unrunnable galera-bundle-docker-0 start ++ * Stop galera-bundle-0 ( metal-1 ) due to unrunnable galera-bundle-docker-0 start + * Stop galera:0 (Slave galera-bundle-0) due to unrunnable galera-bundle-docker-0 start + + Executing cluster transition: +diff --git a/pengine/test10/bundle-order-stop.summary b/pengine/test10/bundle-order-stop.summary +index e7bac73..7bc24d2 100644 +--- a/pengine/test10/bundle-order-stop.summary ++++ b/pengine/test10/bundle-order-stop.summary +@@ -24,15 +24,13 @@ Transition Summary: + * Shutdown undercloud + * Stop rabbitmq-bundle-docker-0 (undercloud) due to node availability + * Stop rabbitmq-bundle-0 (undercloud) due to node availability +- * Stop rabbitmq:0 (Started rabbitmq-bundle-0) due to unrunnable rabbitmq-bundle-0 start ++ * Stop rabbitmq:0 ( rabbitmq-bundle-0 ) due to unrunnable rabbitmq-bundle-0 start + * Stop galera-bundle-docker-0 (undercloud) due to node availability + * Stop galera-bundle-0 (undercloud) due to node availability +- * Demote galera:0 (Master -> Slave galera-bundle-0) +- * Restart galera:0 (Slave galera-bundle-0) due to unrunnable galera-bundle-0 start ++ * Stop galera:0 ( Master galera-bundle-0 ) due to unrunnable galera-bundle-0 start + * Stop redis-bundle-docker-0 (undercloud) due to node availability + * Stop redis-bundle-0 (undercloud) due to node availability +- * Demote redis:0 (Master -> Slave redis-bundle-0) +- * Restart redis:0 (Slave redis-bundle-0) due to unrunnable redis-bundle-0 start ++ * Stop redis:0 ( Master redis-bundle-0 ) due to unrunnable redis-bundle-0 start + * Stop ip-192.168.122.254 (undercloud) due to node availability + * Stop ip-192.168.122.250 (undercloud) due to node availability + * Stop ip-192.168.122.249 (undercloud) due to node availability +diff --git a/pengine/test10/clone-anon-failcount.summary b/pengine/test10/clone-anon-failcount.summary +index 3fb39e3..098b772 100644 +--- a/pengine/test10/clone-anon-failcount.summary ++++ b/pengine/test10/clone-anon-failcount.summary +@@ -37,12 +37,12 @@ Online: [ srv01 srv02 srv03 srv04 ] + Started: [ srv01 srv02 srv03 srv04 ] + + Transition Summary: +- * Move UmVIPcheck (Started srv01 -> srv04) +- * Move UmIPaddr (Started srv01 -> srv04) +- * Move UmDummy01 (Started srv01 -> srv04) +- * Move UmDummy02 (Started srv01 -> srv04) +- * Recover clnUMdummy01:0 (Started srv04) +- * Restart clnUMdummy02:0 (Started srv04) due to required clnUMdummy01:0 start ++ * Move UmVIPcheck ( srv01 -> srv04 ) ++ * Move UmIPaddr ( srv01 -> srv04 ) ++ * Move UmDummy01 ( srv01 -> srv04 ) ++ * Move UmDummy02 ( srv01 -> srv04 ) ++ * Recover clnUMdummy01:0 ( srv04 ) ++ * Restart clnUMdummy02:0 ( srv04 ) due to required clnUMdummy01:0 start + * Stop clnUMdummy01:1 (srv01) due to node availability + * Stop clnUMdummy02:1 (srv01) due to node availability + +diff --git a/pengine/test10/clone-fail-block-colocation.summary b/pengine/test10/clone-fail-block-colocation.summary +index 9b9167a..7f3f765 100644 +--- a/pengine/test10/clone-fail-block-colocation.summary ++++ b/pengine/test10/clone-fail-block-colocation.summary +@@ -16,10 +16,10 @@ Online: [ DEM-1 DEM-2 ] + Started: [ DEM-1 DEM-2 ] + + Transition Summary: +- * Move ipv6_dem_tas_dns (Started DEM-1 -> DEM-2) +- * Move d_bird_subnet_state (Started DEM-1 -> DEM-2) +- * Move ip_mgmt (Started DEM-1 -> DEM-2) +- * Move ip_trf_tas (Started DEM-1 -> DEM-2) ++ * Move ipv6_dem_tas_dns ( DEM-1 -> DEM-2 ) ++ * Move d_bird_subnet_state ( DEM-1 -> DEM-2 ) ++ * Move ip_mgmt ( DEM-1 -> DEM-2 ) ++ * Move ip_trf_tas ( DEM-1 -> DEM-2 ) + + Executing cluster transition: + * Pseudo action: svc_stop_0 +diff --git a/pengine/test10/clone-interleave-2.summary b/pengine/test10/clone-interleave-2.summary +index 78d46cd..e4c9aa4 100644 +--- a/pengine/test10/clone-interleave-2.summary ++++ b/pengine/test10/clone-interleave-2.summary +@@ -11,7 +11,7 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Started: [ pcmk-1 pcmk-2 pcmk-3 ] + + Transition Summary: +- * Restart dummy (Started pcmk-1) due to required clone-3 running ++ * Restart dummy ( pcmk-1 ) due to required clone-3 running + * Stop child-2:0 (pcmk-1) due to node availability + * Stop child-3:0 (pcmk-1) + +diff --git a/pengine/test10/clone-interleave-3.summary b/pengine/test10/clone-interleave-3.summary +index 8b13dc4..a40dc1d 100644 +--- a/pengine/test10/clone-interleave-3.summary ++++ b/pengine/test10/clone-interleave-3.summary +@@ -12,9 +12,9 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Started: [ pcmk-1 pcmk-2 pcmk-3 ] + + Transition Summary: +- * Restart dummy (Started pcmk-1) due to required clone-3 running +- * Recover child-2:0 (Started pcmk-1) +- * Restart child-3:0 (Started pcmk-1) due to required child-2:0 start ++ * Restart dummy ( pcmk-1 ) due to required clone-3 running ++ * Recover child-2:0 ( pcmk-1 ) ++ * Restart child-3:0 ( pcmk-1 ) due to required child-2:0 start + + Executing cluster transition: + * Resource action: dummy stop on pcmk-1 +diff --git a/pengine/test10/clone-no-shuffle.summary b/pengine/test10/clone-no-shuffle.summary +index 59ffbbe..c69d8f3 100644 +--- a/pengine/test10/clone-no-shuffle.summary ++++ b/pengine/test10/clone-no-shuffle.summary +@@ -10,7 +10,7 @@ Online: [ dktest1sles10 dktest2sles10 ] + + Transition Summary: + * Start stonith-1 (dktest1sles10) +- * Demote drbd1:0 (Master -> Stopped dktest2sles10) ++ * Stop drbd1:0 ( Master dktest2sles10 ) due to node availability + * Start drbd1:1 (dktest1sles10) + * Stop testip (dktest2sles10) + +diff --git a/pengine/test10/clone-require-all-2.summary b/pengine/test10/clone-require-all-2.summary +index f5861e7..2ebd230 100644 +--- a/pengine/test10/clone-require-all-2.summary ++++ b/pengine/test10/clone-require-all-2.summary +@@ -12,11 +12,11 @@ Online: [ rhel7-auto3 rhel7-auto4 ] + Stopped: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 rhel7-auto4 ] + + Transition Summary: +- * Move shooter (Started rhel7-auto1 -> rhel7-auto3) ++ * Move shooter ( rhel7-auto1 -> rhel7-auto3 ) + * Stop A:0 (rhel7-auto1) due to node availability + * Stop A:1 (rhel7-auto2) due to node availability +- * Start B:0 (rhel7-auto4 - blocked) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory +- * Start B:1 (rhel7-auto3 - blocked) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory ++ * Start B:0 ( rhel7-auto4 ) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory (blocked) ++ * Start B:1 ( rhel7-auto3 ) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory (blocked) + + Executing cluster transition: + * Resource action: shooter stop on rhel7-auto1 +diff --git a/pengine/test10/clone-require-all-3.summary b/pengine/test10/clone-require-all-3.summary +index 1c887e5..78076e5 100644 +--- a/pengine/test10/clone-require-all-3.summary ++++ b/pengine/test10/clone-require-all-3.summary +@@ -13,11 +13,11 @@ Online: [ rhel7-auto3 rhel7-auto4 ] + Stopped: [ rhel7-auto1 rhel7-auto2 ] + + Transition Summary: +- * Move shooter (Started rhel7-auto1 -> rhel7-auto3) ++ * Move shooter ( rhel7-auto1 -> rhel7-auto3 ) + * Stop A:0 (rhel7-auto1) due to node availability + * Stop A:1 (rhel7-auto2) due to node availability +- * Stop B:0 (Started rhel7-auto3) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory +- * Stop B:1 (Started rhel7-auto4) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory ++ * Stop B:0 ( rhel7-auto3 ) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory ++ * Stop B:1 ( rhel7-auto4 ) due to unrunnable clone-one-or-more:order-A-clone-B-clone-mandatory + + Executing cluster transition: + * Resource action: shooter stop on rhel7-auto1 +diff --git a/pengine/test10/clone-require-all-4.summary b/pengine/test10/clone-require-all-4.summary +index 90d2059..5b50e04 100644 +--- a/pengine/test10/clone-require-all-4.summary ++++ b/pengine/test10/clone-require-all-4.summary +@@ -12,7 +12,7 @@ Online: [ rhel7-auto2 rhel7-auto3 rhel7-auto4 ] + Stopped: [ rhel7-auto1 rhel7-auto2 ] + + Transition Summary: +- * Move shooter (Started rhel7-auto1 -> rhel7-auto2) ++ * Move shooter ( rhel7-auto1 -> rhel7-auto2 ) + * Stop A:0 (rhel7-auto1) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/clone-require-all-no-interleave-3.summary b/pengine/test10/clone-require-all-no-interleave-3.summary +index 4379644..d45d078 100644 +--- a/pengine/test10/clone-require-all-no-interleave-3.summary ++++ b/pengine/test10/clone-require-all-no-interleave-3.summary +@@ -15,9 +15,9 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ] + Stopped: [ rhel7-auto3 ] + + Transition Summary: +- * Move A:0 (Started rhel7-auto4 -> rhel7-auto3) +- * Move B:0 (Started rhel7-auto4 -> rhel7-auto3) +- * Move C:0 (Started rhel7-auto4 -> rhel7-auto3) ++ * Move A:0 ( rhel7-auto4 -> rhel7-auto3 ) ++ * Move B:0 ( rhel7-auto4 -> rhel7-auto3 ) ++ * Move C:0 ( rhel7-auto4 -> rhel7-auto3 ) + + Executing cluster transition: + * Pseudo action: C-clone_stop_0 +diff --git a/pengine/test10/clone_min_interleave_start_one.summary b/pengine/test10/clone_min_interleave_start_one.summary +index 4ee71c4..f2e0c0c 100644 +--- a/pengine/test10/clone_min_interleave_start_one.summary ++++ b/pengine/test10/clone_min_interleave_start_one.summary +@@ -12,12 +12,12 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + + Transition Summary: + * Start FAKE1:0 (c7auto1) +- * Start FAKE2:0 (c7auto2 - blocked) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory +- * Start FAKE2:1 (c7auto3 - blocked) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory +- * Start FAKE2:2 (c7auto1 - blocked) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory +- * Start FAKE3:0 (c7auto2 - blocked) due to unrunnable FAKE2:0 start +- * Start FAKE3:1 (c7auto3 - blocked) due to unrunnable FAKE2:1 start +- * Start FAKE3:2 (c7auto1 - blocked) due to unrunnable FAKE2:2 start ++ * Start FAKE2:0 ( c7auto2 ) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory (blocked) ++ * Start FAKE2:1 ( c7auto3 ) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory (blocked) ++ * Start FAKE2:2 ( c7auto1 ) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory (blocked) ++ * Start FAKE3:0 ( c7auto2 ) due to unrunnable FAKE2:0 start (blocked) ++ * Start FAKE3:1 ( c7auto3 ) due to unrunnable FAKE2:1 start (blocked) ++ * Start FAKE3:2 ( c7auto1 ) due to unrunnable FAKE2:2 start (blocked) + + Executing cluster transition: + * Pseudo action: FAKE1-clone_start_0 +diff --git a/pengine/test10/clone_min_interleave_stop_two.summary b/pengine/test10/clone_min_interleave_stop_two.summary +index 0866f3c..ccdb5e9 100644 +--- a/pengine/test10/clone_min_interleave_stop_two.summary ++++ b/pengine/test10/clone_min_interleave_stop_two.summary +@@ -13,12 +13,12 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + Transition Summary: + * Stop FAKE1:0 (c7auto3) due to node availability + * Stop FAKE1:2 (c7auto2) due to node availability +- * Stop FAKE2:0 (Started c7auto3) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory +- * Stop FAKE2:1 (Started c7auto1) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory +- * Stop FAKE2:2 (Started c7auto2) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory +- * Stop FAKE3:0 (Started c7auto3) due to required FAKE2:0 start +- * Stop FAKE3:1 (Started c7auto1) due to required FAKE2:1 start +- * Stop FAKE3:2 (Started c7auto2) due to required FAKE2:2 start ++ * Stop FAKE2:0 ( c7auto3 ) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory ++ * Stop FAKE2:1 ( c7auto1 ) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory ++ * Stop FAKE2:2 ( c7auto2 ) due to unrunnable clone-one-or-more:order-FAKE1-clone-FAKE2-clone-mandatory ++ * Stop FAKE3:0 ( c7auto3 ) due to required FAKE2:0 start ++ * Stop FAKE3:1 ( c7auto1 ) due to required FAKE2:1 start ++ * Stop FAKE3:2 ( c7auto2 ) due to required FAKE2:2 start + + Executing cluster transition: + * Pseudo action: FAKE3-clone_stop_0 +diff --git a/pengine/test10/clone_min_start_one.summary b/pengine/test10/clone_min_start_one.summary +index 196f1b3..504b81f 100644 +--- a/pengine/test10/clone_min_start_one.summary ++++ b/pengine/test10/clone_min_start_one.summary +@@ -10,9 +10,9 @@ Online: [ c7auto3 c7auto4 ] + FAKE (ocf::heartbeat:Dummy): Stopped + + Transition Summary: +- * Move shooter (Started c7auto1 -> c7auto3) ++ * Move shooter ( c7auto1 -> c7auto3 ) + * Start FAKECLONE:0 (c7auto3) +- * Start FAKE (c7auto4 - blocked) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory ++ * Start FAKE ( c7auto4 ) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory (blocked) + + Executing cluster transition: + * Resource action: shooter stop on c7auto1 +diff --git a/pengine/test10/clone_min_stop_all.summary b/pengine/test10/clone_min_stop_all.summary +index 877d12f..c12b0a3 100644 +--- a/pengine/test10/clone_min_stop_all.summary ++++ b/pengine/test10/clone_min_stop_all.summary +@@ -12,11 +12,11 @@ Online: [ c7auto4 ] + FAKE (ocf::heartbeat:Dummy): Started c7auto4 + + Transition Summary: +- * Move shooter (Started c7auto1 -> c7auto4) ++ * Move shooter ( c7auto1 -> c7auto4 ) + * Stop FAKECLONE:0 (c7auto1) due to node availability + * Stop FAKECLONE:1 (c7auto2) due to node availability + * Stop FAKECLONE:2 (c7auto3) due to node availability +- * Stop FAKE (Started c7auto4) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory ++ * Stop FAKE ( c7auto4 ) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory + + Executing cluster transition: + * Resource action: shooter stop on c7auto1 +diff --git a/pengine/test10/clone_min_stop_two.summary b/pengine/test10/clone_min_stop_two.summary +index 4d8c38f..5223501 100644 +--- a/pengine/test10/clone_min_stop_two.summary ++++ b/pengine/test10/clone_min_stop_two.summary +@@ -11,10 +11,10 @@ Online: [ c7auto3 c7auto4 ] + FAKE (ocf::heartbeat:Dummy): Started c7auto4 + + Transition Summary: +- * Move shooter (Started c7auto1 -> c7auto3) ++ * Move shooter ( c7auto1 -> c7auto3 ) + * Stop FAKECLONE:0 (c7auto1) due to node availability + * Stop FAKECLONE:1 (c7auto2) due to node availability +- * Stop FAKE (Started c7auto4) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory ++ * Stop FAKE ( c7auto4 ) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKE-mandatory + + Executing cluster transition: + * Resource action: shooter stop on c7auto1 +diff --git a/pengine/test10/cloned-group-stop.summary b/pengine/test10/cloned-group-stop.summary +index f7a980c..4d5f390 100644 +--- a/pengine/test10/cloned-group-stop.summary ++++ b/pengine/test10/cloned-group-stop.summary +@@ -20,20 +20,20 @@ Online: [ rhos4-node3 rhos4-node4 ] + Transition Summary: + * Stop qpidd:0 (rhos4-node4) due to node availability + * Stop qpidd:1 (rhos4-node3) due to node availability +- * Stop keystone:0 (Started rhos4-node4) due to unrunnable qpidd-clone running +- * Stop keystone:1 (Started rhos4-node3) due to unrunnable qpidd-clone running +- * Stop glance-fs:0 (Started rhos4-node4) due to required keystone-clone running +- * Stop glance-registry:0 (Started rhos4-node4) due to required glance-fs:0 start +- * Stop glance-api:0 (Started rhos4-node4) due to required glance-registry:0 start +- * Stop glance-fs:1 (Started rhos4-node3) due to required keystone-clone running +- * Stop glance-registry:1 (Started rhos4-node3) due to required glance-fs:1 start +- * Stop glance-api:1 (Started rhos4-node3) due to required glance-registry:1 start +- * Stop cinder-api:0 (Started rhos4-node4) due to required glance-clone running +- * Stop cinder-scheduler:0 (Started rhos4-node4) due to required cinder-api:0 start +- * Stop cinder-volume:0 (Started rhos4-node4) due to required cinder-scheduler:0 start +- * Stop cinder-api:1 (Started rhos4-node3) due to required glance-clone running +- * Stop cinder-scheduler:1 (Started rhos4-node3) due to required cinder-api:1 start +- * Stop cinder-volume:1 (Started rhos4-node3) due to required cinder-scheduler:1 start ++ * Stop keystone:0 ( rhos4-node4 ) due to unrunnable qpidd-clone running ++ * Stop keystone:1 ( rhos4-node3 ) due to unrunnable qpidd-clone running ++ * Stop glance-fs:0 ( rhos4-node4 ) due to required keystone-clone running ++ * Stop glance-registry:0 ( rhos4-node4 ) due to required glance-fs:0 start ++ * Stop glance-api:0 ( rhos4-node4 ) due to required glance-registry:0 start ++ * Stop glance-fs:1 ( rhos4-node3 ) due to required keystone-clone running ++ * Stop glance-registry:1 ( rhos4-node3 ) due to required glance-fs:1 start ++ * Stop glance-api:1 ( rhos4-node3 ) due to required glance-registry:1 start ++ * Stop cinder-api:0 ( rhos4-node4 ) due to required glance-clone running ++ * Stop cinder-scheduler:0 ( rhos4-node4 ) due to required cinder-api:0 start ++ * Stop cinder-volume:0 ( rhos4-node4 ) due to required cinder-scheduler:0 start ++ * Stop cinder-api:1 ( rhos4-node3 ) due to required glance-clone running ++ * Stop cinder-scheduler:1 ( rhos4-node3 ) due to required cinder-api:1 start ++ * Stop cinder-volume:1 ( rhos4-node3 ) due to required cinder-scheduler:1 start + + Executing cluster transition: + * Pseudo action: cinder-clone_stop_0 +diff --git a/pengine/test10/cloned-group.summary b/pengine/test10/cloned-group.summary +index e1456b9..1c4f277 100644 +--- a/pengine/test10/cloned-group.summary ++++ b/pengine/test10/cloned-group.summary +@@ -11,8 +11,8 @@ OFFLINE: [ webcluster02 ] + Stopped: [ webcluster02 ] + + Transition Summary: +- * Restart apache2:0 (Started webcluster01) +- * Restart mysql-proxy:0 (Started webcluster01) due to required apache2:0 start ++ * Restart apache2:0 ( webcluster01 ) ++ * Restart mysql-proxy:0 ( webcluster01 ) due to required apache2:0 start + * Stop apache2:2 (webcluster01) due to node availability + * Stop mysql-proxy:2 (webcluster01) due to node availability + +diff --git a/pengine/test10/cloned_start_one.summary b/pengine/test10/cloned_start_one.summary +index 5dedc18..38bf2ae 100644 +--- a/pengine/test10/cloned_start_one.summary ++++ b/pengine/test10/cloned_start_one.summary +@@ -14,7 +14,7 @@ Online: [ c7auto1 c7auto4 ] + Transition Summary: + * Start FAKECLONE:0 (c7auto1) + * Stop FAKECLONE2:0 (c7auto3) due to node availability +- * Stop FAKECLONE2:1 (Started c7auto4) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory ++ * Stop FAKECLONE2:1 ( c7auto4 ) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory + + Executing cluster transition: + * Pseudo action: FAKECLONE-clone_start_0 +diff --git a/pengine/test10/cloned_stop_two.summary b/pengine/test10/cloned_stop_two.summary +index 8f1e039..d6a4f2c 100644 +--- a/pengine/test10/cloned_stop_two.summary ++++ b/pengine/test10/cloned_stop_two.summary +@@ -16,7 +16,7 @@ Transition Summary: + * Stop FAKECLONE:1 (c7auto2) due to node availability + * Stop FAKECLONE:2 (c7auto3) due to node availability + * Stop FAKECLONE2:0 (c7auto3) due to node availability +- * Stop FAKECLONE2:1 (Started c7auto4) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory ++ * Stop FAKECLONE2:1 ( c7auto4 ) due to unrunnable clone-one-or-more:order-FAKECLONE-clone-FAKECLONE2-clone-mandatory + + Executing cluster transition: + * Pseudo action: FAKECLONE2-clone_stop_0 +diff --git a/pengine/test10/colo_master_w_native.summary b/pengine/test10/colo_master_w_native.summary +index 98774c1..fda8e85 100644 +--- a/pengine/test10/colo_master_w_native.summary ++++ b/pengine/test10/colo_master_w_native.summary +@@ -8,7 +8,7 @@ Online: [ node1 node2 ] + Slaves: [ node1 ] + + Transition Summary: +- * Demote MS_RSC_NATIVE:0 (Master -> Slave node2) ++ * Demote MS_RSC_NATIVE:0 ( Master -> Slave node2 ) + * Promote MS_RSC_NATIVE:1 (Slave -> Master node1) + + Executing cluster transition: +diff --git a/pengine/test10/colo_slave_w_native.summary b/pengine/test10/colo_slave_w_native.summary +index 705935e..f59d93b 100644 +--- a/pengine/test10/colo_slave_w_native.summary ++++ b/pengine/test10/colo_slave_w_native.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + Slaves: [ node1 ] + + Transition Summary: +- * Move A (Started node1 -> node2) +- * Demote MS_RSC_NATIVE:0 (Master -> Slave node2) ++ * Move A ( node1 -> node2 ) ++ * Demote MS_RSC_NATIVE:0 ( Master -> Slave node2 ) + * Promote MS_RSC_NATIVE:1 (Slave -> Master node1) + + Executing cluster transition: +diff --git a/pengine/test10/coloc-clone-stays-active.summary b/pengine/test10/coloc-clone-stays-active.summary +index b017137..df9b92c 100644 +--- a/pengine/test10/coloc-clone-stays-active.summary ++++ b/pengine/test10/coloc-clone-stays-active.summary +@@ -98,7 +98,7 @@ Online: [ s01-0 s01-1 ] + Started: [ s01-0 s01-1 ] + + Transition Summary: +- * Migrate mgmt-vm (Started s01-0 -> s01-1) ++ * Migrate mgmt-vm ( s01-0 -> s01-1 ) + + Executing cluster transition: + * Resource action: mgmt-vm migrate_to on s01-0 +diff --git a/pengine/test10/coloc-intra-set.summary b/pengine/test10/coloc-intra-set.summary +index caefd3e..e313d5a 100644 +--- a/pengine/test10/coloc-intra-set.summary ++++ b/pengine/test10/coloc-intra-set.summary +@@ -9,8 +9,8 @@ Online: [ hex-13 hex-14 ] + dummy3 (ocf::heartbeat:Dummy): Started hex-13 + + Transition Summary: +- * Move dummy1 (Started hex-13 -> hex-14) +- * Move dummy3 (Started hex-13 -> hex-14) ++ * Move dummy1 ( hex-13 -> hex-14 ) ++ * Move dummy3 ( hex-13 -> hex-14 ) + + Executing cluster transition: + * Resource action: dummy1 stop on hex-13 +diff --git a/pengine/test10/coloc_fp_logic.summary b/pengine/test10/coloc_fp_logic.summary +index bf479d8..0c2f4b9 100644 +--- a/pengine/test10/coloc_fp_logic.summary ++++ b/pengine/test10/coloc_fp_logic.summary +@@ -6,7 +6,7 @@ Online: [ node1 node2 ] + B (ocf::pacemaker:Dummy): Started node2 + + Transition Summary: +- * Move A (Started node1 -> node2) ++ * Move A ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: A stop on node1 +diff --git a/pengine/test10/colocation_constraint_stops_master.summary b/pengine/test10/colocation_constraint_stops_master.summary +index e0b6976..e4b8697 100644 +--- a/pengine/test10/colocation_constraint_stops_master.summary ++++ b/pengine/test10/colocation_constraint_stops_master.summary +@@ -6,7 +6,7 @@ Online: [ fc16-builder fc16-builder2 ] + Masters: [ fc16-builder ] + + Transition Summary: +- * Demote NATIVE_RSC_A:0 (Master -> Stopped fc16-builder) ++ * Stop NATIVE_RSC_A:0 ( Master fc16-builder ) due to node availability + + Executing cluster transition: + * Pseudo action: MASTER_RSC_A_pre_notify_demote_0 +diff --git a/pengine/test10/colocation_constraint_stops_slave.summary b/pengine/test10/colocation_constraint_stops_slave.summary +index a97b74b..4af4415 100644 +--- a/pengine/test10/colocation_constraint_stops_slave.summary ++++ b/pengine/test10/colocation_constraint_stops_slave.summary +@@ -9,7 +9,7 @@ OFFLINE: [ fc16-builder2 ] + NATIVE_RSC_B (ocf::pacemaker:Dummy): Started fc16-builder ( disabled ) + + Transition Summary: +- * Stop NATIVE_RSC_A:0 (fc16-builder) due to node availability ++ * Stop NATIVE_RSC_A:0 ( Slave fc16-builder ) due to node availability + * Stop NATIVE_RSC_B (fc16-builder) + + Executing cluster transition: +diff --git a/pengine/test10/complex_enforce_colo.summary b/pengine/test10/complex_enforce_colo.summary +index a21d5c1..2332269 100644 +--- a/pengine/test10/complex_enforce_colo.summary ++++ b/pengine/test10/complex_enforce_colo.summary +@@ -111,9 +111,9 @@ Transition Summary: + * Stop glance-api:0 (rhos6-node1) + * Stop glance-api:1 (rhos6-node2) + * Stop glance-api:2 (rhos6-node3) +- * Stop cinder-api (Started rhos6-node1) due to unrunnable keystone-clone running +- * Stop cinder-scheduler (Started rhos6-node1) due to required cinder-api start +- * Stop cinder-volume (Started rhos6-node1) ++ * Stop cinder-api ( rhos6-node1 ) due to unrunnable keystone-clone running ++ * Stop cinder-scheduler ( rhos6-node1 ) due to required cinder-api start ++ * Stop cinder-volume ( rhos6-node1 ) + * Stop swift-account:0 (rhos6-node1) + * Stop swift-account:1 (rhos6-node2) + * Stop swift-account:2 (rhos6-node3) +@@ -126,7 +126,7 @@ Transition Summary: + * Stop swift-proxy:0 (rhos6-node1) + * Stop swift-proxy:1 (rhos6-node2) + * Stop swift-proxy:2 (rhos6-node3) +- * Stop swift-object-expirer (Started rhos6-node2) due to required swift-proxy-clone running ++ * Stop swift-object-expirer ( rhos6-node2 ) due to required swift-proxy-clone running + * Stop neutron-server:0 (rhos6-node1) + * Stop neutron-server:1 (rhos6-node2) + * Stop neutron-server:2 (rhos6-node3) +@@ -166,35 +166,35 @@ Transition Summary: + * Stop nova-conductor:0 (rhos6-node1) + * Stop nova-conductor:1 (rhos6-node2) + * Stop nova-conductor:2 (rhos6-node3) +- * Stop ceilometer-central (Started rhos6-node3) due to unrunnable keystone-clone running +- * Stop ceilometer-collector:0 (Started rhos6-node1) due to required ceilometer-central start +- * Stop ceilometer-collector:1 (Started rhos6-node2) due to required ceilometer-central start +- * Stop ceilometer-collector:2 (Started rhos6-node3) due to required ceilometer-central start +- * Stop ceilometer-api:0 (Started rhos6-node1) due to required ceilometer-collector:0 start +- * Stop ceilometer-api:1 (Started rhos6-node2) due to required ceilometer-collector:1 start +- * Stop ceilometer-api:2 (Started rhos6-node3) due to required ceilometer-collector:2 start +- * Stop ceilometer-delay:0 (Started rhos6-node1) due to required ceilometer-api:0 start +- * Stop ceilometer-delay:1 (Started rhos6-node2) due to required ceilometer-api:1 start +- * Stop ceilometer-delay:2 (Started rhos6-node3) due to required ceilometer-api:2 start +- * Stop ceilometer-alarm-evaluator:0 (Started rhos6-node1) due to required ceilometer-delay:0 start +- * Stop ceilometer-alarm-evaluator:1 (Started rhos6-node2) due to required ceilometer-delay:1 start +- * Stop ceilometer-alarm-evaluator:2 (Started rhos6-node3) due to required ceilometer-delay:2 start +- * Stop ceilometer-alarm-notifier:0 (Started rhos6-node1) due to required ceilometer-alarm-evaluator:0 start +- * Stop ceilometer-alarm-notifier:1 (Started rhos6-node2) due to required ceilometer-alarm-evaluator:1 start +- * Stop ceilometer-alarm-notifier:2 (Started rhos6-node3) due to required ceilometer-alarm-evaluator:2 start +- * Stop ceilometer-notification:0 (Started rhos6-node1) due to required ceilometer-alarm-notifier:0 start +- * Stop ceilometer-notification:1 (Started rhos6-node2) due to required ceilometer-alarm-notifier:1 start +- * Stop ceilometer-notification:2 (Started rhos6-node3) due to required ceilometer-alarm-notifier:2 start +- * Stop heat-api:0 (Started rhos6-node1) due to required ceilometer-notification:0 start +- * Stop heat-api:1 (Started rhos6-node2) due to required ceilometer-notification:1 start +- * Stop heat-api:2 (Started rhos6-node3) due to required ceilometer-notification:2 start +- * Stop heat-api-cfn:0 (Started rhos6-node1) due to required heat-api:0 start +- * Stop heat-api-cfn:1 (Started rhos6-node2) due to required heat-api:1 start +- * Stop heat-api-cfn:2 (Started rhos6-node3) due to required heat-api:2 start +- * Stop heat-api-cloudwatch:0 (Started rhos6-node1) due to required heat-api-cfn:0 start +- * Stop heat-api-cloudwatch:1 (Started rhos6-node2) due to required heat-api-cfn:1 start +- * Stop heat-api-cloudwatch:2 (Started rhos6-node3) due to required heat-api-cfn:2 start +- * Stop heat-engine (Started rhos6-node2) due to required heat-api-cloudwatch-clone running ++ * Stop ceilometer-central ( rhos6-node3 ) due to unrunnable keystone-clone running ++ * Stop ceilometer-collector:0 ( rhos6-node1 ) due to required ceilometer-central start ++ * Stop ceilometer-collector:1 ( rhos6-node2 ) due to required ceilometer-central start ++ * Stop ceilometer-collector:2 ( rhos6-node3 ) due to required ceilometer-central start ++ * Stop ceilometer-api:0 ( rhos6-node1 ) due to required ceilometer-collector:0 start ++ * Stop ceilometer-api:1 ( rhos6-node2 ) due to required ceilometer-collector:1 start ++ * Stop ceilometer-api:2 ( rhos6-node3 ) due to required ceilometer-collector:2 start ++ * Stop ceilometer-delay:0 ( rhos6-node1 ) due to required ceilometer-api:0 start ++ * Stop ceilometer-delay:1 ( rhos6-node2 ) due to required ceilometer-api:1 start ++ * Stop ceilometer-delay:2 ( rhos6-node3 ) due to required ceilometer-api:2 start ++ * Stop ceilometer-alarm-evaluator:0 ( rhos6-node1 ) due to required ceilometer-delay:0 start ++ * Stop ceilometer-alarm-evaluator:1 ( rhos6-node2 ) due to required ceilometer-delay:1 start ++ * Stop ceilometer-alarm-evaluator:2 ( rhos6-node3 ) due to required ceilometer-delay:2 start ++ * Stop ceilometer-alarm-notifier:0 ( rhos6-node1 ) due to required ceilometer-alarm-evaluator:0 start ++ * Stop ceilometer-alarm-notifier:1 ( rhos6-node2 ) due to required ceilometer-alarm-evaluator:1 start ++ * Stop ceilometer-alarm-notifier:2 ( rhos6-node3 ) due to required ceilometer-alarm-evaluator:2 start ++ * Stop ceilometer-notification:0 ( rhos6-node1 ) due to required ceilometer-alarm-notifier:0 start ++ * Stop ceilometer-notification:1 ( rhos6-node2 ) due to required ceilometer-alarm-notifier:1 start ++ * Stop ceilometer-notification:2 ( rhos6-node3 ) due to required ceilometer-alarm-notifier:2 start ++ * Stop heat-api:0 ( rhos6-node1 ) due to required ceilometer-notification:0 start ++ * Stop heat-api:1 ( rhos6-node2 ) due to required ceilometer-notification:1 start ++ * Stop heat-api:2 ( rhos6-node3 ) due to required ceilometer-notification:2 start ++ * Stop heat-api-cfn:0 ( rhos6-node1 ) due to required heat-api:0 start ++ * Stop heat-api-cfn:1 ( rhos6-node2 ) due to required heat-api:1 start ++ * Stop heat-api-cfn:2 ( rhos6-node3 ) due to required heat-api:2 start ++ * Stop heat-api-cloudwatch:0 ( rhos6-node1 ) due to required heat-api-cfn:0 start ++ * Stop heat-api-cloudwatch:1 ( rhos6-node2 ) due to required heat-api-cfn:1 start ++ * Stop heat-api-cloudwatch:2 ( rhos6-node3 ) due to required heat-api-cfn:2 start ++ * Stop heat-engine ( rhos6-node2 ) due to required heat-api-cloudwatch-clone running + + Executing cluster transition: + * Pseudo action: glance-api-clone_stop_0 +diff --git a/pengine/test10/container-2.summary b/pengine/test10/container-2.summary +index f011cd3..011dbe4 100644 +--- a/pengine/test10/container-2.summary ++++ b/pengine/test10/container-2.summary +@@ -7,9 +7,9 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Restart container1 (Started node1) +- * Recover rsc1 (Started node1) +- * Restart rsc2 (Started node1) due to required container1 start ++ * Restart container1 ( node1 ) ++ * Recover rsc1 ( node1 ) ++ * Restart rsc2 ( node1 ) due to required container1 start + + Executing cluster transition: + * Resource action: rsc1 stop on node1 +diff --git a/pengine/test10/container-3.summary b/pengine/test10/container-3.summary +index f853ab2..424f27e 100644 +--- a/pengine/test10/container-3.summary ++++ b/pengine/test10/container-3.summary +@@ -7,9 +7,9 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Restart container1 (Started node1) ++ * Restart container1 ( node1 ) + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) due to required container1 start ++ * Restart rsc2 ( node1 ) due to required container1 start + + Executing cluster transition: + * Resource action: rsc2 stop on node1 +diff --git a/pengine/test10/container-4.summary b/pengine/test10/container-4.summary +index c5852de..387db8f 100644 +--- a/pengine/test10/container-4.summary ++++ b/pengine/test10/container-4.summary +@@ -7,9 +7,9 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Move container1 (Started node1 -> node2) +- * Recover rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) ++ * Move container1 ( node1 -> node2 ) ++ * Recover rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc1 stop on node1 +diff --git a/pengine/test10/container-group-2.summary b/pengine/test10/container-group-2.summary +index c0dbbf8..68d62aa 100644 +--- a/pengine/test10/container-group-2.summary ++++ b/pengine/test10/container-group-2.summary +@@ -8,9 +8,9 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Restart container1 (Started node1) +- * Recover rsc1 (Started node1) +- * Restart rsc2 (Started node1) due to required rsc1 start ++ * Restart container1 ( node1 ) ++ * Recover rsc1 ( node1 ) ++ * Restart rsc2 ( node1 ) due to required rsc1 start + + Executing cluster transition: + * Pseudo action: container-group_stop_0 +diff --git a/pengine/test10/container-group-3.summary b/pengine/test10/container-group-3.summary +index d5fac46..3e327ff 100644 +--- a/pengine/test10/container-group-3.summary ++++ b/pengine/test10/container-group-3.summary +@@ -8,7 +8,7 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Stopped + + Transition Summary: +- * Restart container1 (Started node1) ++ * Restart container1 ( node1 ) + * Start rsc1 (node1) + * Start rsc2 (node1) + +diff --git a/pengine/test10/container-group-4.summary b/pengine/test10/container-group-4.summary +index 3ee85bc..8ab682b 100644 +--- a/pengine/test10/container-group-4.summary ++++ b/pengine/test10/container-group-4.summary +@@ -8,9 +8,9 @@ Online: [ node1 node2 ] + rsc2 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Move container1 (Started node1 -> node2) +- * Recover rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) ++ * Move container1 ( node1 -> node2 ) ++ * Recover rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) + + Executing cluster transition: + * Pseudo action: container-group_stop_0 +diff --git a/pengine/test10/enforce-colo1.summary b/pengine/test10/enforce-colo1.summary +index b79b8cb..64815ae 100644 +--- a/pengine/test10/enforce-colo1.summary ++++ b/pengine/test10/enforce-colo1.summary +@@ -10,11 +10,11 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ] + central (ocf::heartbeat:Dummy): Started rhel7-auto3 + + Transition Summary: +- * Stop engine (Started rhel7-auto3) ++ * Stop engine ( rhel7-auto3 ) + * Stop keystone:0 (rhel7-auto2) due to node availability + * Stop keystone:1 (rhel7-auto3) due to node availability + * Stop keystone:2 (rhel7-auto1) due to node availability +- * Stop central (Started rhel7-auto3) due to unrunnable keystone-clone running ++ * Stop central ( rhel7-auto3 ) due to unrunnable keystone-clone running + + Executing cluster transition: + * Resource action: engine stop on rhel7-auto3 +diff --git a/pengine/test10/group-dependents.summary b/pengine/test10/group-dependents.summary +index c136656..15b750b 100644 +--- a/pengine/test10/group-dependents.summary ++++ b/pengine/test10/group-dependents.summary +@@ -30,25 +30,25 @@ Online: [ asttest1 asttest2 ] + Slaves: [ asttest2 ] + + Transition Summary: +- * Migrate mysqld (Started asttest1 -> asttest2) +- * Migrate dahdi (Started asttest1 -> asttest2) ++ * Migrate mysqld ( asttest1 -> asttest2 ) ++ * Migrate dahdi ( asttest1 -> asttest2 ) + * Start fonulator (asttest2) + * Start asterisk (asttest2) + * Start iax2_mon (asttest2) + * Start httpd (asttest2) + * Start tftp (asttest2) +- * Migrate ip_voip_route_test1 (Started asttest1 -> asttest2) +- * Migrate ip_voip_route_test2 (Started asttest1 -> asttest2) +- * Migrate ip_voip_vlan850 (Started asttest1 -> asttest2) +- * Migrate ip_voip_vlan998 (Started asttest1 -> asttest2) +- * Migrate ip_voip_vlan851 (Started asttest1 -> asttest2) +- * Migrate ip_voip_vlan852 (Started asttest1 -> asttest2) +- * Migrate ip_voip_vlan853 (Started asttest1 -> asttest2) +- * Migrate ip_voip_vlan854 (Started asttest1 -> asttest2) +- * Migrate ip_voip_vlan855 (Started asttest1 -> asttest2) +- * Migrate ip_voip_vlan856 (Started asttest1 -> asttest2) +- * Move fs_drbd (Started asttest1 -> asttest2) +- * Demote drbd:0 (Master -> Slave asttest1) ++ * Migrate ip_voip_route_test1 ( asttest1 -> asttest2 ) ++ * Migrate ip_voip_route_test2 ( asttest1 -> asttest2 ) ++ * Migrate ip_voip_vlan850 ( asttest1 -> asttest2 ) ++ * Migrate ip_voip_vlan998 ( asttest1 -> asttest2 ) ++ * Migrate ip_voip_vlan851 ( asttest1 -> asttest2 ) ++ * Migrate ip_voip_vlan852 ( asttest1 -> asttest2 ) ++ * Migrate ip_voip_vlan853 ( asttest1 -> asttest2 ) ++ * Migrate ip_voip_vlan854 ( asttest1 -> asttest2 ) ++ * Migrate ip_voip_vlan855 ( asttest1 -> asttest2 ) ++ * Migrate ip_voip_vlan856 ( asttest1 -> asttest2 ) ++ * Move fs_drbd ( asttest1 -> asttest2 ) ++ * Demote drbd:0 ( Master -> Slave asttest1 ) + * Promote drbd:1 (Slave -> Master asttest2) + + Executing cluster transition: +diff --git a/pengine/test10/group-fail.summary b/pengine/test10/group-fail.summary +index 9067bf6..8ed59ca 100644 +--- a/pengine/test10/group-fail.summary ++++ b/pengine/test10/group-fail.summary +@@ -10,9 +10,9 @@ Online: [ node1 node2 ] + + Transition Summary: + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) due to required rsc1 start ++ * Restart rsc2 ( node1 ) due to required rsc1 start + * Start rsc3 (node1) +- * Restart rsc4 (Started node1) due to required rsc3 start ++ * Restart rsc4 ( node1 ) due to required rsc3 start + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/group10.summary b/pengine/test10/group10.summary +index 570fd28..f7d0eef 100644 +--- a/pengine/test10/group10.summary ++++ b/pengine/test10/group10.summary +@@ -18,9 +18,9 @@ Online: [ c001n01 c001n02 c001n03 c001n08 ] + child_DoFencing:3 (stonith:ssh): Started c001n08 + + Transition Summary: +- * Recover child_192.168.100.181 (Started c001n01) +- * Restart child_192.168.100.182 (Started c001n01) due to required child_192.168.100.181 start +- * Restart child_192.168.100.183 (Started c001n01) due to required child_192.168.100.182 start ++ * Recover child_192.168.100.181 ( c001n01 ) ++ * Restart child_192.168.100.182 ( c001n01 ) due to required child_192.168.100.181 start ++ * Restart child_192.168.100.183 ( c001n01 ) due to required child_192.168.100.182 start + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/group14.summary b/pengine/test10/group14.summary +index 69e688b..f7a5d7d 100644 +--- a/pengine/test10/group14.summary ++++ b/pengine/test10/group14.summary +@@ -33,24 +33,24 @@ OFFLINE: [ c001n02 c001n03 c001n04 c001n05 ] + ocf_msdummy:11 (ocf::heartbeat:Stateful): Stopped + + Transition Summary: +- * Start DcIPaddr (c001n06 - blocked) due to no quorum +- * Stop r192.168.100.181 (Started c001n06) due to no quorum +- * Start r192.168.100.182 (c001n07 - blocked) due to no quorum +- * Start r192.168.100.183 (c001n07 - blocked) due to no quorum +- * Start lsb_dummy (c001n06 - blocked) due to no quorum +- * Start migrator (c001n06 - blocked) due to no quorum +- * Start rsc_c001n03 (c001n06 - blocked) due to no quorum +- * Start rsc_c001n02 (c001n07 - blocked) due to no quorum +- * Start rsc_c001n04 (c001n06 - blocked) due to no quorum +- * Start rsc_c001n05 (c001n07 - blocked) due to no quorum +- * Start rsc_c001n06 (c001n06 - blocked) due to no quorum +- * Start rsc_c001n07 (c001n07 - blocked) due to no quorum ++ * Start DcIPaddr ( c001n06 ) due to no quorum (blocked) ++ * Stop r192.168.100.181 ( c001n06 ) due to no quorum ++ * Start r192.168.100.182 ( c001n07 ) due to no quorum (blocked) ++ * Start r192.168.100.183 ( c001n07 ) due to no quorum (blocked) ++ * Start lsb_dummy ( c001n06 ) due to no quorum (blocked) ++ * Start migrator ( c001n06 ) due to no quorum (blocked) ++ * Start rsc_c001n03 ( c001n06 ) due to no quorum (blocked) ++ * Start rsc_c001n02 ( c001n07 ) due to no quorum (blocked) ++ * Start rsc_c001n04 ( c001n06 ) due to no quorum (blocked) ++ * Start rsc_c001n05 ( c001n07 ) due to no quorum (blocked) ++ * Start rsc_c001n06 ( c001n06 ) due to no quorum (blocked) ++ * Start rsc_c001n07 ( c001n07 ) due to no quorum (blocked) + * Start child_DoFencing:0 (c001n06) + * Start child_DoFencing:1 (c001n07) +- * Start ocf_msdummy:0 (c001n06 - blocked) due to no quorum +- * Start ocf_msdummy:1 (c001n07 - blocked) due to no quorum +- * Start ocf_msdummy:2 (c001n06 - blocked) due to no quorum +- * Start ocf_msdummy:3 (c001n07 - blocked) due to no quorum ++ * Start ocf_msdummy:0 ( c001n06 ) due to no quorum (blocked) ++ * Start ocf_msdummy:1 ( c001n07 ) due to no quorum (blocked) ++ * Start ocf_msdummy:2 ( c001n06 ) due to no quorum (blocked) ++ * Start ocf_msdummy:3 ( c001n07 ) due to no quorum (blocked) + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/group5.summary b/pengine/test10/group5.summary +index 68a0e91..f709b9d 100644 +--- a/pengine/test10/group5.summary ++++ b/pengine/test10/group5.summary +@@ -10,11 +10,11 @@ Online: [ node1 node2 ] + rsc3 (heartbeat:apache): Started node1 + + Transition Summary: +- * Move rsc1 (Started node1 -> node2) +- * Move child_rsc1 (Started node1 -> node2) +- * Move child_rsc2 (Started node1 -> node2) +- * Move child_rsc3 (Started node1 -> node2) +- * Move rsc3 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move child_rsc1 ( node1 -> node2 ) ++ * Move child_rsc2 ( node1 -> node2 ) ++ * Move child_rsc3 ( node1 -> node2 ) ++ * Move rsc3 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/group6.summary b/pengine/test10/group6.summary +index f2fa1e2..0d18868 100644 +--- a/pengine/test10/group6.summary ++++ b/pengine/test10/group6.summary +@@ -12,12 +12,12 @@ Online: [ node1 node2 ] + child_rsc6 (heartbeat:apache): Started node1 + + Transition Summary: +- * Move child_rsc1 (Started node1 -> node2) +- * Move child_rsc2 (Started node1 -> node2) +- * Move child_rsc3 (Started node1 -> node2) +- * Move child_rsc4 (Started node1 -> node2) +- * Move child_rsc5 (Started node1 -> node2) +- * Move child_rsc6 (Started node1 -> node2) ++ * Move child_rsc1 ( node1 -> node2 ) ++ * Move child_rsc2 ( node1 -> node2 ) ++ * Move child_rsc3 ( node1 -> node2 ) ++ * Move child_rsc4 ( node1 -> node2 ) ++ * Move child_rsc5 ( node1 -> node2 ) ++ * Move child_rsc6 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: child_rsc1 monitor on node2 +diff --git a/pengine/test10/group9.summary b/pengine/test10/group9.summary +index f6755b9..ede3c82 100644 +--- a/pengine/test10/group9.summary ++++ b/pengine/test10/group9.summary +@@ -14,11 +14,11 @@ Online: [ node1 node2 ] + rsc8 (heartbeat:apache): Started node1 + + Transition Summary: +- * Recover rsc4 (Started node1) +- * Restart rsc5 (Started node1) due to required rsc4 start +- * Move rsc6 (Started node1 -> node2) +- * Recover rsc7 (Started node1 -> node2) +- * Move rsc8 (Started node1 -> node2) ++ * Recover rsc4 ( node1 ) ++ * Restart rsc5 ( node1 ) due to required rsc4 start ++ * Move rsc6 ( node1 -> node2 ) ++ * Recover rsc7 ( node1 -> node2 ) ++ * Move rsc8 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary +index d22fe3d..ea0fa4c 100644 +--- a/pengine/test10/guest-node-host-dies.summary ++++ b/pengine/test10/guest-node-host-dies.summary +@@ -14,14 +14,14 @@ Transition Summary: + * Fence (reboot) lxc2 (resource: container2) 'guest is unclean' + * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' + * Fence (reboot) rhel7-1 'rsc_rhel7-1 is thought to be active there' +- * Restart Fencing (Started rhel7-4) +- * Move rsc_rhel7-1 (Started rhel7-1 -> rhel7-5) +- * Recover container1 (Started rhel7-1 -> rhel7-2) +- * Recover container2 (Started rhel7-1 -> rhel7-3) ++ * Restart Fencing ( rhel7-4 ) ++ * Move rsc_rhel7-1 ( rhel7-1 -> rhel7-5 ) ++ * Recover container1 ( rhel7-1 -> rhel7-2 ) ++ * Recover container2 ( rhel7-1 -> rhel7-3 ) + * Recover lxc-ms:0 (Master lxc1) + * Recover lxc-ms:1 (Slave lxc2) +- * Move lxc1 (Started rhel7-1 -> rhel7-2) +- * Move lxc2 (Started rhel7-1 -> rhel7-3) ++ * Move lxc1 ( rhel7-1 -> rhel7-2 ) ++ * Move lxc2 ( rhel7-1 -> rhel7-3 ) + + Executing cluster transition: + * Resource action: Fencing stop on rhel7-4 +diff --git a/pengine/test10/inc11.summary b/pengine/test10/inc11.summary +index 6f4d8ef..08d024d 100644 +--- a/pengine/test10/inc11.summary ++++ b/pengine/test10/inc11.summary +@@ -10,8 +10,7 @@ Online: [ node0 node1 node2 ] + Transition Summary: + * Start simple-rsc (node2) + * Start child_rsc1:0 (node1) +- * Start child_rsc1:1 (node2) +- * Promote child_rsc1:1 (Stopped -> Master node2) ++ * Promote child_rsc1:1 ( Stopped -> Master node2 ) + + Executing cluster transition: + * Resource action: simple-rsc monitor on node2 +diff --git a/pengine/test10/inc12.summary b/pengine/test10/inc12.summary +index e950c67..cfe99c5 100644 +--- a/pengine/test10/inc12.summary ++++ b/pengine/test10/inc12.summary +@@ -53,16 +53,16 @@ Transition Summary: + * Stop child_DoFencing:2 (c001n05) due to node availability + * Stop child_DoFencing:3 (c001n06) due to node availability + * Stop child_DoFencing:4 (c001n07) due to node availability +- * Stop ocf_msdummy:10 (c001n02) due to node availability +- * Stop ocf_msdummy:11 (c001n02) due to node availability +- * Stop ocf_msdummy:2 (c001n04) due to node availability +- * Stop ocf_msdummy:3 (c001n04) due to node availability +- * Stop ocf_msdummy:4 (c001n05) due to node availability +- * Stop ocf_msdummy:5 (c001n05) due to node availability +- * Stop ocf_msdummy:6 (c001n06) due to node availability +- * Stop ocf_msdummy:7 (c001n06) due to node availability +- * Stop ocf_msdummy:8 (c001n07) due to node availability +- * Stop ocf_msdummy:9 (c001n07) due to node availability ++ * Stop ocf_msdummy:10 ( Slave c001n02 ) due to node availability ++ * Stop ocf_msdummy:11 ( Slave c001n02 ) due to node availability ++ * Stop ocf_msdummy:2 ( Slave c001n04 ) due to node availability ++ * Stop ocf_msdummy:3 ( Slave c001n04 ) due to node availability ++ * Stop ocf_msdummy:4 ( Slave c001n05 ) due to node availability ++ * Stop ocf_msdummy:5 ( Slave c001n05 ) due to node availability ++ * Stop ocf_msdummy:6 ( Slave c001n06 ) due to node availability ++ * Stop ocf_msdummy:7 ( Slave c001n06 ) due to node availability ++ * Stop ocf_msdummy:8 ( Slave c001n07 ) due to node availability ++ * Stop ocf_msdummy:9 ( Slave c001n07 ) due to node availability + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/inc2.summary b/pengine/test10/inc2.summary +index 898729f..b3049be 100644 +--- a/pengine/test10/inc2.summary ++++ b/pengine/test10/inc2.summary +@@ -10,8 +10,8 @@ Online: [ node1 node2 ] + child_rsc1:4 (heartbeat:apache): Started node1 + + Transition Summary: +- * Move child_rsc1:2 (Started node1 -> node2) +- * Move child_rsc1:3 (Started node1 -> node2) ++ * Move child_rsc1:2 ( node1 -> node2 ) ++ * Move child_rsc1:3 ( node1 -> node2 ) + * Stop child_rsc1:4 (node1) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/inc3.summary b/pengine/test10/inc3.summary +index 955ff3b..5ceab72 100644 +--- a/pengine/test10/inc3.summary ++++ b/pengine/test10/inc3.summary +@@ -16,11 +16,11 @@ Online: [ node1 node2 ] + child_rsc2:4 (heartbeat:apache): Started node2 + + Transition Summary: +- * Move child_rsc1:2 (Started node1 -> node2) +- * Move child_rsc1:3 (Started node1 -> node2) ++ * Move child_rsc1:2 ( node1 -> node2 ) ++ * Move child_rsc1:3 ( node1 -> node2 ) + * Stop child_rsc1:4 (node1) due to node availability +- * Move child_rsc2:3 (Started node2 -> node1) +- * Move child_rsc2:4 (Started node2 -> node1) ++ * Move child_rsc2:3 ( node2 -> node1 ) ++ * Move child_rsc2:4 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: child_rsc1:0 monitor on node2 +diff --git a/pengine/test10/inc4.summary b/pengine/test10/inc4.summary +index e730360..3b3f10b 100644 +--- a/pengine/test10/inc4.summary ++++ b/pengine/test10/inc4.summary +@@ -16,11 +16,11 @@ Online: [ node1 node2 ] + child_rsc2:4 (heartbeat:apache): Started node2 + + Transition Summary: +- * Move child_rsc1:2 (Started node1 -> node2) +- * Move child_rsc1:3 (Started node1 -> node2) ++ * Move child_rsc1:2 ( node1 -> node2 ) ++ * Move child_rsc1:3 ( node1 -> node2 ) + * Stop child_rsc1:4 (node1) due to node availability +- * Move child_rsc2:3 (Started node2 -> node1) +- * Move child_rsc2:4 (Started node2 -> node1) ++ * Move child_rsc2:3 ( node2 -> node1 ) ++ * Move child_rsc2:4 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: child_rsc1:0 monitor on node2 +diff --git a/pengine/test10/inc5.summary b/pengine/test10/inc5.summary +index d4e8863..93e0d8e 100644 +--- a/pengine/test10/inc5.summary ++++ b/pengine/test10/inc5.summary +@@ -36,10 +36,10 @@ Online: [ node1 node2 ] + child_rsc8:2 (heartbeat:apache): Stopped + + Transition Summary: +- * Move child_rsc2:1 (Started node1 -> node2) +- * Move child_rsc4:1 (Started node1 -> node2) +- * Move child_rsc5:1 (Started node2 -> node1) +- * Move child_rsc7:1 (Started node2 -> node1) ++ * Move child_rsc2:1 ( node1 -> node2 ) ++ * Move child_rsc4:1 ( node1 -> node2 ) ++ * Move child_rsc5:1 ( node2 -> node1 ) ++ * Move child_rsc7:1 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: child_rsc1:0 monitor on node2 +diff --git a/pengine/test10/inc6.summary b/pengine/test10/inc6.summary +index 4c754d7..99f3702 100644 +--- a/pengine/test10/inc6.summary ++++ b/pengine/test10/inc6.summary +@@ -28,12 +28,12 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Move child_rsc2:1 (Started node1 -> node2) +- * Move child_rsc4:1 (Started node1 -> node2) +- * Move child_rsc5:1 (Started node2 -> node1) +- * Restart child_rsc6:0 (Started node1) due to required rsc5 running +- * Restart child_rsc6:1 (Started node2) due to required rsc5 running +- * Move child_rsc7:1 (Started node2 -> node1) ++ * Move child_rsc2:1 ( node1 -> node2 ) ++ * Move child_rsc4:1 ( node1 -> node2 ) ++ * Move child_rsc5:1 ( node2 -> node1 ) ++ * Restart child_rsc6:0 ( node1 ) due to required rsc5 running ++ * Restart child_rsc6:1 ( node2 ) due to required rsc5 running ++ * Move child_rsc7:1 ( node2 -> node1 ) + + Executing cluster transition: + * Pseudo action: rsc2_stop_0 +diff --git a/pengine/test10/interleave-restart.summary b/pengine/test10/interleave-restart.summary +index 5ac19b9..c151581 100644 +--- a/pengine/test10/interleave-restart.summary ++++ b/pengine/test10/interleave-restart.summary +@@ -13,9 +13,9 @@ Online: [ node1 node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Recover evmsclone:1 (Started node1) +- * Restart imagestoreclone:1 (Started node1) due to required evmsclone:1 start +- * Restart configstoreclone:1 (Started node1) due to required evmsclone:1 start ++ * Recover evmsclone:1 ( node1 ) ++ * Restart imagestoreclone:1 ( node1 ) due to required evmsclone:1 start ++ * Restart configstoreclone:1 ( node1 ) due to required evmsclone:1 start + + Executing cluster transition: + * Pseudo action: evmscloneset_pre_notify_stop_0 +diff --git a/pengine/test10/isolation-restart-all.summary b/pengine/test10/isolation-restart-all.summary +index a2939f1..595c254 100644 +--- a/pengine/test10/isolation-restart-all.summary ++++ b/pengine/test10/isolation-restart-all.summary +@@ -22,17 +22,17 @@ OFFLINE: [ rhel7-auto4 ] + iso_mem2 (ocf::heartbeat:Dummy): Started rhel7-auto5 + + Transition Summary: +- * Restart fake (Started rhel7-auto5) +- * Restart replicated:0 (Started rhel7-auto5) +- * Restart replicated:1 (Started rhel7-auto5) +- * Restart s1 (Started rhel7-auto5) +- * Restart s2 (Started rhel7-auto5) +- * Restart g1:0 (Started rhel7-auto5) +- * Restart g2:0 (Started rhel7-auto5) +- * Restart g1:1 (Started rhel7-auto5) +- * Restart g2:1 (Started rhel7-auto5) +- * Restart iso_mem1 (Started rhel7-auto5) +- * Restart iso_mem2 (Started rhel7-auto5) ++ * Restart fake ( rhel7-auto5 ) ++ * Restart replicated:0 ( rhel7-auto5 ) ++ * Restart replicated:1 ( rhel7-auto5 ) ++ * Restart s1 ( rhel7-auto5 ) ++ * Restart s2 ( rhel7-auto5 ) ++ * Restart g1:0 ( rhel7-auto5 ) ++ * Restart g2:0 ( rhel7-auto5 ) ++ * Restart g1:1 ( rhel7-auto5 ) ++ * Restart g2:1 ( rhel7-auto5 ) ++ * Restart iso_mem1 ( rhel7-auto5 ) ++ * Restart iso_mem2 ( rhel7-auto5 ) + + Executing cluster transition: + * Resource action: fake stop on rhel7-auto5 +diff --git a/pengine/test10/load-stopped-loop-2.summary b/pengine/test10/load-stopped-loop-2.summary +index 5da41e4..d4fbf14 100644 +--- a/pengine/test10/load-stopped-loop-2.summary ++++ b/pengine/test10/load-stopped-loop-2.summary +@@ -26,14 +26,14 @@ Online: [ xfc0 xfc1 xfc2 xfc3 ] + xu-test9 (ocf::heartbeat:Xen): Started xfc0 + + Transition Summary: +- * Migrate xu-test12 (Started xfc2 -> xfc3) +- * Migrate xu-test2 (Started xfc3 -> xfc1) +- * Migrate xu-test3 (Started xfc1 -> xfc0) +- * Migrate xu-test4 (Started xfc0 -> xfc2) +- * Migrate xu-test5 (Started xfc2 -> xfc3) +- * Migrate xu-test6 (Started xfc3 -> xfc1) +- * Migrate xu-test7 (Started xfc1 -> xfc0) +- * Migrate xu-test9 (Started xfc0 -> xfc2) ++ * Migrate xu-test12 ( xfc2 -> xfc3 ) ++ * Migrate xu-test2 ( xfc3 -> xfc1 ) ++ * Migrate xu-test3 ( xfc1 -> xfc0 ) ++ * Migrate xu-test4 ( xfc0 -> xfc2 ) ++ * Migrate xu-test5 ( xfc2 -> xfc3 ) ++ * Migrate xu-test6 ( xfc3 -> xfc1 ) ++ * Migrate xu-test7 ( xfc1 -> xfc0 ) ++ * Migrate xu-test9 ( xfc0 -> xfc2 ) + * Start xu-test13 (xfc3) + + Executing cluster transition: +diff --git a/pengine/test10/load-stopped-loop.summary b/pengine/test10/load-stopped-loop.summary +index 2ff5bae..bc5a0f9 100644 +--- a/pengine/test10/load-stopped-loop.summary ++++ b/pengine/test10/load-stopped-loop.summary +@@ -143,16 +143,16 @@ Online: [ mgmt01 v03-a v03-b ] + Stopped: [ mgmt01 ] + + Transition Summary: +- * Reload vds-ok-pool-0-iscsi:0 (Started mgmt01) +- * Reload vds-ok-pool-0-iscsi:1 (Started v03-b) +- * Reload vds-ok-pool-0-iscsi:2 (Started v03-a) +- * Reload vds-ok-pool-1-iscsi:0 (Started mgmt01) +- * Reload vds-ok-pool-1-iscsi:1 (Started v03-b) +- * Reload vds-ok-pool-1-iscsi:2 (Started v03-a) +- * Restart stonith-v03-b (Started v03-a) +- * Restart stonith-v03-a (Started v03-b) +- * Migrate license.anbriz.vds-ok.com-vm (Started v03-b -> v03-a) +- * Migrate terminal0.anbriz.vds-ok.com-vm (Started v03-a -> v03-b) ++ * Reload vds-ok-pool-0-iscsi:0 ( mgmt01 ) ++ * Reload vds-ok-pool-0-iscsi:1 ( v03-b ) ++ * Reload vds-ok-pool-0-iscsi:2 ( v03-a ) ++ * Reload vds-ok-pool-1-iscsi:0 ( mgmt01 ) ++ * Reload vds-ok-pool-1-iscsi:1 ( v03-b ) ++ * Reload vds-ok-pool-1-iscsi:2 ( v03-a ) ++ * Restart stonith-v03-b ( v03-a ) ++ * Restart stonith-v03-a ( v03-b ) ++ * Migrate license.anbriz.vds-ok.com-vm ( v03-b -> v03-a ) ++ * Migrate terminal0.anbriz.vds-ok.com-vm ( v03-a -> v03-b ) + * Start vd01-d.cdev.ttc.prague.cz.vds-ok.com-vm (v03-a) + + Executing cluster transition: +diff --git a/pengine/test10/master-1.summary b/pengine/test10/master-1.summary +index a45943c..b0e5025 100644 +--- a/pengine/test10/master-1.summary ++++ b/pengine/test10/master-1.summary +@@ -11,7 +11,6 @@ Online: [ node1 node2 ] + + Transition Summary: + * Start child_rsc1:0 (node1) +- * Start child_rsc1:1 (node2) + * Promote child_rsc1:1 (Stopped -> Master node2) + * Start child_rsc1:2 (node1) + * Start child_rsc1:3 (node2) +diff --git a/pengine/test10/master-10.summary b/pengine/test10/master-10.summary +index cd0efc3..d736f7c 100644 +--- a/pengine/test10/master-10.summary ++++ b/pengine/test10/master-10.summary +@@ -10,7 +10,6 @@ Online: [ node1 node2 ] + child_rsc1:4 (ocf::heartbeat:apache): Stopped + + Transition Summary: +- * Start child_rsc1:0 (node1) + * Promote child_rsc1:0 (Stopped -> Master node1) + * Start child_rsc1:1 (node2) + * Start child_rsc1:2 (node1) +diff --git a/pengine/test10/master-11.summary b/pengine/test10/master-11.summary +index a13760f..b88e5be 100644 +--- a/pengine/test10/master-11.summary ++++ b/pengine/test10/master-11.summary +@@ -10,7 +10,6 @@ Online: [ node1 node2 ] + Transition Summary: + * Start simple-rsc (node2) + * Start child_rsc1:0 (node1) +- * Start child_rsc1:1 (node2) + * Promote child_rsc1:1 (Stopped -> Master node2) + + Executing cluster transition: +diff --git a/pengine/test10/master-2.summary b/pengine/test10/master-2.summary +index b8f5447..391bf83 100644 +--- a/pengine/test10/master-2.summary ++++ b/pengine/test10/master-2.summary +@@ -10,7 +10,6 @@ Online: [ node1 node2 ] + child_rsc1:4 (ocf::heartbeat:apache): Stopped + + Transition Summary: +- * Start child_rsc1:0 (node1) + * Promote child_rsc1:0 (Stopped -> Master node1) + * Start child_rsc1:1 (node2) + * Start child_rsc1:2 (node1) +diff --git a/pengine/test10/master-3.summary b/pengine/test10/master-3.summary +index a45943c..b0e5025 100644 +--- a/pengine/test10/master-3.summary ++++ b/pengine/test10/master-3.summary +@@ -11,7 +11,6 @@ Online: [ node1 node2 ] + + Transition Summary: + * Start child_rsc1:0 (node1) +- * Start child_rsc1:1 (node2) + * Promote child_rsc1:1 (Stopped -> Master node2) + * Start child_rsc1:2 (node1) + * Start child_rsc1:3 (node2) +diff --git a/pengine/test10/master-7.summary b/pengine/test10/master-7.summary +index 58ef275..105966f 100644 +--- a/pengine/test10/master-7.summary ++++ b/pengine/test10/master-7.summary +@@ -30,15 +30,15 @@ Online: [ c001n02 c001n03 c001n08 ] + + Transition Summary: + * Fence (reboot) c001n01 'peer is no longer part of the cluster' +- * Move DcIPaddr (Started c001n01 -> c001n03) +- * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) +- * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) +- * Move ocf_192.168.100.183 (Started c001n03 -> c001n02) +- * Move lsb_dummy (Started c001n02 -> c001n08) +- * Move rsc_c001n01 (Started c001n01 -> c001n03) ++ * Move DcIPaddr ( c001n01 -> c001n03 ) ++ * Move ocf_192.168.100.181 ( c001n03 -> c001n02 ) ++ * Move heartbeat_192.168.100.182 ( c001n03 -> c001n02 ) ++ * Move ocf_192.168.100.183 ( c001n03 -> c001n02 ) ++ * Move lsb_dummy ( c001n02 -> c001n08 ) ++ * Move rsc_c001n01 ( c001n01 -> c001n03 ) + * Stop child_DoFencing:0 (c001n01) due to node availability +- * Demote ocf_msdummy:0 (Master -> Stopped c001n01) +- * Stop ocf_msdummy:4 (c001n01) due to node availability ++ * Stop ocf_msdummy:0 ( Master c001n01 ) due to node availability ++ * Stop ocf_msdummy:4 ( Slave c001n01 ) due to node availability + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/master-8.summary b/pengine/test10/master-8.summary +index c18e884..5792e16 100644 +--- a/pengine/test10/master-8.summary ++++ b/pengine/test10/master-8.summary +@@ -30,15 +30,14 @@ Online: [ c001n02 c001n03 c001n08 ] + + Transition Summary: + * Fence (reboot) c001n01 'peer is no longer part of the cluster' +- * Move DcIPaddr (Started c001n01 -> c001n03) +- * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) +- * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) +- * Move ocf_192.168.100.183 (Started c001n03 -> c001n02) +- * Move lsb_dummy (Started c001n02 -> c001n08) +- * Move rsc_c001n01 (Started c001n01 -> c001n03) ++ * Move DcIPaddr ( c001n01 -> c001n03 ) ++ * Move ocf_192.168.100.181 ( c001n03 -> c001n02 ) ++ * Move heartbeat_192.168.100.182 ( c001n03 -> c001n02 ) ++ * Move ocf_192.168.100.183 ( c001n03 -> c001n02 ) ++ * Move lsb_dummy ( c001n02 -> c001n08 ) ++ * Move rsc_c001n01 ( c001n01 -> c001n03 ) + * Stop child_DoFencing:0 (c001n01) due to node availability +- * Demote ocf_msdummy:0 (Master -> Slave c001n01 - blocked) +- * Move ocf_msdummy:0 (Slave c001n01 -> c001n03) ++ * Move ocf_msdummy:0 ( Master c001n01 -> Slave c001n03 ) + + Executing cluster transition: + * Pseudo action: group-1_stop_0 +diff --git a/pengine/test10/master-9.summary b/pengine/test10/master-9.summary +index ef59602..55ccf50 100644 +--- a/pengine/test10/master-9.summary ++++ b/pengine/test10/master-9.summary +@@ -31,18 +31,18 @@ Online: [ ibm1 va1 ] + + Transition Summary: + * Shutdown ibm1 +- * Start DcIPaddr (va1 - blocked) due to no quorum +- * Start ocf_127.0.0.11 (va1 - blocked) due to no quorum +- * Start heartbeat_127.0.0.12 (va1 - blocked) due to no quorum +- * Start ocf_127.0.0.13 (va1 - blocked) due to no quorum +- * Start lsb_dummy (va1 - blocked) due to no quorum +- * Start rsc_sgi2 (va1 - blocked) due to no quorum +- * Start rsc_ibm1 (va1 - blocked) due to no quorum +- * Start rsc_va1 (va1 - blocked) due to no quorum +- * Start rsc_test02 (va1 - blocked) due to no quorum ++ * Start DcIPaddr ( va1 ) due to no quorum (blocked) ++ * Start ocf_127.0.0.11 ( va1 ) due to no quorum (blocked) ++ * Start heartbeat_127.0.0.12 ( va1 ) due to no quorum (blocked) ++ * Start ocf_127.0.0.13 ( va1 ) due to no quorum (blocked) ++ * Start lsb_dummy ( va1 ) due to no quorum (blocked) ++ * Start rsc_sgi2 ( va1 ) due to no quorum (blocked) ++ * Start rsc_ibm1 ( va1 ) due to no quorum (blocked) ++ * Start rsc_va1 ( va1 ) due to no quorum (blocked) ++ * Start rsc_test02 ( va1 ) due to no quorum (blocked) + * Stop child_DoFencing:1 (ibm1) due to node availability +- * Start ocf_msdummy:0 (va1 - blocked) due to no quorum +- * Start ocf_msdummy:1 (va1 - blocked) due to no quorum ++ * Promote ocf_msdummy:0 ( Stopped -> Master va1 ) due to node availability (blocked) ++ * Start ocf_msdummy:1 ( va1 ) due to no quorum (blocked) + + Executing cluster transition: + * Resource action: child_DoFencing:1 monitor on va1 +diff --git a/pengine/test10/master-asymmetrical-order.summary b/pengine/test10/master-asymmetrical-order.summary +index d09f62e..50f717e 100644 +--- a/pengine/test10/master-asymmetrical-order.summary ++++ b/pengine/test10/master-asymmetrical-order.summary +@@ -11,8 +11,8 @@ Online: [ node1 node2 ] + Slaves: [ node1 ] + + Transition Summary: +- * Demote rsc1:0 (Master -> Stopped node1) +- * Stop rsc1:1 (node2) due to node availability ++ * Stop rsc1:0 ( Master node1 ) due to node availability ++ * Stop rsc1:1 ( Slave node2 ) due to node availability + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +diff --git a/pengine/test10/master-demote.summary b/pengine/test10/master-demote.summary +index 678e20c..b50fb90 100644 +--- a/pengine/test10/master-demote.summary ++++ b/pengine/test10/master-demote.summary +@@ -23,7 +23,7 @@ Online: [ cxa1 cxb1 ] + Started: [ cxa1 cxb1 ] + + Transition Summary: +- * Move named_address (Started cxa1 -> cxb1) ++ * Move named_address ( cxa1 -> cxb1 ) + * Promote named_drbd_node:1 (Slave -> Master cxb1) + + Executing cluster transition: +diff --git a/pengine/test10/master-failed-demote-2.summary b/pengine/test10/master-failed-demote-2.summary +index 086d02e..f5f535c 100644 +--- a/pengine/test10/master-failed-demote-2.summary ++++ b/pengine/test10/master-failed-demote-2.summary +@@ -11,7 +11,7 @@ Online: [ dl380g5a dl380g5b ] + stateful-2:1 (ocf::heartbeat:Stateful): Slave dl380g5a + + Transition Summary: +- * Stop stateful-1:0 (dl380g5b) due to node availability ++ * Stop stateful-1:0 ( Slave dl380g5b ) due to node availability + * Promote stateful-1:1 (Slave -> Master dl380g5a) + * Promote stateful-2:1 (Slave -> Master dl380g5a) + +diff --git a/pengine/test10/master-failed-demote.summary b/pengine/test10/master-failed-demote.summary +index 0f6c410..ec31e42 100644 +--- a/pengine/test10/master-failed-demote.summary ++++ b/pengine/test10/master-failed-demote.summary +@@ -11,7 +11,7 @@ Online: [ dl380g5a dl380g5b ] + stateful-2:1 (ocf::heartbeat:Stateful): Slave dl380g5a + + Transition Summary: +- * Stop stateful-1:0 (dl380g5b) due to node availability ++ * Stop stateful-1:0 ( Slave dl380g5b ) due to node availability + * Promote stateful-1:1 (Slave -> Master dl380g5a) + * Promote stateful-2:1 (Slave -> Master dl380g5a) + +diff --git a/pengine/test10/master-move.summary b/pengine/test10/master-move.summary +index 2f7d877..e42fa27 100644 +--- a/pengine/test10/master-move.summary ++++ b/pengine/test10/master-move.summary +@@ -11,8 +11,8 @@ Online: [ bl460g1n13 bl460g1n14 ] + Slaves: [ bl460g1n14 ] + + Transition Summary: +- * Recover dummy01 (Started bl460g1n13 -> bl460g1n14) +- * Move dummy02 (Started bl460g1n13 -> bl460g1n14) ++ * Recover dummy01 ( bl460g1n13 -> bl460g1n14 ) ++ * Move dummy02 ( bl460g1n13 -> bl460g1n14 ) + * Start dummy03 (bl460g1n14) + * Demote prmDRBD:0 (Master -> Slave bl460g1n13) + * Promote prmDRBD:1 (Slave -> Master bl460g1n14) +diff --git a/pengine/test10/master-partially-demoted-group.summary b/pengine/test10/master-partially-demoted-group.summary +index 0bda605..0abf07c 100644 +--- a/pengine/test10/master-partially-demoted-group.summary ++++ b/pengine/test10/master-partially-demoted-group.summary +@@ -22,14 +22,14 @@ Online: [ sd01-0 sd01-1 ] + Slaves: [ sd01-0 ] + + Transition Summary: +- * Move vip-164 (Started sd01-1 -> sd01-0) +- * Move vip-165 (Started sd01-1 -> sd01-0) +- * Move cdev-pool-0-iscsi-target (Started sd01-1 -> sd01-0) +- * Move cdev-pool-0-iscsi-lun-1 (Started sd01-1 -> sd01-0) +- * Demote vip-164-fw:0 (Master -> Slave sd01-1) ++ * Move vip-164 ( sd01-1 -> sd01-0 ) ++ * Move vip-165 ( sd01-1 -> sd01-0 ) ++ * Move cdev-pool-0-iscsi-target ( sd01-1 -> sd01-0 ) ++ * Move cdev-pool-0-iscsi-lun-1 ( sd01-1 -> sd01-0 ) ++ * Demote vip-164-fw:0 ( Master -> Slave sd01-1 ) + * Promote vip-164-fw:1 (Slave -> Master sd01-0) + * Promote vip-165-fw:1 (Slave -> Master sd01-0) +- * Demote cdev-pool-0-drbd:0 (Master -> Slave sd01-1) ++ * Demote cdev-pool-0-drbd:0 ( Master -> Slave sd01-1 ) + * Promote cdev-pool-0-drbd:1 (Slave -> Master sd01-0) + + Executing cluster transition: +diff --git a/pengine/test10/master-probed-score.summary b/pengine/test10/master-probed-score.summary +index e7f2ba3..3c67fe9 100644 +--- a/pengine/test10/master-probed-score.summary ++++ b/pengine/test10/master-probed-score.summary +@@ -37,10 +37,8 @@ Online: [ hypatia-corosync.nevis.columbia.edu orestes-corosync.nevis.columbia.ed + Proxy (ocf::heartbeat:VirtualDomain): Stopped + + Transition Summary: +- * Start AdminDrbd:0 (hypatia-corosync.nevis.columbia.edu) +- * Promote AdminDrbd:0 (Stopped -> Master hypatia-corosync.nevis.columbia.edu) +- * Start AdminDrbd:1 (orestes-corosync.nevis.columbia.edu) +- * Promote AdminDrbd:1 (Stopped -> Master orestes-corosync.nevis.columbia.edu) ++ * Promote AdminDrbd:0 ( Stopped -> Master hypatia-corosync.nevis.columbia.edu ) ++ * Promote AdminDrbd:1 ( Stopped -> Master orestes-corosync.nevis.columbia.edu ) + * Start CronAmbientTemperature (hypatia-corosync.nevis.columbia.edu) + * Start StonithHypatia (orestes-corosync.nevis.columbia.edu) + * Start StonithOrestes (hypatia-corosync.nevis.columbia.edu) +diff --git a/pengine/test10/master-pseudo.summary b/pengine/test10/master-pseudo.summary +index 2ee2d03..8f67a68 100644 +--- a/pengine/test10/master-pseudo.summary ++++ b/pengine/test10/master-pseudo.summary +@@ -12,8 +12,7 @@ Online: [ sambuca.linbit ] + + Transition Summary: + * Start ip_float_right (sambuca.linbit) +- * Restart drbd_float:0 (Slave sambuca.linbit) due to required ip_float_right start +- * Promote drbd_float:0 (Slave -> Master sambuca.linbit) ++ * Restart drbd_float:0 ( Slave -> Master sambuca.linbit ) due to required ip_float_right start + * Start ip_nfs (sambuca.linbit) + + Executing cluster transition: +diff --git a/pengine/test10/master-stop.summary b/pengine/test10/master-stop.summary +index 8b802d4..8b861df 100644 +--- a/pengine/test10/master-stop.summary ++++ b/pengine/test10/master-stop.summary +@@ -6,7 +6,7 @@ Online: [ node1 node2 node3 ] + Slaves: [ node1 node2 node3 ] + + Transition Summary: +- * Stop dummy:2 (node3) due to node availability ++ * Stop dummy:2 ( Slave node3 ) due to node availability + + Executing cluster transition: + * Pseudo action: m_stop_0 +diff --git a/pengine/test10/migrate-1.summary b/pengine/test10/migrate-1.summary +index 8df4fdb..ee3d352 100644 +--- a/pengine/test10/migrate-1.summary ++++ b/pengine/test10/migrate-1.summary +@@ -6,7 +6,7 @@ Online: [ node2 ] + rsc3 (heartbeat:apache): Started node1 + + Transition Summary: +- * Migrate rsc3 (Started node1 -> node2) ++ * Migrate rsc3 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc3 monitor on node2 +diff --git a/pengine/test10/migrate-3.summary b/pengine/test10/migrate-3.summary +index 9ccc062..9288be3 100644 +--- a/pengine/test10/migrate-3.summary ++++ b/pengine/test10/migrate-3.summary +@@ -6,7 +6,7 @@ Online: [ node2 ] + rsc3 (heartbeat:apache): FAILED node1 + + Transition Summary: +- * Recover rsc3 (Started node1 -> node2) ++ * Recover rsc3 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc3 monitor on node2 +diff --git a/pengine/test10/migrate-4.summary b/pengine/test10/migrate-4.summary +index 32449a7..b95521a 100644 +--- a/pengine/test10/migrate-4.summary ++++ b/pengine/test10/migrate-4.summary +@@ -6,7 +6,7 @@ Online: [ node2 ] + rsc3 (heartbeat:apache): FAILED node2 + + Transition Summary: +- * Recover rsc3 (Started node2) ++ * Recover rsc3 ( node2 ) + + Executing cluster transition: + * Resource action: rsc3 stop on node2 +diff --git a/pengine/test10/migrate-5.summary b/pengine/test10/migrate-5.summary +index a9d5771..0a939dd 100644 +--- a/pengine/test10/migrate-5.summary ++++ b/pengine/test10/migrate-5.summary +@@ -8,7 +8,7 @@ Online: [ dom0-01 ] + Started: [ dom0-01 dom0-02 ] + + Transition Summary: +- * Migrate domU-test01 (Started dom0-02 -> dom0-01) ++ * Migrate domU-test01 ( dom0-02 -> dom0-01 ) + * Stop dom0-iscsi1-cnx1:1 (dom0-02) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/migrate-begin.summary b/pengine/test10/migrate-begin.summary +index b4b5870..56e0f33 100644 +--- a/pengine/test10/migrate-begin.summary ++++ b/pengine/test10/migrate-begin.summary +@@ -7,7 +7,7 @@ Online: [ hex-13 hex-14 ] + Started: [ hex-13 hex-14 ] + + Transition Summary: +- * Migrate test-vm (Started hex-14 -> hex-13) ++ * Migrate test-vm ( hex-14 -> hex-13 ) + + Executing cluster transition: + * Pseudo action: load_stopped_hex-13 +diff --git a/pengine/test10/migrate-both-vms.summary b/pengine/test10/migrate-both-vms.summary +index 5b60727..d45d267 100644 +--- a/pengine/test10/migrate-both-vms.summary ++++ b/pengine/test10/migrate-both-vms.summary +@@ -32,8 +32,8 @@ Transition Summary: + * Stop p-libvirtd:2 (cvmh03) due to node availability + * Stop fs-bind-libvirt-VM-cvmh:0 (cvmh04) due to node availability + * Stop fs-bind-libvirt-VM-cvmh:2 (cvmh03) due to node availability +- * Migrate vm-compute-test (Started cvmh03 -> cvmh01) +- * Migrate vm-swbuildsl6 (Started cvmh04 -> cvmh02) ++ * Migrate vm-compute-test ( cvmh03 -> cvmh01 ) ++ * Migrate vm-swbuildsl6 ( cvmh04 -> cvmh02 ) + + Executing cluster transition: + * Pseudo action: c-watch-ib0_stop_0 +diff --git a/pengine/test10/migrate-fail-2.summary b/pengine/test10/migrate-fail-2.summary +index 3786533..f3709c4 100644 +--- a/pengine/test10/migrate-fail-2.summary ++++ b/pengine/test10/migrate-fail-2.summary +@@ -7,7 +7,7 @@ Online: [ hex-13 hex-14 ] + Started: [ hex-13 hex-14 ] + + Transition Summary: +- * Recover test-vm (Started hex-13) ++ * Recover test-vm ( hex-13 ) + + Executing cluster transition: + * Resource action: test-vm stop on hex-14 +diff --git a/pengine/test10/migrate-fail-3.summary b/pengine/test10/migrate-fail-3.summary +index 547c110..35ec75f 100644 +--- a/pengine/test10/migrate-fail-3.summary ++++ b/pengine/test10/migrate-fail-3.summary +@@ -7,7 +7,7 @@ Online: [ hex-13 hex-14 ] + Started: [ hex-13 hex-14 ] + + Transition Summary: +- * Recover test-vm (Started hex-13) ++ * Recover test-vm ( hex-13 ) + + Executing cluster transition: + * Resource action: test-vm stop on hex-13 +diff --git a/pengine/test10/migrate-fail-4.summary b/pengine/test10/migrate-fail-4.summary +index cda25ad..d36c61b 100644 +--- a/pengine/test10/migrate-fail-4.summary ++++ b/pengine/test10/migrate-fail-4.summary +@@ -7,7 +7,7 @@ Online: [ hex-13 hex-14 ] + Started: [ hex-13 hex-14 ] + + Transition Summary: +- * Restart test-vm (Started hex-13) ++ * Restart test-vm ( hex-13 ) + + Executing cluster transition: + * Resource action: test-vm stop on hex-14 +diff --git a/pengine/test10/migrate-fail-6.summary b/pengine/test10/migrate-fail-6.summary +index dfde8bb..7cabcf3 100644 +--- a/pengine/test10/migrate-fail-6.summary ++++ b/pengine/test10/migrate-fail-6.summary +@@ -7,7 +7,7 @@ Online: [ hex-13 hex-14 ] + Started: [ hex-13 hex-14 ] + + Transition Summary: +- * Recover test-vm (Started hex-13) ++ * Recover test-vm ( hex-13 ) + + Executing cluster transition: + * Resource action: test-vm stop on hex-13 +diff --git a/pengine/test10/migrate-fail-7.summary b/pengine/test10/migrate-fail-7.summary +index 7a41758..f05bb75 100644 +--- a/pengine/test10/migrate-fail-7.summary ++++ b/pengine/test10/migrate-fail-7.summary +@@ -7,7 +7,7 @@ Online: [ hex-13 hex-14 ] + Started: [ hex-13 hex-14 ] + + Transition Summary: +- * Restart test-vm (Started hex-13) ++ * Restart test-vm ( hex-13 ) + + Executing cluster transition: + * Pseudo action: load_stopped_hex-14 +diff --git a/pengine/test10/migrate-fail-8.summary b/pengine/test10/migrate-fail-8.summary +index 9a651eb..018a466 100644 +--- a/pengine/test10/migrate-fail-8.summary ++++ b/pengine/test10/migrate-fail-8.summary +@@ -7,7 +7,7 @@ Online: [ hex-13 hex-14 ] + Started: [ hex-13 hex-14 ] + + Transition Summary: +- * Recover test-vm (Started hex-14 -> hex-13) ++ * Recover test-vm ( hex-14 -> hex-13 ) + + Executing cluster transition: + * Resource action: test-vm stop on hex-14 +diff --git a/pengine/test10/migrate-fail-9.summary b/pengine/test10/migrate-fail-9.summary +index 82ded68..0f99e92 100644 +--- a/pengine/test10/migrate-fail-9.summary ++++ b/pengine/test10/migrate-fail-9.summary +@@ -7,7 +7,7 @@ Online: [ hex-13 hex-14 ] + Started: [ hex-13 hex-14 ] + + Transition Summary: +- * Start test-vm (hex-13) ++ * Restart test-vm ( hex-13 ) + + Executing cluster transition: + * Resource action: test-vm stop on hex-14 +diff --git a/pengine/test10/migrate-fencing.summary b/pengine/test10/migrate-fencing.summary +index cfd4eb8..44f3262 100644 +--- a/pengine/test10/migrate-fencing.summary ++++ b/pengine/test10/migrate-fencing.summary +@@ -24,14 +24,14 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Transition Summary: + * Fence (reboot) pcmk-4 'termination was requested' + * Stop FencingChild:0 (pcmk-4) due to node availability +- * Move r192.168.101.181 (Started pcmk-4 -> pcmk-1) +- * Move r192.168.101.182 (Started pcmk-4 -> pcmk-1) +- * Move r192.168.101.183 (Started pcmk-4 -> pcmk-1) +- * Move rsc_pcmk-4 (Started pcmk-4 -> pcmk-2) +- * Move lsb-dummy (Started pcmk-4 -> pcmk-1) +- * Migrate migrator (Started pcmk-1 -> pcmk-3) ++ * Move r192.168.101.181 ( pcmk-4 -> pcmk-1 ) ++ * Move r192.168.101.182 ( pcmk-4 -> pcmk-1 ) ++ * Move r192.168.101.183 ( pcmk-4 -> pcmk-1 ) ++ * Move rsc_pcmk-4 ( pcmk-4 -> pcmk-2 ) ++ * Move lsb-dummy ( pcmk-4 -> pcmk-1 ) ++ * Migrate migrator ( pcmk-1 -> pcmk-3 ) + * Stop ping-1:0 (pcmk-4) due to node availability +- * Demote stateful-1:0 (Master -> Stopped pcmk-4) ++ * Stop stateful-1:0 ( Master pcmk-4 ) due to node availability + * Promote stateful-1:1 (Slave -> Master pcmk-1) + + Executing cluster transition: +diff --git a/pengine/test10/migrate-partial-2.summary b/pengine/test10/migrate-partial-2.summary +index 0cf3c75..7ef1a5d 100644 +--- a/pengine/test10/migrate-partial-2.summary ++++ b/pengine/test10/migrate-partial-2.summary +@@ -7,7 +7,7 @@ Online: [ hex-13 hex-14 ] + Started: [ hex-13 hex-14 ] + + Transition Summary: +- * Migrate test-vm (Started hex-14 -> hex-13) ++ * Migrate test-vm ( hex-14 -> hex-13 ) + + Executing cluster transition: + * Resource action: test-vm migrate_from on hex-13 +diff --git a/pengine/test10/migrate-partial-3.summary b/pengine/test10/migrate-partial-3.summary +index 516e470..0dbe46a 100644 +--- a/pengine/test10/migrate-partial-3.summary ++++ b/pengine/test10/migrate-partial-3.summary +@@ -9,7 +9,7 @@ OFFLINE: [ hex-15 ] + Stopped: [ hex-15 ] + + Transition Summary: +- * Recover test-vm (Started hex-14 -> hex-13) ++ * Recover test-vm ( hex-14 -> hex-13 ) + + Executing cluster transition: + * Resource action: test-vm stop on hex-14 +diff --git a/pengine/test10/migrate-partial-4.summary b/pengine/test10/migrate-partial-4.summary +index c3f7012..8fd1d4c 100644 +--- a/pengine/test10/migrate-partial-4.summary ++++ b/pengine/test10/migrate-partial-4.summary +@@ -48,8 +48,8 @@ Online: [ lustre01-left lustre02-left lustre03-left lustre04-left ] + Transition Summary: + * Start drbd-stacked (lustre02-left) + * Start drbd-testfs-local (lustre03-left) +- * Migrate lustre (Started lustre03-left -> lustre04-left) +- * Move testfs (Started lustre02-left -> lustre03-left) ++ * Migrate lustre ( lustre03-left -> lustre04-left ) ++ * Move testfs ( lustre02-left -> lustre03-left ) + * Start drbd-mgs:0 (lustre01-left) + * Start drbd-mgs:1 (lustre02-left) + +diff --git a/pengine/test10/migrate-shutdown.summary b/pengine/test10/migrate-shutdown.summary +index b9aa5b2..508c399 100644 +--- a/pengine/test10/migrate-shutdown.summary ++++ b/pengine/test10/migrate-shutdown.summary +@@ -38,9 +38,9 @@ Transition Summary: + * Stop ping-1:0 (pcmk-1) due to node availability + * Stop ping-1:1 (pcmk-2) due to node availability + * Stop ping-1:2 (pcmk-4) due to node availability +- * Stop stateful-1:0 (pcmk-1) due to node availability +- * Demote stateful-1:1 (Master -> Stopped pcmk-2) +- * Stop stateful-1:2 (pcmk-4) due to node availability ++ * Stop stateful-1:0 ( Slave pcmk-1 ) due to node availability ++ * Stop stateful-1:1 ( Master pcmk-2 ) due to node availability ++ * Stop stateful-1:2 ( Slave pcmk-4 ) due to node availability + + Executing cluster transition: + * Resource action: Fencing stop on pcmk-1 +diff --git a/pengine/test10/migrate-start-complex.summary b/pengine/test10/migrate-start-complex.summary +index ed88398..9dd0924 100644 +--- a/pengine/test10/migrate-start-complex.summary ++++ b/pengine/test10/migrate-start-complex.summary +@@ -11,8 +11,8 @@ Online: [ dom0-01 dom0-02 ] + Stopped: [ dom0-01 dom0-02 ] + + Transition Summary: +- * Move top (Started dom0-02 -> dom0-01) +- * Migrate domU-test01 (Started dom0-02 -> dom0-01) ++ * Move top ( dom0-02 -> dom0-01 ) ++ * Migrate domU-test01 ( dom0-02 -> dom0-01 ) + * Start dom0-iscsi1-cnx1:1 (dom0-01) + * Start bottom:0 (dom0-01) + * Start bottom:1 (dom0-02) +diff --git a/pengine/test10/migrate-start.summary b/pengine/test10/migrate-start.summary +index 8bbe3e5..fb6fdef 100644 +--- a/pengine/test10/migrate-start.summary ++++ b/pengine/test10/migrate-start.summary +@@ -8,7 +8,7 @@ Online: [ dom0-01 dom0-02 ] + Stopped: [ dom0-01 ] + + Transition Summary: +- * Migrate domU-test01 (Started dom0-02 -> dom0-01) ++ * Migrate domU-test01 ( dom0-02 -> dom0-01 ) + * Start dom0-iscsi1-cnx1:1 (dom0-01) + + Executing cluster transition: +diff --git a/pengine/test10/migrate-stop-complex.summary b/pengine/test10/migrate-stop-complex.summary +index 04dce1a..4aa6cf7 100644 +--- a/pengine/test10/migrate-stop-complex.summary ++++ b/pengine/test10/migrate-stop-complex.summary +@@ -11,8 +11,8 @@ Online: [ dom0-01 ] + Started: [ dom0-01 dom0-02 ] + + Transition Summary: +- * Move top (Started dom0-02 -> dom0-01) +- * Migrate domU-test01 (Started dom0-02 -> dom0-01) ++ * Move top ( dom0-02 -> dom0-01 ) ++ * Migrate domU-test01 ( dom0-02 -> dom0-01 ) + * Stop dom0-iscsi1-cnx1:1 (dom0-02) due to node availability + * Stop bottom:1 (dom0-02) due to node availability + +diff --git a/pengine/test10/migrate-stop-start-complex.summary b/pengine/test10/migrate-stop-start-complex.summary +index 5408bb6..f0f5843 100644 +--- a/pengine/test10/migrate-stop-start-complex.summary ++++ b/pengine/test10/migrate-stop-start-complex.summary +@@ -12,9 +12,9 @@ Online: [ dom0-01 ] + Stopped: [ dom0-01 ] + + Transition Summary: +- * Migrate domU-test01 (Started dom0-02 -> dom0-01) ++ * Migrate domU-test01 ( dom0-02 -> dom0-01 ) + * Stop dom0-iscsi1-cnx1:1 (dom0-02) due to node availability +- * Move bottom:0 (Started dom0-02 -> dom0-01) ++ * Move bottom:0 ( dom0-02 -> dom0-01 ) + + Executing cluster transition: + * Resource action: domU-test01 migrate_to on dom0-02 +diff --git a/pengine/test10/migrate-stop.summary b/pengine/test10/migrate-stop.summary +index a9d5771..0a939dd 100644 +--- a/pengine/test10/migrate-stop.summary ++++ b/pengine/test10/migrate-stop.summary +@@ -8,7 +8,7 @@ Online: [ dom0-01 ] + Started: [ dom0-01 dom0-02 ] + + Transition Summary: +- * Migrate domU-test01 (Started dom0-02 -> dom0-01) ++ * Migrate domU-test01 ( dom0-02 -> dom0-01 ) + * Stop dom0-iscsi1-cnx1:1 (dom0-02) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/migrate-stop_start.summary b/pengine/test10/migrate-stop_start.summary +index cadb24f..de89651 100644 +--- a/pengine/test10/migrate-stop_start.summary ++++ b/pengine/test10/migrate-stop_start.summary +@@ -9,8 +9,8 @@ Online: [ dom0-01 ] + Stopped: [ dom0-01 ] + + Transition Summary: +- * Migrate domU-test01 (Started dom0-02 -> dom0-01) +- * Move dom0-iscsi1-cnx1:0 (Started dom0-02 -> dom0-01) ++ * Migrate domU-test01 ( dom0-02 -> dom0-01 ) ++ * Move dom0-iscsi1-cnx1:0 ( dom0-02 -> dom0-01 ) + + Executing cluster transition: + * Pseudo action: clone-dom0-iscsi1_stop_0 +diff --git a/pengine/test10/mon-rsc-2.summary b/pengine/test10/mon-rsc-2.summary +index 3f649e5..6c219b5 100644 +--- a/pengine/test10/mon-rsc-2.summary ++++ b/pengine/test10/mon-rsc-2.summary +@@ -6,7 +6,7 @@ Online: [ node1 ] + rsc1 (heartbeat:apache): Started node2 + + Transition Summary: +- * Move rsc1 (Started node2 -> node1) ++ * Move rsc1 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: rsc1 monitor on node1 +diff --git a/pengine/test10/mon-rsc-4.summary b/pengine/test10/mon-rsc-4.summary +index 92b0f68..d4debf3 100644 +--- a/pengine/test10/mon-rsc-4.summary ++++ b/pengine/test10/mon-rsc-4.summary +@@ -6,7 +6,7 @@ Online: [ node1 ] + rsc1 (heartbeat:apache): Starting node2 + + Transition Summary: +- * Move rsc1 (Started node2 -> node1) ++ * Move rsc1 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: rsc1 monitor on node1 +diff --git a/pengine/test10/monitor-onfail-restart.summary b/pengine/test10/monitor-onfail-restart.summary +index 34bc76a..b15c321 100644 +--- a/pengine/test10/monitor-onfail-restart.summary ++++ b/pengine/test10/monitor-onfail-restart.summary +@@ -6,7 +6,7 @@ OFFLINE: [ fc16-builder2 ] + A (ocf::pacemaker:Dummy): FAILED fc16-builder + + Transition Summary: +- * Recover A (Started fc16-builder) ++ * Recover A ( fc16-builder ) + + Executing cluster transition: + * Resource action: A stop on fc16-builder +diff --git a/pengine/test10/multi1.summary b/pengine/test10/multi1.summary +index c782e98..40222fc 100644 +--- a/pengine/test10/multi1.summary ++++ b/pengine/test10/multi1.summary +@@ -5,7 +5,7 @@ Online: [ node1 node2 ] + rsc1 (heartbeat:apache): Started [ node1 node2 ] + + Transition Summary: +- * Restart rsc1 (Started node1) ++ * Restart rsc1 ( node1 ) + + Executing cluster transition: + * Resource action: rsc1 stop on node2 +diff --git a/pengine/test10/multiple-monitor-one-failed.summary b/pengine/test10/multiple-monitor-one-failed.summary +index b10abbe..0691ebf 100644 +--- a/pengine/test10/multiple-monitor-one-failed.summary ++++ b/pengine/test10/multiple-monitor-one-failed.summary +@@ -5,7 +5,7 @@ Online: [ dhcp180 dhcp69 ] + Dummy-test2 (ocf::test:Dummy): FAILED dhcp180 + + Transition Summary: +- * Recover Dummy-test2 (Started dhcp180) ++ * Recover Dummy-test2 ( dhcp180 ) + + Executing cluster transition: + * Resource action: Dummy-test2 stop on dhcp180 +diff --git a/pengine/test10/not-installed-agent.summary b/pengine/test10/not-installed-agent.summary +index 5d0b311..b32cdd4 100644 +--- a/pengine/test10/not-installed-agent.summary ++++ b/pengine/test10/not-installed-agent.summary +@@ -7,8 +7,8 @@ Online: [ sles11-1 sles11-2 ] + rsc2 (ocf::pacemaker:Dummy): FAILED sles11-1 + + Transition Summary: +- * Recover rsc1 (Started sles11-1 -> sles11-2) +- * Recover rsc2 (Started sles11-1 -> sles11-2) ++ * Recover rsc1 ( sles11-1 -> sles11-2 ) ++ * Recover rsc2 ( sles11-1 -> sles11-2 ) + + Executing cluster transition: + * Resource action: rsc1 stop on sles11-1 +diff --git a/pengine/test10/not-installed-tools.summary b/pengine/test10/not-installed-tools.summary +index e429333..01ae7f7 100644 +--- a/pengine/test10/not-installed-tools.summary ++++ b/pengine/test10/not-installed-tools.summary +@@ -7,7 +7,7 @@ Online: [ sles11-1 sles11-2 ] + rsc2 (ocf::pacemaker:Dummy): Started sles11-1 (failure ignored) + + Transition Summary: +- * Recover rsc1 (Started sles11-1 -> sles11-2) ++ * Recover rsc1 ( sles11-1 -> sles11-2 ) + + Executing cluster transition: + * Resource action: rsc1 stop on sles11-1 +diff --git a/pengine/test10/not-reschedule-unneeded-monitor.summary b/pengine/test10/not-reschedule-unneeded-monitor.summary +index 2a872d5..429ea24 100644 +--- a/pengine/test10/not-reschedule-unneeded-monitor.summary ++++ b/pengine/test10/not-reschedule-unneeded-monitor.summary +@@ -14,7 +14,7 @@ Online: [ castor kimball ] + Started: [ castor kimball ] + + Transition Summary: +- * Recover sle12-kvm (Started castor -> kimball) ++ * Recover sle12-kvm ( castor -> kimball ) + + Executing cluster transition: + * Resource action: sle12-kvm stop on castor +diff --git a/pengine/test10/notify-3.summary b/pengine/test10/notify-3.summary +index 03a3d4e..6c3367e 100644 +--- a/pengine/test10/notify-3.summary ++++ b/pengine/test10/notify-3.summary +@@ -10,7 +10,7 @@ Online: [ node1 node2 ] + child_rsc2:1 (heartbeat:apache): Stopped + + Transition Summary: +- * Move child_rsc1:1 (Started node2 -> node1) ++ * Move child_rsc1:1 ( node2 -> node1 ) + * Stop child_rsc2:0 (node1) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/novell-239082.summary b/pengine/test10/novell-239082.summary +index b596de4..2bafd1b 100644 +--- a/pengine/test10/novell-239082.summary ++++ b/pengine/test10/novell-239082.summary +@@ -9,9 +9,9 @@ Online: [ xen-1 xen-2 ] + + Transition Summary: + * Shutdown xen-1 +- * Move fs_1 (Started xen-1 -> xen-2) ++ * Move fs_1 ( xen-1 -> xen-2 ) + * Promote drbd0:0 (Slave -> Master xen-2) +- * Demote drbd0:1 (Master -> Stopped xen-1) ++ * Stop drbd0:1 ( Master xen-1 ) due to node availability + + Executing cluster transition: + * Resource action: fs_1 stop on xen-1 +diff --git a/pengine/test10/novell-252693-2.summary b/pengine/test10/novell-252693-2.summary +index 2e8e4b5..392d26e 100644 +--- a/pengine/test10/novell-252693-2.summary ++++ b/pengine/test10/novell-252693-2.summary +@@ -25,7 +25,7 @@ Transition Summary: + * Start evmsclone:1 (node1) + * Start imagestoreclone:1 (node1) + * Start configstoreclone:1 (node1) +- * Migrate sles10 (Started node2 -> node1) ++ * Migrate sles10 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: stonithclone:1 monitor on node1 +diff --git a/pengine/test10/novell-252693-3.summary b/pengine/test10/novell-252693-3.summary +index 741c957..5c3b0ca 100644 +--- a/pengine/test10/novell-252693-3.summary ++++ b/pengine/test10/novell-252693-3.summary +@@ -23,10 +23,10 @@ Transition Summary: + * Start stonithclone:1 (node1) + * Start evmsdclone:1 (node1) + * Start evmsclone:1 (node1) +- * Recover imagestoreclone:0 (Started node2 -> node1) ++ * Recover imagestoreclone:0 ( node2 -> node1 ) + * Start imagestoreclone:1 (node2) + * Start configstoreclone:1 (node1) +- * Migrate sles10 (Started node2 -> node1) ++ * Migrate sles10 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: stonithclone:1 monitor on node1 +diff --git a/pengine/test10/novell-252693.summary b/pengine/test10/novell-252693.summary +index 0682119..27cf315 100644 +--- a/pengine/test10/novell-252693.summary ++++ b/pengine/test10/novell-252693.summary +@@ -21,7 +21,7 @@ Transition Summary: + * Stop evmsclone:1 (node1) due to node availability + * Stop imagestoreclone:1 (node1) due to node availability + * Stop configstoreclone:1 (node1) due to node availability +- * Migrate sles10 (Started node1 -> node2) ++ * Migrate sles10 ( node1 -> node2 ) + + Executing cluster transition: + * Pseudo action: stonithcloneset_stop_0 +diff --git a/pengine/test10/one-or-more-1.summary b/pengine/test10/one-or-more-1.summary +index af31a46..d2e79bc 100644 +--- a/pengine/test10/one-or-more-1.summary ++++ b/pengine/test10/one-or-more-1.summary +@@ -10,9 +10,9 @@ OFFLINE: [ fc16-builder2 ] + D (ocf::pacemaker:Dummy): Stopped + + Transition Summary: +- * Start B (fc16-builder - blocked) due to unrunnable A start +- * Start C (fc16-builder - blocked) due to unrunnable A start +- * Start D (fc16-builder - blocked) due to unrunnable one-or-more:require-all-set-1 ++ * Start B ( fc16-builder ) due to unrunnable A start (blocked) ++ * Start C ( fc16-builder ) due to unrunnable A start (blocked) ++ * Start D ( fc16-builder ) due to unrunnable one-or-more:require-all-set-1 (blocked) + + Executing cluster transition: + * Resource action: A monitor on fc16-builder +diff --git a/pengine/test10/one-or-more-3.summary b/pengine/test10/one-or-more-3.summary +index abf4081..97511e8 100644 +--- a/pengine/test10/one-or-more-3.summary ++++ b/pengine/test10/one-or-more-3.summary +@@ -11,7 +11,7 @@ OFFLINE: [ fc16-builder2 ] + + Transition Summary: + * Start A (fc16-builder) +- * Start D (fc16-builder - blocked) due to unrunnable one-or-more:require-all-set-1 ++ * Start D ( fc16-builder ) due to unrunnable one-or-more:require-all-set-1 (blocked) + + Executing cluster transition: + * Resource action: A monitor on fc16-builder +diff --git a/pengine/test10/order-mandatory.summary b/pengine/test10/order-mandatory.summary +index 021d8fe..955a281 100644 +--- a/pengine/test10/order-mandatory.summary ++++ b/pengine/test10/order-mandatory.summary +@@ -9,8 +9,8 @@ Online: [ node1 ] + + Transition Summary: + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) due to required rsc1 start +- * Stop rsc4 (Started node1) due to unrunnable rsc3 start ++ * Restart rsc2 ( node1 ) due to required rsc1 start ++ * Stop rsc4 ( node1 ) due to unrunnable rsc3 start + + Executing cluster transition: + * Resource action: rsc1 start on node1 +diff --git a/pengine/test10/order-required.summary b/pengine/test10/order-required.summary +index 021d8fe..955a281 100644 +--- a/pengine/test10/order-required.summary ++++ b/pengine/test10/order-required.summary +@@ -9,8 +9,8 @@ Online: [ node1 ] + + Transition Summary: + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) due to required rsc1 start +- * Stop rsc4 (Started node1) due to unrunnable rsc3 start ++ * Restart rsc2 ( node1 ) due to required rsc1 start ++ * Stop rsc4 ( node1 ) due to unrunnable rsc3 start + + Executing cluster transition: + * Resource action: rsc1 start on node1 +diff --git a/pengine/test10/order-serialize-set.summary b/pengine/test10/order-serialize-set.summary +index 5483cca..a9f97a3 100644 +--- a/pengine/test10/order-serialize-set.summary ++++ b/pengine/test10/order-serialize-set.summary +@@ -15,13 +15,13 @@ Online: [ xen-b ] + Email_Alerting (ocf::heartbeat:MailTo): Started xen-b + + Transition Summary: +- * Restart xen-a-fencing (Started xen-b) ++ * Restart xen-a-fencing ( xen-b ) + * Stop xen-b-fencing (xen-a) +- * Migrate db (Started xen-a -> xen-b) +- * Migrate core-101 (Started xen-a -> xen-b) +- * Migrate core-200 (Started xen-a -> xen-b) +- * Migrate edge (Started xen-a -> xen-b) +- * Migrate base (Started xen-a -> xen-b) ++ * Migrate db ( xen-a -> xen-b ) ++ * Migrate core-101 ( xen-a -> xen-b ) ++ * Migrate core-200 ( xen-a -> xen-b ) ++ * Migrate edge ( xen-a -> xen-b ) ++ * Migrate base ( xen-a -> xen-b ) + + Executing cluster transition: + * Resource action: xen-a-fencing stop on xen-b +diff --git a/pengine/test10/order-serialize.summary b/pengine/test10/order-serialize.summary +index b2114eb..f58a6bc 100644 +--- a/pengine/test10/order-serialize.summary ++++ b/pengine/test10/order-serialize.summary +@@ -15,13 +15,13 @@ Online: [ xen-b ] + Email_Alerting (ocf::heartbeat:MailTo): Started xen-b + + Transition Summary: +- * Restart xen-a-fencing (Started xen-b) ++ * Restart xen-a-fencing ( xen-b ) + * Stop xen-b-fencing (xen-a) +- * Migrate db (Started xen-a -> xen-b) +- * Migrate core-101 (Started xen-a -> xen-b) +- * Migrate core-200 (Started xen-a -> xen-b) +- * Migrate edge (Started xen-a -> xen-b) +- * Migrate base (Started xen-a -> xen-b) ++ * Migrate db ( xen-a -> xen-b ) ++ * Migrate core-101 ( xen-a -> xen-b ) ++ * Migrate core-200 ( xen-a -> xen-b ) ++ * Migrate edge ( xen-a -> xen-b ) ++ * Migrate base ( xen-a -> xen-b ) + + Executing cluster transition: + * Resource action: xen-a-fencing stop on xen-b +diff --git a/pengine/test10/order-sets.summary b/pengine/test10/order-sets.summary +index 0d5de5e..e16e1f4 100644 +--- a/pengine/test10/order-sets.summary ++++ b/pengine/test10/order-sets.summary +@@ -9,10 +9,10 @@ Online: [ ubuntu_1 ] + world4 (ocf::bbnd:world4test): Started ubuntu_2 + + Transition Summary: +- * Move world1 (Started ubuntu_2 -> ubuntu_1) +- * Move world2 (Started ubuntu_2 -> ubuntu_1) +- * Move world3 (Started ubuntu_2 -> ubuntu_1) +- * Move world4 (Started ubuntu_2 -> ubuntu_1) ++ * Move world1 ( ubuntu_2 -> ubuntu_1 ) ++ * Move world2 ( ubuntu_2 -> ubuntu_1 ) ++ * Move world3 ( ubuntu_2 -> ubuntu_1 ) ++ * Move world4 ( ubuntu_2 -> ubuntu_1 ) + + Executing cluster transition: + * Resource action: world4 stop on ubuntu_2 +diff --git a/pengine/test10/order-wrong-kind.summary b/pengine/test10/order-wrong-kind.summary +index c498caf..df3bd80 100644 +--- a/pengine/test10/order-wrong-kind.summary ++++ b/pengine/test10/order-wrong-kind.summary +@@ -10,7 +10,7 @@ Online: [ node1 ] + + Transition Summary: + * Start rsc1 (node1) +- * Restart rsc2 (Started node1) due to required rsc1 start ++ * Restart rsc2 ( node1 ) due to required rsc1 start + + Executing cluster transition: + * Resource action: rsc1 start on node1 +diff --git a/pengine/test10/order3.summary b/pengine/test10/order3.summary +index 4289a47..6694c0e 100644 +--- a/pengine/test10/order3.summary ++++ b/pengine/test10/order3.summary +@@ -8,10 +8,10 @@ Online: [ node1 node2 ] + rsc4 (heartbeat:apache): Started node1 + + Transition Summary: +- * Move rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) +- * Move rsc3 (Started node1 -> node2) +- * Move rsc4 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) ++ * Move rsc3 ( node1 -> node2 ) ++ * Move rsc4 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/order5.summary b/pengine/test10/order5.summary +index 0c3e0bc..db222c4 100644 +--- a/pengine/test10/order5.summary ++++ b/pengine/test10/order5.summary +@@ -12,10 +12,10 @@ Online: [ node1 node2 ] + rsc8 (heartbeat:apache): Started node2 + + Transition Summary: +- * Move rsc2 (Started node1 -> node2) +- * Move rsc4 (Started node1 -> node2) +- * Move rsc6 (Started node2 -> node1) +- * Move rsc8 (Started node2 -> node1) ++ * Move rsc2 ( node1 -> node2 ) ++ * Move rsc4 ( node1 -> node2 ) ++ * Move rsc6 ( node2 -> node1 ) ++ * Move rsc8 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/order6.summary b/pengine/test10/order6.summary +index 0c3e0bc..db222c4 100644 +--- a/pengine/test10/order6.summary ++++ b/pengine/test10/order6.summary +@@ -12,10 +12,10 @@ Online: [ node1 node2 ] + rsc8 (heartbeat:apache): Started node2 + + Transition Summary: +- * Move rsc2 (Started node1 -> node2) +- * Move rsc4 (Started node1 -> node2) +- * Move rsc6 (Started node2 -> node1) +- * Move rsc8 (Started node2 -> node1) ++ * Move rsc2 ( node1 -> node2 ) ++ * Move rsc4 ( node1 -> node2 ) ++ * Move rsc6 ( node2 -> node1 ) ++ * Move rsc8 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/order7.summary b/pengine/test10/order7.summary +index 6648bff..a8610e9 100644 +--- a/pengine/test10/order7.summary ++++ b/pengine/test10/order7.summary +@@ -13,7 +13,7 @@ Transition Summary: + * Start rsc2 (node1) + * Start rsc3 (node1) + * Start rscB (node1) +- * Start rscC (node1 - blocked) due to unrunnable rscA start ++ * Start rscC ( node1 ) due to unrunnable rscA start (blocked) + + Executing cluster transition: + * Resource action: rsc2 monitor on node1 +diff --git a/pengine/test10/ordered-set-basic-startup.summary b/pengine/test10/ordered-set-basic-startup.summary +index cfa8f8b..ef546ed 100644 +--- a/pengine/test10/ordered-set-basic-startup.summary ++++ b/pengine/test10/ordered-set-basic-startup.summary +@@ -12,10 +12,10 @@ OFFLINE: [ fc16-builder2 ] + F (ocf::pacemaker:Dummy): Stopped + + Transition Summary: +- * Start A (fc16-builder - blocked) due to unrunnable C start ++ * Start A ( fc16-builder ) due to unrunnable C start (blocked) + * Start B (fc16-builder) +- * Start E (fc16-builder - blocked) due to unrunnable A start +- * Start F (fc16-builder - blocked) due to unrunnable D start ++ * Start E ( fc16-builder ) due to unrunnable A start (blocked) ++ * Start F ( fc16-builder ) due to unrunnable D start (blocked) + + Executing cluster transition: + * Resource action: A monitor on fc16-builder +diff --git a/pengine/test10/ordered-set-natural.summary b/pengine/test10/ordered-set-natural.summary +index 1888d66..e9667fd 100644 +--- a/pengine/test10/ordered-set-natural.summary ++++ b/pengine/test10/ordered-set-natural.summary +@@ -21,12 +21,12 @@ Online: [ node1 node2 ] + dummy2-5 (ocf::heartbeat:Dummy): Stopped + + Transition Summary: +- * Start dummy1-1 (node1 - blocked) due to no quorum +- * Start dummy1-2 (node1 - blocked) due to no quorum +- * Start dummy2-1 (node2 - blocked) due to no quorum +- * Start dummy2-2 (node2 - blocked) due to no quorum +- * Start dummy3-4 (node1 - blocked) due to no quorum +- * Start dummy3-5 (node1 - blocked) due to no quorum ++ * Start dummy1-1 ( node1 ) due to no quorum (blocked) ++ * Start dummy1-2 ( node1 ) due to no quorum (blocked) ++ * Start dummy2-1 ( node2 ) due to no quorum (blocked) ++ * Start dummy2-2 ( node2 ) due to no quorum (blocked) ++ * Start dummy3-4 ( node1 ) due to no quorum (blocked) ++ * Start dummy3-5 ( node1 ) due to no quorum (blocked) + + Executing cluster transition: + +diff --git a/pengine/test10/params-1.summary b/pengine/test10/params-1.summary +index 357c2ac..26fdecf 100644 +--- a/pengine/test10/params-1.summary ++++ b/pengine/test10/params-1.summary +@@ -9,7 +9,7 @@ Online: [ c001n01 c001n02 c001n03 c001n08 ] + rsc_c001n01 (ocf::heartbeat:IPaddr): Started c001n01 + + Transition Summary: +- * Restart DcIPaddr (Started c001n02) ++ * Restart DcIPaddr ( c001n02 ) + + Executing cluster transition: + * Resource action: DcIPaddr monitor on c001n08 +diff --git a/pengine/test10/params-2.summary b/pengine/test10/params-2.summary +index f4169f2..1e73e77 100644 +--- a/pengine/test10/params-2.summary ++++ b/pengine/test10/params-2.summary +@@ -9,7 +9,7 @@ Online: [ node1 node2 node3 ] + Transition Summary: + * Shutdown node1 + * Stop rsc1 (node1) +- * Restart rsc2 (Started node2) ++ * Restart rsc2 ( node2 ) + * Start rsc3 (node3) + + Executing cluster transition: +diff --git a/pengine/test10/params-4.summary b/pengine/test10/params-4.summary +index 67b0426..dfd3122 100644 +--- a/pengine/test10/params-4.summary ++++ b/pengine/test10/params-4.summary +@@ -9,7 +9,7 @@ Online: [ c001n01 c001n02 c001n03 c001n08 ] + rsc_c001n01 (ocf::heartbeat:IPaddr): Started c001n01 + + Transition Summary: +- * Reload DcIPaddr (Started c001n02) ++ * Reload DcIPaddr ( c001n02 ) + + Executing cluster transition: + * Resource action: DcIPaddr monitor on c001n08 +diff --git a/pengine/test10/params-5.summary b/pengine/test10/params-5.summary +index 357c2ac..26fdecf 100644 +--- a/pengine/test10/params-5.summary ++++ b/pengine/test10/params-5.summary +@@ -9,7 +9,7 @@ Online: [ c001n01 c001n02 c001n03 c001n08 ] + rsc_c001n01 (ocf::heartbeat:IPaddr): Started c001n01 + + Transition Summary: +- * Restart DcIPaddr (Started c001n02) ++ * Restart DcIPaddr ( c001n02 ) + + Executing cluster transition: + * Resource action: DcIPaddr monitor on c001n08 +diff --git a/pengine/test10/params-6.summary b/pengine/test10/params-6.summary +index b7c9ff4..20ba745 100644 +--- a/pengine/test10/params-6.summary ++++ b/pengine/test10/params-6.summary +@@ -182,7 +182,7 @@ Online: [ mgmt01 v03-a v03-b ] + dist.fly-uni.org-vm (ocf::vds-ok:VirtualDomain): Stopped ( disabled ) + + Transition Summary: +- * Reload c5-x64-devel.vds-ok.com-vm (Started v03-a) ++ * Reload c5-x64-devel.vds-ok.com-vm ( v03-a ) + + Executing cluster transition: + * Resource action: vd01-b.cdev.ttc.prague.cz.vds-ok.com-vm monitor=10000 on v03-b +diff --git a/pengine/test10/probe-2.summary b/pengine/test10/probe-2.summary +index 05588c8..7e74efc 100644 +--- a/pengine/test10/probe-2.summary ++++ b/pengine/test10/probe-2.summary +@@ -32,16 +32,16 @@ Online: [ wc01 ] + + Transition Summary: + * Promote drbd_mysql:0 (Slave -> Master wc01) +- * Demote drbd_mysql:1 (Master -> Stopped wc02) +- * Move fs_mysql (Started wc02 -> wc01) +- * Move intip_sql (Started wc02 -> wc01) +- * Move mysql-server (Started wc02 -> wc01) +- * Stop drbd_www:1 (wc02) due to node availability ++ * Stop drbd_mysql:1 ( Master wc02 ) due to node availability ++ * Move fs_mysql ( wc02 -> wc01 ) ++ * Move intip_sql ( wc02 -> wc01 ) ++ * Move mysql-server ( wc02 -> wc01 ) ++ * Stop drbd_www:1 ( Slave wc02 ) due to node availability + * Stop nfs-common:1 (wc02) due to node availability + * Stop mysql-proxy:1 (wc02) due to node availability + * Stop fs_www:1 (wc02) due to node availability + * Stop apache2:1 (wc02) due to node availability +- * Restart stonith_rackpdu:0 (Started wc01) ++ * Restart stonith_rackpdu:0 ( wc01 ) + * Stop stonith_rackpdu:1 (wc02) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/probe-4.summary b/pengine/test10/probe-4.summary +index 00ecbc2..691c4b6 100644 +--- a/pengine/test10/probe-4.summary ++++ b/pengine/test10/probe-4.summary +@@ -25,7 +25,7 @@ Online: [ pcmk-1 pcmk-2 pcmk-3 ] + Stopped: [ pcmk-4 ] + + Transition Summary: +- * Start migrator (pcmk-3 - blocked) ++ * Start migrator ( pcmk-3 ) blocked + + Executing cluster transition: + +diff --git a/pengine/test10/quorum-1.summary b/pengine/test10/quorum-1.summary +index 83428d4..fd01941 100644 +--- a/pengine/test10/quorum-1.summary ++++ b/pengine/test10/quorum-1.summary +@@ -7,7 +7,7 @@ Online: [ node1 node2 ] + rsc3 (heartbeat:apache): Stopped + + Transition Summary: +- * Move rsc2 (Started node1 -> node2) ++ * Move rsc2 ( node1 -> node2 ) + * Start rsc3 (node1) + + Executing cluster transition: +diff --git a/pengine/test10/quorum-2.summary b/pengine/test10/quorum-2.summary +index 1dce96c..f603bfa 100644 +--- a/pengine/test10/quorum-2.summary ++++ b/pengine/test10/quorum-2.summary +@@ -7,8 +7,8 @@ Online: [ node1 node2 ] + rsc3 (heartbeat:apache): Stopped + + Transition Summary: +- * Move rsc2 (Started node1 -> node2) +- * Start rsc3 (node1 - blocked) due to quorum freeze ++ * Move rsc2 ( node1 -> node2 ) ++ * Start rsc3 ( node1 ) due to quorum freeze (blocked) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/quorum-3.summary b/pengine/test10/quorum-3.summary +index e8e4bf3..cda0278 100644 +--- a/pengine/test10/quorum-3.summary ++++ b/pengine/test10/quorum-3.summary +@@ -7,9 +7,9 @@ Online: [ node1 node2 ] + rsc3 (heartbeat:apache): Stopped + + Transition Summary: +- * Stop rsc1 (Started node1) due to no quorum +- * Stop rsc2 (Started node1) due to no quorum +- * Start rsc3 (node1 - blocked) due to no quorum ++ * Stop rsc1 ( node1 ) due to no quorum ++ * Stop rsc2 ( node1 ) due to no quorum ++ * Start rsc3 ( node1 ) due to no quorum (blocked) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/rec-node-10.summary b/pengine/test10/rec-node-10.summary +index 503dd0d..296a018 100644 +--- a/pengine/test10/rec-node-10.summary ++++ b/pengine/test10/rec-node-10.summary +@@ -8,9 +8,9 @@ Online: [ node2 ] + rsc2 (heartbeat:apache): Started node1 (UNCLEAN) + + Transition Summary: +- * Start stonith-1 (node2 - blocked) due to no quorum +- * Stop rsc1 (Started node1 - blocked) due to no quorum +- * Stop rsc2 (Started node1 - blocked) due to no quorum ++ * Start stonith-1 ( node2 ) due to no quorum (blocked) ++ * Stop rsc1 ( node1 ) due to no quorum (blocked) ++ * Stop rsc2 ( node1 ) due to no quorum (blocked) + + Executing cluster transition: + * Resource action: stonith-1 monitor on node2 +diff --git a/pengine/test10/rec-node-11.summary b/pengine/test10/rec-node-11.summary +index e884ffa..a354e49 100644 +--- a/pengine/test10/rec-node-11.summary ++++ b/pengine/test10/rec-node-11.summary +@@ -12,9 +12,9 @@ Online: [ node2 ] + Transition Summary: + * Fence (reboot) node1 'peer process is no longer available' + * Start stonith-1 (node2) +- * Move rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) +- * Restart rsc3 (Started node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) ++ * Restart rsc3 ( node2 ) + + Executing cluster transition: + * Resource action: stonith-1 monitor on node2 +diff --git a/pengine/test10/rec-node-13.summary b/pengine/test10/rec-node-13.summary +index de2fa28..8bab0db 100644 +--- a/pengine/test10/rec-node-13.summary ++++ b/pengine/test10/rec-node-13.summary +@@ -35,7 +35,7 @@ OFFLINE: [ c001n03 c001n05 ] + + Transition Summary: + * Fence (reboot) c001n04 'ocf_msdummy:6 failed there' +- * Stop ocf_msdummy:6 (c001n04) due to node availability ++ * Stop ocf_msdummy:6 ( Slave c001n04 ) due to node availability + + Executing cluster transition: + * Fencing c001n04 (reboot) +diff --git a/pengine/test10/rec-node-15.summary b/pengine/test10/rec-node-15.summary +index 760942c..07355c7 100644 +--- a/pengine/test10/rec-node-15.summary ++++ b/pengine/test10/rec-node-15.summary +@@ -23,9 +23,9 @@ Online: [ sapcl01 ] + Transition Summary: + * Fence (reboot) sapcl03 'peer is no longer part of the cluster' + * Start stonith-1 (sapcl01) +- * Move IPaddr_192_168_1_102 (Started sapcl02 -> sapcl01) +- * Move LVM_12 (Started sapcl02 -> sapcl01) +- * Move Filesystem_13 (Started sapcl02 -> sapcl01) ++ * Move IPaddr_192_168_1_102 ( sapcl02 -> sapcl01 ) ++ * Move LVM_12 ( sapcl02 -> sapcl01 ) ++ * Move Filesystem_13 ( sapcl02 -> sapcl01 ) + * Start IPaddr_192_168_1_104 (sapcl01) + * Start LVM_22 (sapcl01) + * Start Filesystem_23 (sapcl01) +diff --git a/pengine/test10/rec-node-4.summary b/pengine/test10/rec-node-4.summary +index 4a39615..30c9667 100644 +--- a/pengine/test10/rec-node-4.summary ++++ b/pengine/test10/rec-node-4.summary +@@ -10,8 +10,8 @@ Online: [ node2 ] + Transition Summary: + * Fence (reboot) node1 'peer is no longer part of the cluster' + * Start stonith-1 (node2) +- * Move rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: stonith-1 monitor on node2 +diff --git a/pengine/test10/rec-node-6.summary b/pengine/test10/rec-node-6.summary +index 6cb8d01..9a45360 100644 +--- a/pengine/test10/rec-node-6.summary ++++ b/pengine/test10/rec-node-6.summary +@@ -10,8 +10,8 @@ Online: [ node2 ] + Transition Summary: + * Fence (reboot) node1 'peer process is no longer available' + * Start stonith-1 (node2) +- * Move rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: stonith-1 monitor on node2 +diff --git a/pengine/test10/rec-node-7.summary b/pengine/test10/rec-node-7.summary +index 4a39615..30c9667 100644 +--- a/pengine/test10/rec-node-7.summary ++++ b/pengine/test10/rec-node-7.summary +@@ -10,8 +10,8 @@ Online: [ node2 ] + Transition Summary: + * Fence (reboot) node1 'peer is no longer part of the cluster' + * Start stonith-1 (node2) +- * Move rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: stonith-1 monitor on node2 +diff --git a/pengine/test10/rec-node-8.summary b/pengine/test10/rec-node-8.summary +index 1a7232e..bc23fb4 100644 +--- a/pengine/test10/rec-node-8.summary ++++ b/pengine/test10/rec-node-8.summary +@@ -9,10 +9,10 @@ Online: [ node2 ] + rsc3 (heartbeat:apache): Stopped + + Transition Summary: +- * Start stonith-1 (node2 - blocked) due to quorum freeze +- * Stop rsc1 (Started node1 - blocked) +- * Stop rsc2 (Started node1 - blocked) +- * Start rsc3 (node2 - blocked) due to quorum freeze ++ * Start stonith-1 ( node2 ) due to quorum freeze (blocked) ++ * Stop rsc1 ( node1 ) blocked ++ * Stop rsc2 ( node1 ) blocked ++ * Start rsc3 ( node2 ) due to quorum freeze (blocked) + + Executing cluster transition: + * Resource action: stonith-1 monitor on node2 +diff --git a/pengine/test10/rec-node-9.summary b/pengine/test10/rec-node-9.summary +index 7f6d8ae..545d16c 100644 +--- a/pengine/test10/rec-node-9.summary ++++ b/pengine/test10/rec-node-9.summary +@@ -7,8 +7,8 @@ OFFLINE: [ node1 ] + rsc2 (heartbeat:apache): Stopped + + Transition Summary: +- * Start rsc1 (node2 - blocked) due to no quorum +- * Start rsc2 (node2 - blocked) due to no quorum ++ * Start rsc1 ( node2 ) due to no quorum (blocked) ++ * Start rsc2 ( node2 ) due to no quorum (blocked) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/rec-rsc-1.summary b/pengine/test10/rec-rsc-1.summary +index 13e8f60..c8f27d6 100644 +--- a/pengine/test10/rec-rsc-1.summary ++++ b/pengine/test10/rec-rsc-1.summary +@@ -5,7 +5,7 @@ Online: [ node1 node2 ] + rsc1 (heartbeat:apache): FAILED node1 + + Transition Summary: +- * Recover rsc1 (Started node1 -> node2) ++ * Recover rsc1 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/rec-rsc-2.summary b/pengine/test10/rec-rsc-2.summary +index b639599..8dd3a5a 100644 +--- a/pengine/test10/rec-rsc-2.summary ++++ b/pengine/test10/rec-rsc-2.summary +@@ -5,7 +5,7 @@ Online: [ node1 node2 ] + rsc1 (heartbeat:apache): FAILED node1 + + Transition Summary: +- * Recover rsc1 (Started node1) ++ * Recover rsc1 ( node1 ) + + Executing cluster transition: + * Resource action: rsc1 monitor on node2 +diff --git a/pengine/test10/rec-rsc-5.summary b/pengine/test10/rec-rsc-5.summary +index 28e29b5..4717750 100644 +--- a/pengine/test10/rec-rsc-5.summary ++++ b/pengine/test10/rec-rsc-5.summary +@@ -10,8 +10,8 @@ Online: [ node1 ] + Transition Summary: + * Fence (reboot) node2 'rsc1 failed there' + * Start stonith-1 (node1) +- * Recover rsc1 (Started node2 -> node1) +- * Move rsc2 (Started node2 -> node1) ++ * Recover rsc1 ( node2 -> node1 ) ++ * Move rsc2 ( node2 -> node1 ) + + Executing cluster transition: + * Resource action: stonith-1 monitor on node1 +diff --git a/pengine/test10/rec-rsc-6.summary b/pengine/test10/rec-rsc-6.summary +index c782e98..40222fc 100644 +--- a/pengine/test10/rec-rsc-6.summary ++++ b/pengine/test10/rec-rsc-6.summary +@@ -5,7 +5,7 @@ Online: [ node1 node2 ] + rsc1 (heartbeat:apache): Started [ node1 node2 ] + + Transition Summary: +- * Restart rsc1 (Started node1) ++ * Restart rsc1 ( node1 ) + + Executing cluster transition: + * Resource action: rsc1 stop on node2 +diff --git a/pengine/test10/rec-rsc-9.summary b/pengine/test10/rec-rsc-9.summary +index 02a2fe5..f7beb14 100644 +--- a/pengine/test10/rec-rsc-9.summary ++++ b/pengine/test10/rec-rsc-9.summary +@@ -9,9 +9,9 @@ Online: [ node1 ] + rsc3 (heartbeat:apache): FAILED node1 + + Transition Summary: +- * Restart rsc1 (Started node1) due to required bar running +- * Restart rsc2 (Started node1) due to required bar running +- * Recover rsc3 (Started node1) ++ * Restart rsc1 ( node1 ) due to required bar running ++ * Restart rsc2 ( node1 ) due to required bar running ++ * Recover rsc3 ( node1 ) + + Executing cluster transition: + * Resource action: rsc1 stop on node1 +diff --git a/pengine/test10/remote-fence-before-reconnect.summary b/pengine/test10/remote-fence-before-reconnect.summary +index 7a22551..3aee89d 100644 +--- a/pengine/test10/remote-fence-before-reconnect.summary ++++ b/pengine/test10/remote-fence-before-reconnect.summary +@@ -14,7 +14,7 @@ Online: [ c7auto1 c7auto2 c7auto3 ] + Transition Summary: + * Fence (reboot) c7auto4 'remote connection is unrecoverable' + * Stop c7auto4 (c7auto1) +- * Move fake2 (Started c7auto4 -> c7auto1) ++ * Move fake2 ( c7auto4 -> c7auto1 ) + + Executing cluster transition: + * Fencing c7auto4 (reboot) +diff --git a/pengine/test10/remote-fence-unclean.summary b/pengine/test10/remote-fence-unclean.summary +index 9830b96..adfe730 100644 +--- a/pengine/test10/remote-fence-unclean.summary ++++ b/pengine/test10/remote-fence-unclean.summary +@@ -12,10 +12,10 @@ Online: [ 18builder 18node1 18node2 ] + + Transition Summary: + * Fence (reboot) remote1 'remote connection is unrecoverable' +- * Recover remote1 (Started 18node1) +- * Move FAKE2 (Started remote1 -> 18builder) +- * Move FAKE3 (Started 18builder -> 18node1) +- * Move FAKE4 (Started 18node1 -> 18node2) ++ * Recover remote1 ( 18node1 ) ++ * Move FAKE2 ( remote1 -> 18builder ) ++ * Move FAKE3 ( 18builder -> 18node1 ) ++ * Move FAKE4 ( 18node1 -> 18node2 ) + + Executing cluster transition: + * Resource action: FAKE3 stop on 18builder +diff --git a/pengine/test10/remote-move.summary b/pengine/test10/remote-move.summary +index de223fd..d876a3f 100644 +--- a/pengine/test10/remote-move.summary ++++ b/pengine/test10/remote-move.summary +@@ -11,8 +11,8 @@ RemoteOnline: [ remote1 ] + FAKE4 (ocf::heartbeat:Dummy): Started 18node1 + + Transition Summary: +- * Move shooter (Started 18node1 -> 18builder) +- * Migrate remote1 (Started 18builder -> 18node1) ++ * Move shooter ( 18node1 -> 18builder ) ++ * Migrate remote1 ( 18builder -> 18node1 ) + + Executing cluster transition: + * Resource action: shooter stop on 18node1 +diff --git a/pengine/test10/remote-orphaned.summary b/pengine/test10/remote-orphaned.summary +index 2b9add7..f205007 100644 +--- a/pengine/test10/remote-orphaned.summary ++++ b/pengine/test10/remote-orphaned.summary +@@ -25,7 +25,7 @@ RemoteOnline: [ remote1 ] + remote1 (ocf::pacemaker:remote): ORPHANED Started 18node1 + + Transition Summary: +- * Move rsc_18node2 (Started remote1 -> 18node1) ++ * Move rsc_18node2 ( remote1 -> 18node1 ) + * Stop ping-1:2 (remote1) due to node availability + * Stop remote1 (18node1) due to node availability + +diff --git a/pengine/test10/remote-partial-migrate.summary b/pengine/test10/remote-partial-migrate.summary +index e8abf92..7d28db4 100644 +--- a/pengine/test10/remote-partial-migrate.summary ++++ b/pengine/test10/remote-partial-migrate.summary +@@ -62,22 +62,22 @@ RemoteOFFLINE: [ pcmk_remote5 ] + FAKE50 (ocf::heartbeat:Dummy): Stopped + + Transition Summary: +- * Migrate pcmk_remote3 (Started pcmk1 -> pcmk2) ++ * Migrate pcmk_remote3 ( pcmk1 -> pcmk2 ) + * Start FAKE4 (pcmk_remote3) +- * Move FAKE9 (Started pcmk_remote3 -> pcmk1) ++ * Move FAKE9 ( pcmk_remote3 -> pcmk1 ) + * Start FAKE10 (pcmk1) + * Start FAKE13 (pcmk2) + * Start FAKE15 (pcmk3) +- * Move FAKE16 (Started pcmk1 -> pcmk_remote3) ++ * Move FAKE16 ( pcmk1 -> pcmk_remote3 ) + * Start FAKE22 (pcmk1) +- * Move FAKE23 (Started pcmk1 -> pcmk_remote1) ++ * Move FAKE23 ( pcmk1 -> pcmk_remote1 ) + * Start FAKE26 (pcmk1) + * Start FAKE29 (pcmk2) +- * Move FAKE30 (Started pcmk1 -> pcmk_remote2) ++ * Move FAKE30 ( pcmk1 -> pcmk_remote2 ) + * Start FAKE36 (pcmk1) +- * Move FAKE37 (Started pcmk1 -> pcmk2) ++ * Move FAKE37 ( pcmk1 -> pcmk2 ) + * Start FAKE43 (pcmk1) +- * Move FAKE44 (Started pcmk1 -> pcmk2) ++ * Move FAKE44 ( pcmk1 -> pcmk2 ) + * Start FAKE50 (pcmk1) + + Executing cluster transition: +diff --git a/pengine/test10/remote-partial-migrate2.summary b/pengine/test10/remote-partial-migrate2.summary +index 17fee27..2a242bd 100644 +--- a/pengine/test10/remote-partial-migrate2.summary ++++ b/pengine/test10/remote-partial-migrate2.summary +@@ -64,24 +64,24 @@ RemoteOFFLINE: [ pcmk_remote4 ] + + Transition Summary: + * Fence (reboot) pcmk4 'peer is no longer part of the cluster' +- * Migrate pcmk_remote2 (Started pcmk3 -> pcmk1) ++ * Migrate pcmk_remote2 ( pcmk3 -> pcmk1 ) + * Start pcmk_remote4 (pcmk2) +- * Migrate pcmk_remote5 (Started pcmk1 -> pcmk2) +- * Move FAKE5 (Started pcmk1 -> pcmk_remote4) +- * Move FAKE9 (Started pcmk2 -> pcmk_remote4) +- * Move FAKE12 (Started pcmk1 -> pcmk2) +- * Move FAKE14 (Started pcmk2 -> pcmk_remote1) +- * Move FAKE17 (Started pcmk_remote1 -> pcmk_remote4) +- * Move FAKE25 (Started pcmk_remote1 -> pcmk_remote4) +- * Move FAKE28 (Started pcmk3 -> pcmk1) +- * Move FAKE30 (Started pcmk1 -> pcmk_remote1) +- * Move FAKE33 (Started pcmk_remote1 -> pcmk_remote4) +- * Move FAKE38 (Started pcmk2 -> pcmk_remote1) +- * Move FAKE39 (Started pcmk1 -> pcmk_remote2) +- * Move FAKE41 (Started pcmk_remote2 -> pcmk_remote4) +- * Move FAKE47 (Started pcmk_remote1 -> pcmk_remote2) +- * Move FAKE48 (Started pcmk1 -> pcmk_remote3) +- * Move FAKE49 (Started pcmk_remote3 -> pcmk_remote4) ++ * Migrate pcmk_remote5 ( pcmk1 -> pcmk2 ) ++ * Move FAKE5 ( pcmk1 -> pcmk_remote4 ) ++ * Move FAKE9 ( pcmk2 -> pcmk_remote4 ) ++ * Move FAKE12 ( pcmk1 -> pcmk2 ) ++ * Move FAKE14 ( pcmk2 -> pcmk_remote1 ) ++ * Move FAKE17 ( pcmk_remote1 -> pcmk_remote4 ) ++ * Move FAKE25 ( pcmk_remote1 -> pcmk_remote4 ) ++ * Move FAKE28 ( pcmk3 -> pcmk1 ) ++ * Move FAKE30 ( pcmk1 -> pcmk_remote1 ) ++ * Move FAKE33 ( pcmk_remote1 -> pcmk_remote4 ) ++ * Move FAKE38 ( pcmk2 -> pcmk_remote1 ) ++ * Move FAKE39 ( pcmk1 -> pcmk_remote2 ) ++ * Move FAKE41 ( pcmk_remote2 -> pcmk_remote4 ) ++ * Move FAKE47 ( pcmk_remote1 -> pcmk_remote2 ) ++ * Move FAKE48 ( pcmk1 -> pcmk_remote3 ) ++ * Move FAKE49 ( pcmk_remote3 -> pcmk_remote4 ) + + Executing cluster transition: + * Resource action: FAKE5 stop on pcmk1 +diff --git a/pengine/test10/remote-recover-all.summary b/pengine/test10/remote-recover-all.summary +index a8dac38..de31212 100644 +--- a/pengine/test10/remote-recover-all.summary ++++ b/pengine/test10/remote-recover-all.summary +@@ -42,18 +42,18 @@ Transition Summary: + * Fence (reboot) galera-2 'resources are active and the connection is unrecoverable' + * Fence (reboot) controller-1 'peer is no longer part of the cluster' + * Stop messaging-1 (controller-1) due to node availability +- * Move galera-0 (Started controller-1 -> controller-2) ++ * Move galera-0 ( controller-1 -> controller-2 ) + * Stop galera-2 (controller-1) due to node availability + * Stop rabbitmq:2 (messaging-1) due to node availability +- * Demote galera:1 (Master -> Stopped galera-2) +- * Stop redis:0 (controller-1) due to node availability +- * Move ip-172.17.1.14 (Started controller-1 -> controller-2) +- * Move ip-172.17.1.17 (Started controller-1 -> controller-2) +- * Move ip-172.17.4.11 (Started controller-1 -> controller-2) ++ * Stop galera:1 ( Master galera-2 ) due to node availability ++ * Stop redis:0 ( Slave controller-1 ) due to node availability ++ * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) + * Stop haproxy:0 (controller-1) due to node availability +- * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) +- * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) +- * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) ++ * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) ++ * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) ++ * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) + + Executing cluster transition: + * Pseudo action: galera-master_demote_0 +diff --git a/pengine/test10/remote-recover-connection.summary b/pengine/test10/remote-recover-connection.summary +index 7b5b5fc..8e91068 100644 +--- a/pengine/test10/remote-recover-connection.summary ++++ b/pengine/test10/remote-recover-connection.summary +@@ -39,17 +39,17 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + + Transition Summary: + * Fence (reboot) controller-1 'peer is no longer part of the cluster' +- * Move messaging-1 (Started controller-1 -> controller-2) +- * Move galera-0 (Started controller-1 -> controller-2) +- * Move galera-2 (Started controller-1 -> controller-2) +- * Stop redis:0 (controller-1) due to node availability +- * Move ip-172.17.1.14 (Started controller-1 -> controller-2) +- * Move ip-172.17.1.17 (Started controller-1 -> controller-2) +- * Move ip-172.17.4.11 (Started controller-1 -> controller-2) ++ * Move messaging-1 ( controller-1 -> controller-2 ) ++ * Move galera-0 ( controller-1 -> controller-2 ) ++ * Move galera-2 ( controller-1 -> controller-2 ) ++ * Stop redis:0 ( Slave controller-1 ) due to node availability ++ * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) + * Stop haproxy:0 (controller-1) due to node availability +- * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) +- * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) +- * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) ++ * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) ++ * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) ++ * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) + + Executing cluster transition: + * Pseudo action: redis-master_pre_notify_stop_0 +diff --git a/pengine/test10/remote-recover-fail.summary b/pengine/test10/remote-recover-fail.summary +index 302b716..d4e1aab 100644 +--- a/pengine/test10/remote-recover-fail.summary ++++ b/pengine/test10/remote-recover-fail.summary +@@ -15,10 +15,10 @@ OFFLINE: [ rhel7-auto1 ] + + Transition Summary: + * Fence (reboot) rhel7-auto4 'FAKE2 is thought to be active there' +- * Recover rhel7-auto4 (Started rhel7-auto2) ++ * Recover rhel7-auto4 ( rhel7-auto2 ) + * Start FAKE1 (rhel7-auto2) +- * Move FAKE2 (Started rhel7-auto4 -> rhel7-auto3) +- * Move FAKE6 (Started rhel7-auto4 -> rhel7-auto2) ++ * Move FAKE2 ( rhel7-auto4 -> rhel7-auto3 ) ++ * Move FAKE6 ( rhel7-auto4 -> rhel7-auto2 ) + + Executing cluster transition: + * Resource action: FAKE3 monitor=10000 on rhel7-auto2 +diff --git a/pengine/test10/remote-recover-no-resources.summary b/pengine/test10/remote-recover-no-resources.summary +index 4383e81..c05e355 100644 +--- a/pengine/test10/remote-recover-no-resources.summary ++++ b/pengine/test10/remote-recover-no-resources.summary +@@ -41,17 +41,17 @@ Transition Summary: + * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' + * Fence (reboot) controller-1 'peer is no longer part of the cluster' + * Stop messaging-1 (controller-1) due to node availability +- * Move galera-0 (Started controller-1 -> controller-2) ++ * Move galera-0 ( controller-1 -> controller-2 ) + * Stop galera-2 (controller-1) + * Stop rabbitmq:2 (messaging-1) due to node availability +- * Stop redis:0 (controller-1) due to node availability +- * Move ip-172.17.1.14 (Started controller-1 -> controller-2) +- * Move ip-172.17.1.17 (Started controller-1 -> controller-2) +- * Move ip-172.17.4.11 (Started controller-1 -> controller-2) ++ * Stop redis:0 ( Slave controller-1 ) due to node availability ++ * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) + * Stop haproxy:0 (controller-1) due to node availability +- * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) +- * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) +- * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) ++ * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) ++ * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) ++ * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) + + Executing cluster transition: + * Pseudo action: redis-master_pre_notify_stop_0 +diff --git a/pengine/test10/remote-recover-unknown.summary b/pengine/test10/remote-recover-unknown.summary +index 3ffb5f4..bfbe7d3 100644 +--- a/pengine/test10/remote-recover-unknown.summary ++++ b/pengine/test10/remote-recover-unknown.summary +@@ -42,17 +42,17 @@ Transition Summary: + * Fence (reboot) messaging-1 'resources are active and the connection is unrecoverable' + * Fence (reboot) controller-1 'peer is no longer part of the cluster' + * Stop messaging-1 (controller-1) due to node availability +- * Move galera-0 (Started controller-1 -> controller-2) ++ * Move galera-0 ( controller-1 -> controller-2 ) + * Stop galera-2 (controller-1) due to node availability + * Stop rabbitmq:2 (messaging-1) due to node availability +- * Stop redis:0 (controller-1) due to node availability +- * Move ip-172.17.1.14 (Started controller-1 -> controller-2) +- * Move ip-172.17.1.17 (Started controller-1 -> controller-2) +- * Move ip-172.17.4.11 (Started controller-1 -> controller-2) ++ * Stop redis:0 ( Slave controller-1 ) due to node availability ++ * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) + * Stop haproxy:0 (controller-1) due to node availability +- * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) +- * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) +- * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) ++ * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) ++ * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) ++ * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) + + Executing cluster transition: + * Pseudo action: redis-master_pre_notify_stop_0 +diff --git a/pengine/test10/remote-recovery.summary b/pengine/test10/remote-recovery.summary +index 7b5b5fc..8e91068 100644 +--- a/pengine/test10/remote-recovery.summary ++++ b/pengine/test10/remote-recovery.summary +@@ -39,17 +39,17 @@ RemoteOnline: [ galera-0 galera-1 galera-2 messaging-0 messaging-1 messaging-2 ] + + Transition Summary: + * Fence (reboot) controller-1 'peer is no longer part of the cluster' +- * Move messaging-1 (Started controller-1 -> controller-2) +- * Move galera-0 (Started controller-1 -> controller-2) +- * Move galera-2 (Started controller-1 -> controller-2) +- * Stop redis:0 (controller-1) due to node availability +- * Move ip-172.17.1.14 (Started controller-1 -> controller-2) +- * Move ip-172.17.1.17 (Started controller-1 -> controller-2) +- * Move ip-172.17.4.11 (Started controller-1 -> controller-2) ++ * Move messaging-1 ( controller-1 -> controller-2 ) ++ * Move galera-0 ( controller-1 -> controller-2 ) ++ * Move galera-2 ( controller-1 -> controller-2 ) ++ * Stop redis:0 ( Slave controller-1 ) due to node availability ++ * Move ip-172.17.1.14 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.1.17 ( controller-1 -> controller-2 ) ++ * Move ip-172.17.4.11 ( controller-1 -> controller-2 ) + * Stop haproxy:0 (controller-1) due to node availability +- * Restart stonith-fence_ipmilan-525400bbf613 (Started controller-0) +- * Restart stonith-fence_ipmilan-525400b4f6bd (Started controller-0) +- * Move stonith-fence_ipmilan-5254005bdbb5 (Started controller-1 -> controller-2) ++ * Restart stonith-fence_ipmilan-525400bbf613 ( controller-0 ) ++ * Restart stonith-fence_ipmilan-525400b4f6bd ( controller-0 ) ++ * Move stonith-fence_ipmilan-5254005bdbb5 ( controller-1 -> controller-2 ) + + Executing cluster transition: + * Pseudo action: redis-master_pre_notify_stop_0 +diff --git a/pengine/test10/remote-start-fail.summary b/pengine/test10/remote-start-fail.summary +index 4f64c75..aa08600 100644 +--- a/pengine/test10/remote-start-fail.summary ++++ b/pengine/test10/remote-start-fail.summary +@@ -7,7 +7,7 @@ RemoteOFFLINE: [ rhel7-auto4 ] + rhel7-auto4 (ocf::pacemaker:remote): FAILED rhel7-auto2 + + Transition Summary: +- * Recover rhel7-auto4 (Started rhel7-auto2 -> rhel7-auto3) ++ * Recover rhel7-auto4 ( rhel7-auto2 -> rhel7-auto3 ) + + Executing cluster transition: + * Resource action: rhel7-auto4 stop on rhel7-auto2 +diff --git a/pengine/test10/remote-startup-probes.summary b/pengine/test10/remote-startup-probes.summary +index d7a382e..f47bf47 100644 +--- a/pengine/test10/remote-startup-probes.summary ++++ b/pengine/test10/remote-startup-probes.summary +@@ -12,8 +12,8 @@ RemoteOFFLINE: [ remote1 ] + + Transition Summary: + * Start remote1 (18builder) +- * Move FAKE1 (Started 18builder -> 18node2) +- * Move FAKE2 (Started 18node2 -> remote1) ++ * Move FAKE1 ( 18builder -> 18node2 ) ++ * Move FAKE2 ( 18node2 -> remote1 ) + + Executing cluster transition: + * Resource action: remote1 start on 18builder +diff --git a/pengine/test10/remote-startup.summary b/pengine/test10/remote-startup.summary +index 56c9a95..4eb6887 100644 +--- a/pengine/test10/remote-startup.summary ++++ b/pengine/test10/remote-startup.summary +@@ -8,7 +8,7 @@ Online: [ 18builder 18node1 18node2 ] + remote1 (ocf::pacemaker:remote): Stopped + + Transition Summary: +- * Move shooter (Started 18builder -> 18node1) ++ * Move shooter ( 18builder -> 18node1 ) + * Start fake (18node2) + * Start remote1 (18builder) + +diff --git a/pengine/test10/remote-unclean2.summary b/pengine/test10/remote-unclean2.summary +index 1408c09..e1666b5 100644 +--- a/pengine/test10/remote-unclean2.summary ++++ b/pengine/test10/remote-unclean2.summary +@@ -8,7 +8,7 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ] + + Transition Summary: + * Fence (reboot) rhel7-auto4 'remote connection is unrecoverable' +- * Recover rhel7-auto4 (Started rhel7-auto1) ++ * Recover rhel7-auto4 ( rhel7-auto1 ) + + Executing cluster transition: + * Resource action: rhel7-auto4 stop on rhel7-auto1 +diff --git a/pengine/test10/rsc-discovery-per-node.summary b/pengine/test10/rsc-discovery-per-node.summary +index ea82c3e..13dff85 100644 +--- a/pengine/test10/rsc-discovery-per-node.summary ++++ b/pengine/test10/rsc-discovery-per-node.summary +@@ -18,9 +18,9 @@ RemoteOFFLINE: [ remote1 ] + Transition Summary: + * Start remote1 (18builder) + * Start FAKE1 (18node2) +- * Move FAKE2 (Started 18node2 -> 18node3) +- * Move FAKE3 (Started 18builder -> 18node4) +- * Move FAKE4 (Started 18node1 -> remote1) ++ * Move FAKE2 ( 18node2 -> 18node3 ) ++ * Move FAKE3 ( 18builder -> 18node4 ) ++ * Move FAKE4 ( 18node1 -> remote1 ) + * Start FAKE5 (18builder) + * Start FAKECLONE1:0 (18node1) + * Start FAKECLONE1:1 (18node2) +diff --git a/pengine/test10/rsc-sets-clone-1.summary b/pengine/test10/rsc-sets-clone-1.summary +index afafc55..211ad97 100644 +--- a/pengine/test10/rsc-sets-clone-1.summary ++++ b/pengine/test10/rsc-sets-clone-1.summary +@@ -16,7 +16,7 @@ Online: [ sys2 sys3 ] + Stopped (disabled): [ sys2 sys3 ] + + Transition Summary: +- * Restart stonithsys3 (Started sys2) ++ * Restart stonithsys3 ( sys2 ) + * Start controld:1 (sys3) + * Start clvmd:1 (sys3) + * Start o2cb:1 (sys3) +diff --git a/pengine/test10/rsc-sets-clone.summary b/pengine/test10/rsc-sets-clone.summary +index 70570e6..43f9e0f 100644 +--- a/pengine/test10/rsc-sets-clone.summary ++++ b/pengine/test10/rsc-sets-clone.summary +@@ -10,8 +10,8 @@ Online: [ node2 ] + Started: [ node1 node2 ] + + Transition Summary: +- * Move rsc1 (Started node1 -> node2) +- * Move rsc3 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc3 ( node1 -> node2 ) + * Stop rsc:0 (node1) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/rsc-sets-master.summary b/pengine/test10/rsc-sets-master.summary +index 5415cda..126edc7 100644 +--- a/pengine/test10/rsc-sets-master.summary ++++ b/pengine/test10/rsc-sets-master.summary +@@ -11,11 +11,11 @@ Online: [ node2 ] + rsc3 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Demote rsc:0 (Master -> Stopped node1) ++ * Stop rsc:0 ( Master node1 ) due to node availability + * Promote rsc:1 (Slave -> Master node2) +- * Move rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) +- * Move rsc3 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) ++ * Move rsc3 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc1 stop on node1 +diff --git a/pengine/test10/rsc-sets-seq-false.summary b/pengine/test10/rsc-sets-seq-false.summary +index 2d6dccf..5774f69 100644 +--- a/pengine/test10/rsc-sets-seq-false.summary ++++ b/pengine/test10/rsc-sets-seq-false.summary +@@ -11,12 +11,12 @@ Online: [ node2 ] + rsc6 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Move rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) +- * Move rsc3 (Started node1 -> node2) +- * Move rsc4 (Started node1 -> node2) +- * Move rsc5 (Started node1 -> node2) +- * Move rsc6 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) ++ * Move rsc3 ( node1 -> node2 ) ++ * Move rsc4 ( node1 -> node2 ) ++ * Move rsc5 ( node1 -> node2 ) ++ * Move rsc6 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc4 stop on node1 +diff --git a/pengine/test10/rsc-sets-seq-true.summary b/pengine/test10/rsc-sets-seq-true.summary +index e4fca51..aba86c6 100644 +--- a/pengine/test10/rsc-sets-seq-true.summary ++++ b/pengine/test10/rsc-sets-seq-true.summary +@@ -11,12 +11,12 @@ Online: [ node2 ] + rsc6 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Move rsc1 (Started node1 -> node2) +- * Move rsc2 (Started node1 -> node2) +- * Move rsc3 (Started node1 -> node2) +- * Move rsc4 (Started node1 -> node2) +- * Move rsc5 (Started node1 -> node2) +- * Move rsc6 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) ++ * Move rsc2 ( node1 -> node2 ) ++ * Move rsc3 ( node1 -> node2 ) ++ * Move rsc4 ( node1 -> node2 ) ++ * Move rsc5 ( node1 -> node2 ) ++ * Move rsc6 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc6 stop on node1 +diff --git a/pengine/test10/rsc_dep4.summary b/pengine/test10/rsc_dep4.summary +index 2fe21cf..b911b46 100644 +--- a/pengine/test10/rsc_dep4.summary ++++ b/pengine/test10/rsc_dep4.summary +@@ -9,7 +9,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Start rsc2 (node1) +- * Move rsc4 (Started node1 -> node2) ++ * Move rsc4 ( node1 -> node2 ) + * Start rsc3 (node2) + + Executing cluster transition: +diff --git a/pengine/test10/standby.summary b/pengine/test10/standby.summary +index 89534cb..f4d53a1 100644 +--- a/pengine/test10/standby.summary ++++ b/pengine/test10/standby.summary +@@ -20,14 +20,14 @@ Online: [ sapcl01 ] + oralsnr_25 (ocf::heartbeat:oralsnr): Started sapcl03 + + Transition Summary: +- * Move IPaddr_192_168_1_102 (Started sapcl02 -> sapcl01) +- * Move LVM_12 (Started sapcl02 -> sapcl01) +- * Move Filesystem_13 (Started sapcl02 -> sapcl01) +- * Move IPaddr_192_168_1_104 (Started sapcl03 -> sapcl01) +- * Move LVM_22 (Started sapcl03 -> sapcl01) +- * Move Filesystem_23 (Started sapcl03 -> sapcl01) +- * Move oracle_24 (Started sapcl03 -> sapcl01) +- * Move oralsnr_25 (Started sapcl03 -> sapcl01) ++ * Move IPaddr_192_168_1_102 ( sapcl02 -> sapcl01 ) ++ * Move LVM_12 ( sapcl02 -> sapcl01 ) ++ * Move Filesystem_13 ( sapcl02 -> sapcl01 ) ++ * Move IPaddr_192_168_1_104 ( sapcl03 -> sapcl01 ) ++ * Move LVM_22 ( sapcl03 -> sapcl01 ) ++ * Move Filesystem_23 ( sapcl03 -> sapcl01 ) ++ * Move oracle_24 ( sapcl03 -> sapcl01 ) ++ * Move oralsnr_25 ( sapcl03 -> sapcl01 ) + + Executing cluster transition: + * Pseudo action: app02_stop_0 +diff --git a/pengine/test10/start-then-stop-with-unfence.summary b/pengine/test10/start-then-stop-with-unfence.summary +index 5e127f7..2e02a21 100644 +--- a/pengine/test10/start-then-stop-with-unfence.summary ++++ b/pengine/test10/start-then-stop-with-unfence.summary +@@ -13,7 +13,7 @@ Online: [ rhel7-node1.example.com rhel7-node2.example.com ] + Transition Summary: + * Fence (on) rhel7-node1.example.com 'Required by ip1' + * Start mpath-node1 (rhel7-node1.example.com) +- * Move ip1 (Started rhel7-node2.example.com -> rhel7-node1.example.com) ++ * Move ip1 ( rhel7-node2.example.com -> rhel7-node1.example.com ) + * Start jrummy:1 (rhel7-node1.example.com) + + Executing cluster transition: +diff --git a/pengine/test10/stonith-0.summary b/pengine/test10/stonith-0.summary +index 1062520..78f0bbc 100644 +--- a/pengine/test10/stonith-0.summary ++++ b/pengine/test10/stonith-0.summary +@@ -39,11 +39,11 @@ Online: [ c001n02 c001n04 c001n06 c001n07 c001n08 ] + Transition Summary: + * Fence (reboot) c001n05 'ocf_192.168.100.183 failed there' + * Fence (reboot) c001n03 'ocf_192.168.100.183 failed there' +- * Move ocf_192.168.100.181 (Started c001n03 -> c001n02) +- * Move heartbeat_192.168.100.182 (Started c001n03 -> c001n02) +- * Recover ocf_192.168.100.183 (Started c001n03 -> c001n02) +- * Move rsc_c001n05 (Started c001n05 -> c001n07) +- * Move rsc_c001n07 (Started c001n03 -> c001n07) ++ * Move ocf_192.168.100.181 ( c001n03 -> c001n02 ) ++ * Move heartbeat_192.168.100.182 ( c001n03 -> c001n02 ) ++ * Recover ocf_192.168.100.183 ( c001n03 -> c001n02 ) ++ * Move rsc_c001n05 ( c001n05 -> c001n07 ) ++ * Move rsc_c001n07 ( c001n03 -> c001n07 ) + + Executing cluster transition: + * Resource action: child_DoFencing:4 monitor=20000 on c001n08 +diff --git a/pengine/test10/stonith-1.summary b/pengine/test10/stonith-1.summary +index 03c8a39..0e9c2a1 100644 +--- a/pengine/test10/stonith-1.summary ++++ b/pengine/test10/stonith-1.summary +@@ -30,15 +30,15 @@ Online: [ sles-1 sles-2 sles-4 ] + Transition Summary: + * Fence (reboot) sles-3 'peer is no longer part of the cluster' + * Start r192.168.100.183 (sles-1) +- * Move migrator (Started sles-3 -> sles-4) +- * Move rsc_sles-3 (Started sles-3 -> sles-4) +- * Move child_DoFencing:2 (Started sles-3 -> sles-4) ++ * Move migrator ( sles-3 -> sles-4 ) ++ * Move rsc_sles-3 ( sles-3 -> sles-4 ) ++ * Move child_DoFencing:2 ( sles-3 -> sles-4 ) + * Start ocf_msdummy:0 (sles-4) + * Start ocf_msdummy:1 (sles-1) +- * Move ocf_msdummy:2 (Slave sles-3 -> sles-2) ++ * Move ocf_msdummy:2 ( sles-3 -> sles-2 Slave ) + * Start ocf_msdummy:3 (sles-4) + * Start ocf_msdummy:4 (sles-1) +- * Move ocf_msdummy:5 (Slave sles-3 -> sles-2) ++ * Move ocf_msdummy:5 ( sles-3 -> sles-2 Slave ) + + Executing cluster transition: + * Pseudo action: group-1_start_0 +diff --git a/pengine/test10/stonith-4.summary b/pengine/test10/stonith-4.summary +index 76b1f44..7502dad 100644 +--- a/pengine/test10/stonith-4.summary ++++ b/pengine/test10/stonith-4.summary +@@ -18,7 +18,7 @@ Transition Summary: + * Fence (reboot) pcmk-8 'peer has not been seen by the cluster' + * Fence (reboot) pcmk-7 'peer failed the pacemaker membership criteria' + * Fence (reboot) pcmk-5 'peer has not been seen by the cluster' +- * Start Fencing (pcmk-1 - blocked) ++ * Start Fencing ( pcmk-1 ) blocked + + Executing cluster transition: + * Fencing pcmk-10 (reboot) +diff --git a/pengine/test10/stop-failure-no-quorum.summary b/pengine/test10/stop-failure-no-quorum.summary +index 75945b1..52ec84f 100644 +--- a/pengine/test10/stop-failure-no-quorum.summary ++++ b/pengine/test10/stop-failure-no-quorum.summary +@@ -16,11 +16,11 @@ Online: [ pcmk-1 ] + + Transition Summary: + * Fence (reboot) pcmk-2 'clvm:0 failed there' +- * Start dlm:0 (pcmk-1 - blocked) due to no quorum ++ * Start dlm:0 ( pcmk-1 ) due to no quorum (blocked) + * Stop clvm:0 (pcmk-2) due to node availability +- * Start clvm:2 (pcmk-1 - blocked) due to no quorum +- * Start ClusterIP (pcmk-1 - blocked) due to no quorum +- * Start Fencing (pcmk-1 - blocked) due to no quorum ++ * Start clvm:2 ( pcmk-1 ) due to no quorum (blocked) ++ * Start ClusterIP ( pcmk-1 ) due to no quorum (blocked) ++ * Start Fencing ( pcmk-1 ) due to no quorum (blocked) + + Executing cluster transition: + * Fencing pcmk-2 (reboot) +diff --git a/pengine/test10/stop-failure-with-fencing.summary b/pengine/test10/stop-failure-with-fencing.summary +index 09680c8..50c5f97 100644 +--- a/pengine/test10/stop-failure-with-fencing.summary ++++ b/pengine/test10/stop-failure-with-fencing.summary +@@ -15,11 +15,11 @@ Online: [ pcmk-1 ] + + Transition Summary: + * Fence (reboot) pcmk-2 'clvm:0 failed there' +- * Start dlm:0 (pcmk-1 - blocked) due to no quorum ++ * Start dlm:0 ( pcmk-1 ) due to no quorum (blocked) + * Stop clvm:0 (pcmk-2) due to node availability +- * Start clvm:1 (pcmk-1 - blocked) due to no quorum +- * Start ClusterIP (pcmk-1 - blocked) due to no quorum +- * Start Fencing (pcmk-1 - blocked) due to no quorum ++ * Start clvm:1 ( pcmk-1 ) due to no quorum (blocked) ++ * Start ClusterIP ( pcmk-1 ) due to no quorum (blocked) ++ * Start Fencing ( pcmk-1 ) due to no quorum (blocked) + + Executing cluster transition: + * Resource action: Fencing monitor on pcmk-1 +diff --git a/pengine/test10/stopped-monitor-01.summary b/pengine/test10/stopped-monitor-01.summary +index 43cc177..7baf2b9 100644 +--- a/pengine/test10/stopped-monitor-01.summary ++++ b/pengine/test10/stopped-monitor-01.summary +@@ -5,7 +5,7 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): FAILED node1 + + Transition Summary: +- * Recover rsc1 (Started node1) ++ * Recover rsc1 ( node1 ) + + Executing cluster transition: + * Resource action: rsc1 stop on node1 +diff --git a/pengine/test10/stopped-monitor-02.summary b/pengine/test10/stopped-monitor-02.summary +index 6d97bfb..e1d44b5 100644 +--- a/pengine/test10/stopped-monitor-02.summary ++++ b/pengine/test10/stopped-monitor-02.summary +@@ -5,7 +5,7 @@ Online: [ node1 node2 ] + rsc1 (ocf::pacemaker:Dummy): FAILED [ node1 node2 ] + + Transition Summary: +- * Recover rsc1 (Started node1) ++ * Recover rsc1 ( node1 ) + + Executing cluster transition: + * Resource action: rsc1 stop on node1 +diff --git a/pengine/test10/stopped-monitor-08.summary b/pengine/test10/stopped-monitor-08.summary +index 6646859..a15a906 100644 +--- a/pengine/test10/stopped-monitor-08.summary ++++ b/pengine/test10/stopped-monitor-08.summary +@@ -6,7 +6,7 @@ Online: [ node2 ] + rsc1 (ocf::pacemaker:Dummy): Started node1 + + Transition Summary: +- * Move rsc1 (Started node1 -> node2) ++ * Move rsc1 ( node1 -> node2 ) + + Executing cluster transition: + * Resource action: rsc1 stop on node1 +diff --git a/pengine/test10/ticket-master-14.summary b/pengine/test10/ticket-master-14.summary +index 9a42d78..fa14935 100644 +--- a/pengine/test10/ticket-master-14.summary ++++ b/pengine/test10/ticket-master-14.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + Slaves: [ node2 ] + + Transition Summary: +- * Demote rsc1:0 (Master -> Stopped node1) +- * Stop rsc1:1 (node2) due to node availability ++ * Stop rsc1:0 ( Master node1 ) due to node availability ++ * Stop rsc1:1 ( Slave node2 ) due to node availability + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +diff --git a/pengine/test10/ticket-master-15.summary b/pengine/test10/ticket-master-15.summary +index 9a42d78..fa14935 100644 +--- a/pengine/test10/ticket-master-15.summary ++++ b/pengine/test10/ticket-master-15.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + Slaves: [ node2 ] + + Transition Summary: +- * Demote rsc1:0 (Master -> Stopped node1) +- * Stop rsc1:1 (node2) due to node availability ++ * Stop rsc1:0 ( Master node1 ) due to node availability ++ * Stop rsc1:1 ( Slave node2 ) due to node availability + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +diff --git a/pengine/test10/ticket-master-2.summary b/pengine/test10/ticket-master-2.summary +index 96a797e..6f5be53 100644 +--- a/pengine/test10/ticket-master-2.summary ++++ b/pengine/test10/ticket-master-2.summary +@@ -8,7 +8,6 @@ Online: [ node1 node2 ] + + Transition Summary: + * Start rsc1:0 (node2) +- * Start rsc1:1 (node1) + * Promote rsc1:1 (Stopped -> Master node1) + + Executing cluster transition: +diff --git a/pengine/test10/ticket-master-21.summary b/pengine/test10/ticket-master-21.summary +index 8fc50eb..88f62fd 100644 +--- a/pengine/test10/ticket-master-21.summary ++++ b/pengine/test10/ticket-master-21.summary +@@ -9,8 +9,8 @@ Online: [ node1 node2 ] + + Transition Summary: + * Fence (reboot) node1 'deadman ticket was lost' +- * Move rsc_stonith (Started node1 -> node2) +- * Demote rsc1:0 (Master -> Stopped node1) ++ * Move rsc_stonith ( node1 -> node2 ) ++ * Stop rsc1:0 ( Master node1 ) due to node availability + + Executing cluster transition: + * Pseudo action: rsc_stonith_stop_0 +diff --git a/pengine/test10/ticket-master-3.summary b/pengine/test10/ticket-master-3.summary +index 9a42d78..fa14935 100644 +--- a/pengine/test10/ticket-master-3.summary ++++ b/pengine/test10/ticket-master-3.summary +@@ -8,8 +8,8 @@ Online: [ node1 node2 ] + Slaves: [ node2 ] + + Transition Summary: +- * Demote rsc1:0 (Master -> Stopped node1) +- * Stop rsc1:1 (node2) due to node availability ++ * Stop rsc1:0 ( Master node1 ) due to node availability ++ * Stop rsc1:1 ( Slave node2 ) due to node availability + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +diff --git a/pengine/test10/ticket-master-9.summary b/pengine/test10/ticket-master-9.summary +index 8fc50eb..88f62fd 100644 +--- a/pengine/test10/ticket-master-9.summary ++++ b/pengine/test10/ticket-master-9.summary +@@ -9,8 +9,8 @@ Online: [ node1 node2 ] + + Transition Summary: + * Fence (reboot) node1 'deadman ticket was lost' +- * Move rsc_stonith (Started node1 -> node2) +- * Demote rsc1:0 (Master -> Stopped node1) ++ * Move rsc_stonith ( node1 -> node2 ) ++ * Stop rsc1:0 ( Master node1 ) due to node availability + + Executing cluster transition: + * Pseudo action: rsc_stonith_stop_0 +diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary +index f728e39..4ca9344 100644 +--- a/pengine/test10/unfence-definition.summary ++++ b/pengine/test10/unfence-definition.summary +@@ -15,10 +15,10 @@ Transition Summary: + * Fence (reboot) virt-4 'node is unclean' + * Fence (on) virt-3 'Required by dlm:2' + * Fence (on) virt-1 'Device definition changed' +- * Restart fencing (Started virt-1) +- * Restart dlm:0 (Started virt-1) due to required stonith ++ * Restart fencing ( virt-1 ) ++ * Restart dlm:0 ( virt-1 ) due to required stonith + * Start dlm:2 (virt-3) +- * Restart clvmd:0 (Started virt-1) due to required stonith ++ * Restart clvmd:0 ( virt-1 ) due to required stonith + * Start clvmd:1 (virt-2) + * Start clvmd:2 (virt-3) + +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +index e8b15ec..8a6356c 100644 +--- a/pengine/test10/unfence-parameters.summary ++++ b/pengine/test10/unfence-parameters.summary +@@ -16,11 +16,11 @@ Transition Summary: + * Fence (on) virt-3 'Device parameters changed (reload)' + * Fence (on) virt-2 'Device parameters changed (reload)' + * Fence (on) virt-1 'Device parameters changed (reload)' +- * Restart fencing (Started virt-1) +- * Restart dlm:0 (Started virt-1) due to required stonith +- * Restart dlm:1 (Started virt-2) due to required stonith ++ * Restart fencing ( virt-1 ) ++ * Restart dlm:0 ( virt-1 ) due to required stonith ++ * Restart dlm:1 ( virt-2 ) due to required stonith + * Start dlm:2 (virt-3) +- * Restart clvmd:0 (Started virt-1) due to required stonith ++ * Restart clvmd:0 ( virt-1 ) due to required stonith + * Start clvmd:1 (virt-2) + * Start clvmd:2 (virt-3) + +diff --git a/pengine/test10/unmanaged-block-restart.summary b/pengine/test10/unmanaged-block-restart.summary +index 01dffeb..d7725c9 100644 +--- a/pengine/test10/unmanaged-block-restart.summary ++++ b/pengine/test10/unmanaged-block-restart.summary +@@ -10,8 +10,8 @@ Online: [ yingying.site ] + + Transition Summary: + * Start rsc1 (yingying.site) +- * Stop rsc2 (Started yingying.site - blocked) due to required rsc1 start +- * Stop rsc3 (Started yingying.site - blocked) due to required rsc2 start ++ * Stop rsc2 ( yingying.site ) due to required rsc1 start (blocked) ++ * Stop rsc3 ( yingying.site ) due to required rsc2 start (blocked) + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/unmanaged-stop-1.summary b/pengine/test10/unmanaged-stop-1.summary +index 94e0908..b02c7dc 100644 +--- a/pengine/test10/unmanaged-stop-1.summary ++++ b/pengine/test10/unmanaged-stop-1.summary +@@ -7,7 +7,7 @@ Online: [ yingying.site ] + rsc2 (ocf::pacemaker:Dummy): FAILED yingying.site ( blocked ) + + Transition Summary: +- * Stop rsc1 (yingying.site - blocked) due to node availability ++ * Stop rsc1 ( yingying.site ) due to node availability (blocked) + + Executing cluster transition: + +diff --git a/pengine/test10/unmanaged-stop-2.summary b/pengine/test10/unmanaged-stop-2.summary +index 94e0908..b02c7dc 100644 +--- a/pengine/test10/unmanaged-stop-2.summary ++++ b/pengine/test10/unmanaged-stop-2.summary +@@ -7,7 +7,7 @@ Online: [ yingying.site ] + rsc2 (ocf::pacemaker:Dummy): FAILED yingying.site ( blocked ) + + Transition Summary: +- * Stop rsc1 (yingying.site - blocked) due to node availability ++ * Stop rsc1 ( yingying.site ) due to node availability (blocked) + + Executing cluster transition: + +diff --git a/pengine/test10/unmanaged-stop-3.summary b/pengine/test10/unmanaged-stop-3.summary +index c72d451..d457157 100644 +--- a/pengine/test10/unmanaged-stop-3.summary ++++ b/pengine/test10/unmanaged-stop-3.summary +@@ -8,7 +8,7 @@ Online: [ yingying.site ] + rsc2 (ocf::pacemaker:Dummy): FAILED yingying.site ( disabled, blocked ) + + Transition Summary: +- * Stop rsc1 (yingying.site - blocked) unrunnable rsc2 stop due to node availability ++ * Stop rsc1 ( yingying.site ) due to node availability (blocked) + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/unmanaged-stop-4.summary b/pengine/test10/unmanaged-stop-4.summary +index 2704e98..1ddd68b 100644 +--- a/pengine/test10/unmanaged-stop-4.summary ++++ b/pengine/test10/unmanaged-stop-4.summary +@@ -9,7 +9,7 @@ Online: [ yingying.site ] + rsc3 (ocf::heartbeat:Dummy): Stopped ( disabled ) + + Transition Summary: +- * Stop rsc1 (yingying.site - blocked) unrunnable rsc2 stop due to node availability ++ * Stop rsc1 ( yingying.site ) due to node availability (blocked) + + Executing cluster transition: + * Pseudo action: group1_stop_0 +diff --git a/pengine/test10/unrunnable-1.summary b/pengine/test10/unrunnable-1.summary +index 8c27e45..54ca0d7 100644 +--- a/pengine/test10/unrunnable-1.summary ++++ b/pengine/test10/unrunnable-1.summary +@@ -19,15 +19,15 @@ Online: [ c001n03 ] + child_DoFencing:3 (stonith:ssh): Stopped + + Transition Summary: +- * Start DcIPaddr (c001n03 - blocked) due to no quorum +- * Start child_192.168.100.181 (c001n03 - blocked) due to no quorum +- * Start child_192.168.100.182 (c001n03 - blocked) due to no quorum +- * Start child_192.168.100.183 (c001n03 - blocked) due to no quorum +- * Start rsc_c001n08 (c001n03 - blocked) due to no quorum +- * Start rsc_c001n02 (c001n03 - blocked) due to no quorum +- * Start rsc_c001n03 (c001n03 - blocked) due to no quorum +- * Start rsc_c001n01 (c001n03 - blocked) due to no quorum +- * Stop child_DoFencing:1 (c001n02 - blocked) due to node availability ++ * Start DcIPaddr ( c001n03 ) due to no quorum (blocked) ++ * Start child_192.168.100.181 ( c001n03 ) due to no quorum (blocked) ++ * Start child_192.168.100.182 ( c001n03 ) due to no quorum (blocked) ++ * Start child_192.168.100.183 ( c001n03 ) due to no quorum (blocked) ++ * Start rsc_c001n08 ( c001n03 ) due to no quorum (blocked) ++ * Start rsc_c001n02 ( c001n03 ) due to no quorum (blocked) ++ * Start rsc_c001n03 ( c001n03 ) due to no quorum (blocked) ++ * Start rsc_c001n01 ( c001n03 ) due to no quorum (blocked) ++ * Stop child_DoFencing:1 ( c001n02 ) due to node availability (blocked) + + Executing cluster transition: + * Resource action: DcIPaddr monitor on c001n03 +diff --git a/pengine/test10/unrunnable-2.summary b/pengine/test10/unrunnable-2.summary +index b67f7cb..4bbacec 100644 +--- a/pengine/test10/unrunnable-2.summary ++++ b/pengine/test10/unrunnable-2.summary +@@ -85,7 +85,7 @@ Online: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] + Stopped: [ overcloud-controller-0 overcloud-controller-1 overcloud-controller-2 ] + + Transition Summary: +- * Start openstack-cinder-volume (overcloud-controller-2 - blocked) due to unrunnable openstack-cinder-scheduler-clone running ++ * Start openstack-cinder-volume ( overcloud-controller-2 ) due to unrunnable openstack-cinder-scheduler-clone running (blocked) + + Executing cluster transition: + +diff --git a/pengine/test10/utilization-order2.summary b/pengine/test10/utilization-order2.summary +index db65a16..ad3fd38 100644 +--- a/pengine/test10/utilization-order2.summary ++++ b/pengine/test10/utilization-order2.summary +@@ -10,7 +10,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Start rsc4 (node1) +- * Move rsc3 (Started node1 -> node2) ++ * Move rsc3 ( node1 -> node2 ) + * Stop rsc2:0 (node1) due to node availability + * Stop rsc1 (node2) + +diff --git a/pengine/test10/utilization-order3.summary b/pengine/test10/utilization-order3.summary +index cfeea48..9d59f37 100644 +--- a/pengine/test10/utilization-order3.summary ++++ b/pengine/test10/utilization-order3.summary +@@ -7,7 +7,7 @@ Online: [ node1 node2 ] + + Transition Summary: + * Start rsc2 (node1) +- * Migrate rsc1 (Started node1 -> node2) ++ * Migrate rsc1 ( node1 -> node2 ) + + Executing cluster transition: + * Pseudo action: load_stopped_node2 +diff --git a/pengine/test10/utilization-order4.summary b/pengine/test10/utilization-order4.summary +index 09a8c31..04c5f93 100644 +--- a/pengine/test10/utilization-order4.summary ++++ b/pengine/test10/utilization-order4.summary +@@ -15,7 +15,7 @@ Online: [ deglxen001 ] + Started: [ deglxen001 deglxen002 ] + + Transition Summary: +- * Migrate degllx62-vm (Started deglxen002 -> deglxen001) ++ * Migrate degllx62-vm ( deglxen002 -> deglxen001 ) + * Stop degllx61-vm (deglxen001) + * Stop nfs-xen_config:1 (deglxen002) due to node availability + * Stop nfs-xen_swapfiles:1 (deglxen002) due to node availability +diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary +index 1d06552..1271d4f 100644 +--- a/pengine/test10/whitebox-fail1.summary ++++ b/pengine/test10/whitebox-fail1.summary +@@ -15,10 +15,10 @@ Containers: [ lxc2:container2 ] + + Transition Summary: + * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' +- * Recover container1 (Started 18node2) +- * Recover M:4 (Started lxc1) +- * Recover B (Started lxc1) +- * Restart lxc1 (Started 18node2) due to required container1 start ++ * Recover container1 ( 18node2 ) ++ * Recover M:4 ( lxc1 ) ++ * Recover B ( lxc1 ) ++ * Restart lxc1 ( 18node2 ) due to required container1 start + + Executing cluster transition: + * Resource action: lxc1 stop on 18node2 +diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary +index 06364fb..5fd1ebd 100644 +--- a/pengine/test10/whitebox-fail2.summary ++++ b/pengine/test10/whitebox-fail2.summary +@@ -15,10 +15,10 @@ Containers: [ lxc2:container2 ] + + Transition Summary: + * Fence (reboot) lxc1 (resource: container1) 'guest is unclean' +- * Recover container1 (Started 18node2) +- * Recover M:4 (Started lxc1) +- * Recover B (Started lxc1) +- * Recover lxc1 (Started 18node2) ++ * Recover container1 ( 18node2 ) ++ * Recover M:4 ( lxc1 ) ++ * Recover B ( lxc1 ) ++ * Recover lxc1 ( 18node2 ) + + Executing cluster transition: + * Resource action: lxc1 stop on 18node2 +diff --git a/pengine/test10/whitebox-fail3.summary b/pengine/test10/whitebox-fail3.summary +index 19b2e3d..eded099 100644 +--- a/pengine/test10/whitebox-fail3.summary ++++ b/pengine/test10/whitebox-fail3.summary +@@ -14,7 +14,7 @@ Online: [ dvossel-laptop2 ] + + Transition Summary: + * Start vm (dvossel-laptop2) +- * Move FAKE (Started dvossel-laptop2 -> 18builder) ++ * Move FAKE ( dvossel-laptop2 -> 18builder ) + * Start W:1 (18builder) + * Start X:1 (18builder) + * Start 18builder (dvossel-laptop2) +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary +index bea75a2..a3e9ce8 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.summary ++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary +@@ -28,15 +28,15 @@ Transition Summary: + * Fence (reboot) lxc-02_kiff-01 (resource: R-lxc-02_kiff-01) 'guest is unclean' + * Fence (reboot) lxc-01_kiff-01 (resource: R-lxc-01_kiff-01) 'guest is unclean' + * Fence (reboot) kiff-01 'peer is no longer part of the cluster' +- * Move fence-kiff-02 (Started kiff-01 -> kiff-02) ++ * Move fence-kiff-02 ( kiff-01 -> kiff-02 ) + * Stop dlm:0 (kiff-01) due to node availability + * Stop clvmd:0 (kiff-01) due to node availability + * Stop shared0:0 (kiff-01) due to node availability +- * Recover R-lxc-01_kiff-01 (Started kiff-01 -> kiff-02) +- * Move R-lxc-02_kiff-01 (Started kiff-01 -> kiff-02) +- * Recover vm-fs (Started lxc-01_kiff-01) +- * Move lxc-01_kiff-01 (Started kiff-01 -> kiff-02) +- * Move lxc-02_kiff-01 (Started kiff-01 -> kiff-02) ++ * Recover R-lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) ++ * Move R-lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) ++ * Recover vm-fs ( lxc-01_kiff-01 ) ++ * Move lxc-01_kiff-01 ( kiff-01 -> kiff-02 ) ++ * Move lxc-02_kiff-01 ( kiff-01 -> kiff-02 ) + + Executing cluster transition: + * Pseudo action: fence-kiff-02_stop_0 +diff --git a/pengine/test10/whitebox-migrate1.summary b/pengine/test10/whitebox-migrate1.summary +index a6e4c00..0fc0f9a 100644 +--- a/pengine/test10/whitebox-migrate1.summary ++++ b/pengine/test10/whitebox-migrate1.summary +@@ -14,10 +14,10 @@ Containers: [ rhel7-node1:remote-rsc ] + remote-rsc (ocf::heartbeat:Dummy): Started rhel7-node2 + + Transition Summary: +- * Move shooter1 (Started rhel7-node3 -> rhel7-node2) +- * Move FAKE3 (Started rhel7-node3 -> rhel7-node2) +- * Migrate remote-rsc (Started rhel7-node2 -> rhel7-node3) +- * Migrate rhel7-node1 (Started rhel7-node2 -> rhel7-node3) ++ * Move shooter1 ( rhel7-node3 -> rhel7-node2 ) ++ * Move FAKE3 ( rhel7-node3 -> rhel7-node2 ) ++ * Migrate remote-rsc ( rhel7-node2 -> rhel7-node3 ) ++ * Migrate rhel7-node1 ( rhel7-node2 -> rhel7-node3 ) + + Executing cluster transition: + * Resource action: shooter1 stop on rhel7-node3 +diff --git a/pengine/test10/whitebox-move.summary b/pengine/test10/whitebox-move.summary +index dbf0780..5e27a67 100644 +--- a/pengine/test10/whitebox-move.summary ++++ b/pengine/test10/whitebox-move.summary +@@ -11,10 +11,10 @@ Containers: [ lxc1:container1 lxc2:container2 ] + A (ocf::pacemaker:Dummy): Started lxc1 + + Transition Summary: +- * Move container1 (Started 18node1 -> 18node2) +- * Restart M:3 (Started lxc1) due to required container1 start +- * Restart A (Started lxc1) due to required container1 start +- * Move lxc1 (Started 18node1 -> 18node2) ++ * Move container1 ( 18node1 -> 18node2 ) ++ * Restart M:3 ( lxc1 ) due to required container1 start ++ * Restart A ( lxc1 ) due to required container1 start ++ * Move lxc1 ( 18node1 -> 18node2 ) + + Executing cluster transition: + * Pseudo action: M-clone_stop_0 +diff --git a/pengine/test10/whitebox-ms-ordering-move.summary b/pengine/test10/whitebox-ms-ordering-move.summary +index b48d988..80156b0 100644 +--- a/pengine/test10/whitebox-ms-ordering-move.summary ++++ b/pengine/test10/whitebox-ms-ordering-move.summary +@@ -30,9 +30,9 @@ Containers: [ lxc1:container1 lxc2:container2 ] + Slaves: [ lxc2 ] + + Transition Summary: +- * Move container1 (Started rhel7-1 -> rhel7-2) ++ * Move container1 ( rhel7-1 -> rhel7-2 ) + * Restart lxc-ms:0 (Master lxc1) due to required container1 start +- * Move lxc1 (Started rhel7-1 -> rhel7-2) ++ * Move lxc1 ( rhel7-1 -> rhel7-2 ) + + Executing cluster transition: + * Pseudo action: lxc-ms-master_demote_0 +diff --git a/pengine/test10/whitebox-orphan-ms.summary b/pengine/test10/whitebox-orphan-ms.summary +index 3efa6bd..2b0234b 100644 +--- a/pengine/test10/whitebox-orphan-ms.summary ++++ b/pengine/test10/whitebox-orphan-ms.summary +@@ -27,10 +27,11 @@ Containers: [ lxc1:container1 lxc2:container2 ] + container1 (ocf::heartbeat:VirtualDomain): ORPHANED Started 18node1 + + Transition Summary: +- * Move FencingFail (Started 18node3 -> 18node1) ++ * Move FencingFail ( 18node3 -> 18node1 ) + * Stop container2 (18node1) due to node availability + * Stop lxc1 (18node1) due to node availability +- * Demote lxc-ms (Master -> Stopped lxc1) ++ * Stop lxc-ms ( Master lxc1 ) ++ * Stop lxc-ms ( Master lxc2 ) + * Stop lxc2 (18node1) due to node availability + * Stop container1 (18node1) due to node availability + +diff --git a/pengine/test10/whitebox-orphaned.summary b/pengine/test10/whitebox-orphaned.summary +index 52b54aa..7d173b2 100644 +--- a/pengine/test10/whitebox-orphaned.summary ++++ b/pengine/test10/whitebox-orphaned.summary +@@ -17,7 +17,7 @@ Containers: [ lxc1:container1 lxc2:container2 ] + + Transition Summary: + * Stop M:4 (lxc1) due to node availability +- * Move B (Started lxc1 -> lxc2) ++ * Move B ( lxc1 -> lxc2 ) + * Stop container1 (18node2) due to node availability + * Stop lxc1 (18node2) due to node availability + +diff --git a/pengine/test10/whitebox-start.summary b/pengine/test10/whitebox-start.summary +index 6819ebc..e76e28a 100644 +--- a/pengine/test10/whitebox-start.summary ++++ b/pengine/test10/whitebox-start.summary +@@ -17,8 +17,8 @@ Containers: [ lxc2:container2 ] + Transition Summary: + * Start container1 (18node1) + * Start M:4 (lxc1) +- * Move A (Started 18node1 -> lxc1) +- * Move B (Started lxc2 -> 18node3) ++ * Move A ( 18node1 -> lxc1 ) ++ * Move B ( lxc2 -> 18node3 ) + * Start lxc1 (18node1) + + Executing cluster transition: +diff --git a/pengine/test10/whitebox-stop.summary b/pengine/test10/whitebox-stop.summary +index 89094da..9b15ea0 100644 +--- a/pengine/test10/whitebox-stop.summary ++++ b/pengine/test10/whitebox-stop.summary +@@ -17,7 +17,7 @@ Containers: [ lxc1:container1 lxc2:container2 ] + Transition Summary: + * Stop container1 (18node2) due to node availability + * Stop M:4 (lxc1) due to node availability +- * Move B (Started lxc1 -> lxc2) ++ * Move B ( lxc1 -> lxc2 ) + * Stop lxc1 (18node2) due to node availability + + Executing cluster transition: +diff --git a/pengine/test10/whitebox-unexpectedly-running.summary b/pengine/test10/whitebox-unexpectedly-running.summary +index 0e69d3c..eef4f63 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.summary ++++ b/pengine/test10/whitebox-unexpectedly-running.summary +@@ -6,7 +6,7 @@ Online: [ 18builder ] + + Transition Summary: + * Fence (reboot) remote1 (resource: FAKE) 'guest is unclean' +- * Recover FAKE (Started 18builder) ++ * Recover FAKE ( 18builder ) + * Start remote1 (18builder) + + Executing cluster transition: +-- +1.8.3.1 + + +From b5c48ca0bf52c9a03c81ba4fb74dbc5c1a7f6dbc Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 1 Aug 2017 13:36:04 +1000 +Subject: [PATCH 07/12] Fix: PE: Only retrigger unfencing on nodes that ran + operations with the old parameters + +--- + pengine/allocate.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index ef26245..f169db3 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -329,7 +329,7 @@ check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op + op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC)); + + custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set); +- trigger_unfencing(rsc, NULL, "Device parameters changed", NULL, data_set); ++ trigger_unfencing(rsc, active_node, "Device parameters changed", NULL, data_set); + + } else if ((digest_data->rc == RSC_DIGEST_ALL) || (digest_data->rc == RSC_DIGEST_UNKNOWN)) { + /* Changes that can potentially be handled by a reload */ +@@ -337,7 +337,7 @@ check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op + const char *digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST); + + did_change = TRUE; +- trigger_unfencing(rsc, NULL, "Device parameters changed (reload)", NULL, data_set); ++ trigger_unfencing(rsc, active_node, "Device parameters changed (reload)", NULL, data_set); + crm_log_xml_info(digest_data->params_all, "params:reload"); + key = generate_op_key(rsc->id, task, interval); + pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (reload:%s) %s", +-- +1.8.3.1 + + +From a8e0c440577bd082b70e69bc5b6539fc999cbac7 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Mon, 14 Aug 2017 12:35:16 +1000 +Subject: [PATCH 08/12] PE: Unfencing: Correctly detect changes to device + definitions + +--- + lib/pengine/utils.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 222a3e2..d4943a7 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1952,7 +1952,7 @@ fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * da + * We use the resource id + agent + digest so that we can detect + * changes to the agent and/or the parameters used + */ +- char *search_all = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); ++ char *search_all = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_all_calc); + char *search_secure = crm_strdup_printf("%s:%s:%s", rsc->id, (const char*)g_hash_table_lookup(rsc->meta, XML_ATTR_TYPE), data->digest_secure_calc); + + data->rc = RSC_DIGEST_ALL; +-- +1.8.3.1 + + +From 7a3dbd5fd5481ecc3a375d9874f01daac6512943 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 16 Aug 2017 13:50:07 +1000 +Subject: [PATCH 09/12] Test: PE: Update unfencing device change test output + +--- + pengine/test10/unfence-parameters.dot | 22 ------ + pengine/test10/unfence-parameters.exp | 117 ++++++------------------------ + pengine/test10/unfence-parameters.summary | 9 +-- + 3 files changed, 23 insertions(+), 125 deletions(-) + +diff --git a/pengine/test10/unfence-parameters.dot b/pengine/test10/unfence-parameters.dot +index dee850e..ccbddbd 100644 +--- a/pengine/test10/unfence-parameters.dot ++++ b/pengine/test10/unfence-parameters.dot +@@ -21,10 +21,7 @@ digraph "g" { + "clvmd:2_monitor_0 virt-3" -> "clvmd-clone_start_0" [ style = bold] + "clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold] +-"clvmd:2_start_0 virt-3" -> "clvmd:3_start_0 " [ style = dashed] + "clvmd:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] +-"clvmd:3_start_0 " -> "clvmd-clone_running_0" [ style = dashed] +-"clvmd:3_start_0 " [ style=dashed color="red" fontcolor="black"] + "clvmd_start_0 virt-1" -> "clvmd-clone_running_0" [ style = bold] + "clvmd_start_0 virt-1" -> "clvmd:1_start_0 virt-2" [ style = bold] + "clvmd_start_0 virt-1" [ style=bold color="green" fontcolor="black"] +@@ -39,11 +36,9 @@ digraph "g" { + "dlm-clone_start_0" -> "dlm-clone_running_0" [ style = bold] + "dlm-clone_start_0" -> "dlm:2_start_0 virt-3" [ style = bold] + "dlm-clone_start_0" -> "dlm_start_0 virt-1" [ style = bold] +-"dlm-clone_start_0" -> "dlm_start_0 virt-2" [ style = bold] + "dlm-clone_start_0" [ style=bold color="green" fontcolor="orange"] + "dlm-clone_stop_0" -> "dlm-clone_stopped_0" [ style = bold] + "dlm-clone_stop_0" -> "dlm_stop_0 virt-1" [ style = bold] +-"dlm-clone_stop_0" -> "dlm_stop_0 virt-2" [ style = bold] + "dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"] + "dlm-clone_stopped_0" -> "dlm-clone_start_0" [ style = bold] + "dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] +@@ -54,24 +49,12 @@ digraph "g" { + "dlm:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] + "dlm_start_0 virt-1" -> "clvmd_start_0 virt-1" [ style = bold] + "dlm_start_0 virt-1" -> "dlm-clone_running_0" [ style = bold] +-"dlm_start_0 virt-1" -> "dlm_start_0 virt-2" [ style = bold] + "dlm_start_0 virt-1" [ style=bold color="green" fontcolor="black"] +-"dlm_start_0 virt-2" -> "clvmd:1_start_0 virt-2" [ style = bold] +-"dlm_start_0 virt-2" -> "clvmd:3_start_0 " [ style = dashed] +-"dlm_start_0 virt-2" -> "dlm-clone_running_0" [ style = bold] +-"dlm_start_0 virt-2" -> "dlm:2_start_0 virt-3" [ style = bold] +-"dlm_start_0 virt-2" [ style=bold color="green" fontcolor="black"] + "dlm_stop_0 virt-1" -> "all_stopped" [ style = bold] + "dlm_stop_0 virt-1" -> "dlm-clone_stopped_0" [ style = bold] + "dlm_stop_0 virt-1" -> "dlm_start_0 virt-1" [ style = bold] + "dlm_stop_0 virt-1" -> "stonith 'on' virt-1" [ style = bold] + "dlm_stop_0 virt-1" [ style=bold color="green" fontcolor="black"] +-"dlm_stop_0 virt-2" -> "all_stopped" [ style = bold] +-"dlm_stop_0 virt-2" -> "dlm-clone_stopped_0" [ style = bold] +-"dlm_stop_0 virt-2" -> "dlm_start_0 virt-2" [ style = bold] +-"dlm_stop_0 virt-2" -> "dlm_stop_0 virt-1" [ style = bold] +-"dlm_stop_0 virt-2" -> "stonith 'on' virt-2" [ style = bold] +-"dlm_stop_0 virt-2" [ style=bold color="green" fontcolor="black"] + "fencing_monitor_0 virt-3" -> "fencing_start_0 virt-1" [ style = bold] + "fencing_monitor_0 virt-3" -> "fencing_stop_0 virt-1" [ style = bold] + "fencing_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] +@@ -82,10 +65,6 @@ digraph "g" { + "stonith 'on' virt-1" -> "clvmd_start_0 virt-1" [ style = bold] + "stonith 'on' virt-1" -> "dlm_start_0 virt-1" [ style = bold] + "stonith 'on' virt-1" [ style=bold color="green" fontcolor="black"] +-"stonith 'on' virt-2" -> "clvmd:1_monitor_0 virt-2" [ style = bold] +-"stonith 'on' virt-2" -> "clvmd:1_start_0 virt-2" [ style = bold] +-"stonith 'on' virt-2" -> "dlm_start_0 virt-2" [ style = bold] +-"stonith 'on' virt-2" [ style=bold color="green" fontcolor="black"] + "stonith 'on' virt-3" -> "clvmd:2_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] + "stonith 'on' virt-3" -> "dlm:2_monitor_0 virt-3" [ style = bold] +@@ -99,6 +78,5 @@ digraph "g" { + "stonith_complete" -> "clvmd_start_0 virt-1" [ style = bold] + "stonith_complete" -> "dlm:2_start_0 virt-3" [ style = bold] + "stonith_complete" -> "dlm_start_0 virt-1" [ style = bold] +-"stonith_complete" -> "dlm_start_0 virt-2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/unfence-parameters.exp b/pengine/test10/unfence-parameters.exp +index 54c242a..64286a8 100644 +--- a/pengine/test10/unfence-parameters.exp ++++ b/pengine/test10/unfence-parameters.exp +@@ -71,9 +71,6 @@ + + + +- +- +- + + + +@@ -83,44 +80,6 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- + + + +@@ -131,9 +90,6 @@ + + + +- +- +- + + + +@@ -141,7 +97,7 @@ + + + +- ++ + + + +@@ -154,7 +110,7 @@ + + + +- ++ + + + +@@ -165,14 +121,11 @@ + + + +- +- +- + + + + +- ++ + + + +@@ -184,7 +137,7 @@ + + + +- ++ + + + +@@ -195,9 +148,6 @@ + + + +- +- +- + + + +@@ -205,7 +155,7 @@ + + + +- ++ + + + +@@ -220,7 +170,7 @@ + + + +- ++ + + + +@@ -245,7 +195,7 @@ + + + +- ++ + + + +@@ -258,7 +208,7 @@ + + + +- ++ + + + +@@ -267,12 +217,6 @@ + + + +- +- +- +- +- +- + + + +@@ -283,20 +227,16 @@ + + + +- ++ + + + + + + +- +- +- +- +- ++ + +- ++ + + + +@@ -321,7 +261,7 @@ + + + +- ++ + + + +@@ -334,7 +274,7 @@ + + + +- ++ + + + +@@ -349,7 +289,7 @@ + + + +- ++ + + + +@@ -357,7 +297,7 @@ + + + +- ++ + + + +@@ -378,7 +318,7 @@ + + + +- ++ + + + +@@ -399,7 +339,7 @@ + + + +- ++ + + + +@@ -410,7 +350,7 @@ + + + +- ++ + + + +@@ -422,7 +362,7 @@ + + + +- ++ + + + +@@ -430,19 +370,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -457,7 +385,7 @@ + + + +- ++ + + + +@@ -471,9 +399,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +index 8a6356c..b9f4953 100644 +--- a/pengine/test10/unfence-parameters.summary ++++ b/pengine/test10/unfence-parameters.summary +@@ -13,12 +13,10 @@ Online: [ virt-1 virt-2 virt-3 ] + + Transition Summary: + * Fence (reboot) virt-4 'node is unclean' +- * Fence (on) virt-3 'Device parameters changed (reload)' +- * Fence (on) virt-2 'Device parameters changed (reload)' ++ * Fence (on) virt-3 'Required by dlm:2' + * Fence (on) virt-1 'Device parameters changed (reload)' + * Restart fencing ( virt-1 ) + * Restart dlm:0 ( virt-1 ) due to required stonith +- * Restart dlm:1 ( virt-2 ) due to required stonith + * Start dlm:2 (virt-3) + * Restart clvmd:0 ( virt-1 ) due to required stonith + * Start clvmd:1 (virt-2) +@@ -26,6 +24,7 @@ Transition Summary: + + Executing cluster transition: + * Resource action: fencing monitor on virt-3 ++ * Resource action: clvmd monitor on virt-2 + * Pseudo action: clvmd-clone_stop_0 + * Fencing virt-4 (reboot) + * Pseudo action: stonith_complete +@@ -36,17 +35,13 @@ Executing cluster transition: + * Resource action: clvmd monitor on virt-3 + * Pseudo action: clvmd-clone_stopped_0 + * Pseudo action: dlm-clone_stop_0 +- * Resource action: dlm stop on virt-2 +- * Fencing virt-2 (on) + * Resource action: dlm stop on virt-1 + * Pseudo action: dlm-clone_stopped_0 + * Pseudo action: dlm-clone_start_0 +- * Resource action: clvmd monitor on virt-2 + * Fencing virt-1 (on) + * Pseudo action: all_stopped + * Resource action: fencing start on virt-1 + * Resource action: dlm start on virt-1 +- * Resource action: dlm start on virt-2 + * Resource action: dlm start on virt-3 + * Pseudo action: dlm-clone_running_0 + * Pseudo action: clvmd-clone_start_0 +-- +1.8.3.1 + + +From d86c8d491d8ed2beb13c86a32e96658f54caae62 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Mon, 14 Aug 2017 12:51:25 +1000 +Subject: [PATCH 10/12] Fix: crmd: Correctly record that unfencing is complete + +--- + crmd/te_callbacks.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/crmd/te_callbacks.c b/crmd/te_callbacks.c +index b5c857d..4476d4a 100644 +--- a/crmd/te_callbacks.c ++++ b/crmd/te_callbacks.c +@@ -799,10 +799,11 @@ tengine_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data) + if (safe_str_eq("on", op)) { + const char *key = NULL; + const char *value = NULL; ++ char *now = crm_itoa(time(NULL)); + + key = XML_NODE_IS_UNFENCED; +- value = crm_meta_value(action->params, key); +- update_attrd(target, key, value, NULL, FALSE); ++ update_attrd(target, key, now, NULL, FALSE); ++ free(now); + + key = "digests-all"; + value = crm_meta_value(action->params, key); +-- +1.8.3.1 + + +From 31d4f2e01d47312ff21da845f0fc6816630c00ec Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Mon, 7 Aug 2017 17:07:54 +1000 +Subject: [PATCH 11/12] Fix: PE: Allow resources inside bundles to receive + notifications + +--- + lib/pengine/container.c | 5 +++++ + pengine/container.c | 8 +++++--- + 2 files changed, 10 insertions(+), 3 deletions(-) + +diff --git a/lib/pengine/container.c b/lib/pengine/container.c +index 175a366..118e716 100644 +--- a/lib/pengine/container.c ++++ b/lib/pengine/container.c +@@ -757,6 +757,11 @@ container_unpack(resource_t * rsc, pe_working_set_t * data_set) + tuple->child = childIter->data; + tuple->offset = lpc++; + ++ // Ensure the child's notify gets set based on the underlying primitive's value ++ if(is_set(tuple->child->flags, pe_rsc_notify)) { ++ set_bit(container_data->child->flags, pe_rsc_notify); ++ } ++ + offset += allocate_ip(container_data, tuple, buffer+offset, max-offset); + container_data->tuples = g_list_append(container_data->tuples, tuple); + } +diff --git a/pengine/container.c b/pengine/container.c +index 280a61a..1e9b971 100644 +--- a/pengine/container.c ++++ b/pengine/container.c +@@ -770,6 +770,11 @@ container_expand(resource_t * rsc, pe_working_set_t * data_set) + CRM_CHECK(rsc != NULL, return); + + get_container_variant_data(container_data, rsc); ++ ++ if(container_data->child) { ++ container_data->child->cmds->expand(container_data->child, data_set); ++ } ++ + for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { + container_grouping_t *tuple = (container_grouping_t *)gIter->data; + +@@ -787,9 +792,6 @@ container_expand(resource_t * rsc, pe_working_set_t * data_set) + if(tuple->ip) { + tuple->ip->cmds->expand(tuple->ip, data_set); + } +- if(tuple->child) { +- tuple->child->cmds->expand(tuple->child, data_set); +- } + if(tuple->docker) { + tuple->docker->cmds->expand(tuple->docker, data_set); + } +-- +1.8.3.1 + + +From 5ce481caa150564ae04b71d87b8737fcf88b5852 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 16 Aug 2017 14:09:56 +1000 +Subject: [PATCH 12/12] Test: PE: Update bundle regression tests now that + notify works + +--- + pengine/test10/bundle-nested-colocation.dot | 23 +- + pengine/test10/bundle-nested-colocation.exp | 355 +++--- + pengine/test10/bundle-nested-colocation.summary | 15 +- + pengine/test10/bundle-order-partial-start-2.dot | 33 +- + pengine/test10/bundle-order-partial-start-2.exp | 414 ++++--- + .../test10/bundle-order-partial-start-2.summary | 25 +- + pengine/test10/bundle-order-partial-start.dot | 33 +- + pengine/test10/bundle-order-partial-start.exp | 400 ++++--- + pengine/test10/bundle-order-partial-start.summary | 27 +- + pengine/test10/bundle-order-partial-stop.dot | 78 +- + pengine/test10/bundle-order-partial-stop.exp | 772 ++++++++----- + pengine/test10/bundle-order-partial-stop.summary | 40 +- + pengine/test10/bundle-order-startup-clone-2.dot | 56 +- + pengine/test10/bundle-order-startup-clone-2.exp | 1146 ++++++++++++-------- + .../test10/bundle-order-startup-clone-2.summary | 37 +- + pengine/test10/bundle-order-startup-clone.dot | 16 +- + pengine/test10/bundle-order-startup-clone.exp | 162 ++- + pengine/test10/bundle-order-startup-clone.summary | 9 +- + pengine/test10/bundle-order-startup.dot | 31 +- + pengine/test10/bundle-order-startup.exp | 597 ++++++---- + pengine/test10/bundle-order-startup.summary | 26 +- + pengine/test10/bundle-order-stop-clone.exp | 88 +- + pengine/test10/bundle-order-stop-clone.summary | 2 +- + pengine/test10/bundle-order-stop.dot | 78 +- + pengine/test10/bundle-order-stop.exp | 772 ++++++++----- + pengine/test10/bundle-order-stop.summary | 40 +- + pengine/test10/remote-fence-unclean-3.exp | 4 +- + 27 files changed, 3511 insertions(+), 1768 deletions(-) + +diff --git a/pengine/test10/bundle-nested-colocation.dot b/pengine/test10/bundle-nested-colocation.dot +index a84f857..2a2d71a 100644 +--- a/pengine/test10/bundle-nested-colocation.dot ++++ b/pengine/test10/bundle-nested-colocation.dot +@@ -15,7 +15,22 @@ digraph "g" { + "rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq:2_monitor_10000 rabbitmq-bundle-2" [ style = bold] + "rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq:2_start_0 rabbitmq-bundle-2" [ style = bold] + "rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] +-"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq:2_monitor_10000 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq:1_post_notify_start_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq:2_post_notify_start_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] + "rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] +@@ -56,16 +71,22 @@ digraph "g" { + "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-2_start_0 overcloud-rabbit-2" [ style = bold] + "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:1_post_notify_start_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq:1_post_notify_start_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] + "rabbitmq:1_start_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq:1_start_0 rabbitmq-bundle-1" -> "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style = bold] + "rabbitmq:1_start_0 rabbitmq-bundle-1" -> "rabbitmq:2_start_0 rabbitmq-bundle-2" [ style = bold] + "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] + "rabbitmq:2_monitor_10000 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:2_post_notify_start_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq:2_post_notify_start_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] + "rabbitmq:2_start_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq:2_start_0 rabbitmq-bundle-2" -> "rabbitmq:2_monitor_10000 rabbitmq-bundle-2" [ style = bold] + "rabbitmq:2_start_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-nested-colocation.exp b/pengine/test10/bundle-nested-colocation.exp +index 3daf7d6..c82ee16 100644 +--- a/pengine/test10/bundle-nested-colocation.exp ++++ b/pengine/test10/bundle-nested-colocation.exp +@@ -1,9 +1,22 @@ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + + +@@ -13,13 +26,16 @@ + + + ++ ++ ++ + + +- ++ + + + +- ++ + + + +@@ -34,211 +50,335 @@ + + + +- ++ + +- +- +- ++ ++ ++ + + +- ++ ++ ++ ++ ++ + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ ++ ++ ++ + + + + ++ ++ ++ + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + ++ ++ ++ ++ ++ ++ + + + + +- ++ + +- +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- +- +- +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + +- ++ + + + +@@ -247,7 +387,7 @@ + + + +- ++ + + + +@@ -263,7 +403,7 @@ + + + +- ++ + + + +@@ -276,7 +416,7 @@ + + + +- ++ + + + +@@ -289,7 +429,7 @@ + + + +- ++ + + + +@@ -302,40 +442,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -352,11 +459,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -364,7 +471,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-nested-colocation.summary b/pengine/test10/bundle-nested-colocation.summary +index 0e2a68d..b2cec51 100644 +--- a/pengine/test10/bundle-nested-colocation.summary ++++ b/pengine/test10/bundle-nested-colocation.summary +@@ -28,11 +28,14 @@ Transition Summary: + * Start rabbitmq:2 (rabbitmq-bundle-2) + + Executing cluster transition: ++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 + * Resource action: rabbitmq-bundle-docker-0 stop on overcloud-rabbit-0 + * Resource action: rabbitmq-bundle-docker-1 stop on overcloud-rabbit-1 + * Resource action: rabbitmq-bundle-docker-2 stop on overcloud-rabbit-2 + * Pseudo action: rabbitmq-bundle_start_0 + * Pseudo action: all_stopped ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 + * Resource action: rabbitmq-bundle-docker-0 start on overcloud-rabbit-0 + * Resource action: rabbitmq-bundle-docker-0 monitor=60000 on overcloud-rabbit-0 + * Resource action: rabbitmq-bundle-0 start on overcloud-controller-0 +@@ -42,14 +45,18 @@ Executing cluster transition: + * Resource action: rabbitmq-bundle-docker-2 start on overcloud-rabbit-2 + * Resource action: rabbitmq-bundle-docker-2 monitor=60000 on overcloud-rabbit-2 + * Resource action: rabbitmq-bundle-2 start on overcloud-controller-2 +- * Pseudo action: rabbitmq-bundle-clone_start_0 + * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 +- * Resource action: rabbitmq-bundle-0 monitor=60000 on overcloud-controller-0 + * Resource action: rabbitmq:1 start on rabbitmq-bundle-1 +- * Resource action: rabbitmq-bundle-1 monitor=60000 on overcloud-controller-1 + * Resource action: rabbitmq:2 start on rabbitmq-bundle-2 +- * Resource action: rabbitmq-bundle-2 monitor=60000 on overcloud-controller-2 + * Pseudo action: rabbitmq-bundle-clone_running_0 ++ * Resource action: rabbitmq-bundle-0 monitor=60000 on overcloud-controller-0 ++ * Resource action: rabbitmq-bundle-1 monitor=60000 on overcloud-controller-1 ++ * Resource action: rabbitmq-bundle-2 monitor=60000 on overcloud-controller-2 ++ * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 ++ * Resource action: rabbitmq:0 notify on rabbitmq-bundle-0 ++ * Resource action: rabbitmq:1 notify on rabbitmq-bundle-1 ++ * Resource action: rabbitmq:2 notify on rabbitmq-bundle-2 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 + * Pseudo action: rabbitmq-bundle_running_0 + * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 + * Resource action: rabbitmq:1 monitor=10000 on rabbitmq-bundle-1 +diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot +index b5ecaf5..d58e809 100644 +--- a/pengine/test10/bundle-order-partial-start-2.dot ++++ b/pengine/test10/bundle-order-partial-start-2.dot +@@ -44,7 +44,18 @@ digraph "g" { + "haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] + "haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] +-"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] + "rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] +@@ -54,18 +65,36 @@ digraph "g" { + "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] + "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold] + "redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] +-"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_promoted_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] + "redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] + "redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle_promoted_0" -> "galera-bundle_start_0" [ style = bold] + "redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] + "redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold] + "redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] + "redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp +index a2740d7..b533469 100644 +--- a/pengine/test10/bundle-order-partial-start-2.exp ++++ b/pengine/test10/bundle-order-partial-start-2.exp +@@ -1,22 +1,38 @@ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + + + + + ++ ++ ++ + + +- ++ + + + +- ++ + + + +@@ -25,87 +41,194 @@ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + + + +@@ -114,29 +237,29 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -146,7 +269,7 @@ + + + +- ++ + + + +@@ -155,226 +278,257 @@ + + + +- ++ + + + +- ++ + +- ++ + +- ++ + + + + +- ++ + + + +- ++ + +- ++ + +- ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- +- +- +- ++ + + + +- +- +- +- +- +- +- +- +- +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- ++ ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- +- +- +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + +- +- +- +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + +- +- +- +- +- +- +- ++ + + + +- ++ + +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + + +@@ -382,11 +536,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -394,7 +548,7 @@ + + + +- ++ + + + +@@ -402,10 +556,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary +index 790bd17..bf14db0 100644 +--- a/pengine/test10/bundle-order-partial-start-2.summary ++++ b/pengine/test10/bundle-order-partial-start-2.summary +@@ -29,32 +29,43 @@ Transition Summary: + * Start haproxy-bundle-docker-0 (undercloud) + + Executing cluster transition: ++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 + * Resource action: galera-bundle-0 stop on undercloud ++ * Pseudo action: redis-bundle-master_pre_notify_promote_0 + * Resource action: haproxy-bundle-docker-0 monitor on undercloud + * Pseudo action: haproxy-bundle_start_0 + * Pseudo action: redis-bundle_promote_0 +- * Pseudo action: redis-bundle-master_promote_0 + * Pseudo action: rabbitmq-bundle_start_0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 + * Resource action: galera-bundle-docker-0 stop on undercloud +- * Resource action: redis promote on redis-bundle-0 ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0 ++ * Pseudo action: redis-bundle-master_promote_0 + * Resource action: haproxy-bundle-docker-0 start on undercloud + * Pseudo action: haproxy-bundle_running_0 +- * Pseudo action: redis-bundle-master_promoted_0 +- * Pseudo action: rabbitmq-bundle-clone_start_0 + * Pseudo action: all_stopped + * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 +- * Resource action: redis monitor=20000 on redis-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_running_0 ++ * Resource action: redis promote on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_promoted_0 + * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud ++ * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 ++ * Pseudo action: redis-bundle-master_post_notify_promoted_0 ++ * Resource action: rabbitmq:0 notify on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0 + * Pseudo action: redis-bundle_promoted_0 +- * Pseudo action: rabbitmq-bundle-clone_running_0 + * Pseudo action: rabbitmq-bundle_running_0 + * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 ++ * Resource action: redis monitor=20000 on redis-bundle-0 + * Pseudo action: galera-bundle_start_0 ++ * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera-bundle-docker-0 start on undercloud + * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud + * Resource action: galera-bundle-0 start on undercloud + * Resource action: galera-bundle-0 monitor=60000 on undercloud +- * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera:0 start on galera-bundle-0 + * Pseudo action: galera-bundle-master_running_0 + * Pseudo action: galera-bundle_running_0 +diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot +index 756acfd..796749d 100644 +--- a/pengine/test10/bundle-order-partial-start.dot ++++ b/pengine/test10/bundle-order-partial-start.dot +@@ -38,7 +38,18 @@ digraph "g" { + "haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] + "haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 undercloud" [ style = bold] + "haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] +-"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] + "rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] +@@ -48,18 +59,36 @@ digraph "g" { + "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] + "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-0" [ style = bold] + "redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] +-"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_promoted_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] + "redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] + "redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle_promoted_0" -> "galera-bundle_start_0" [ style = bold] + "redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] + "redis_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold] + "redis_promote_0 redis-bundle-0" -> "redis_monitor_20000 redis-bundle-0" [ style = bold] + "redis_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp +index ebd5785..1f8a3e1 100644 +--- a/pengine/test10/bundle-order-partial-start.exp ++++ b/pengine/test10/bundle-order-partial-start.exp +@@ -1,22 +1,38 @@ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + + + + + ++ ++ ++ + + +- ++ + + + +- ++ + + + +@@ -25,73 +41,180 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -101,11 +224,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -114,248 +237,279 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + +- ++ + + + + +- ++ + + + +- ++ + +- ++ + +- ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- +- +- +- ++ + + + +- +- +- +- +- +- +- +- +- +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- ++ ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- +- +- +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + +- +- +- +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + +- +- +- +- +- +- +- ++ + + + +- ++ + +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + + +@@ -363,11 +517,11 @@ + + + +- ++ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start.summary b/pengine/test10/bundle-order-partial-start.summary +index e56e55f..4e02e88 100644 +--- a/pengine/test10/bundle-order-partial-start.summary ++++ b/pengine/test10/bundle-order-partial-start.summary +@@ -29,32 +29,43 @@ Transition Summary: + * Start haproxy-bundle-docker-0 (undercloud) + + Executing cluster transition: ++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 + * Resource action: galera-bundle-docker-0 monitor on undercloud ++ * Pseudo action: redis-bundle-master_pre_notify_promote_0 + * Resource action: haproxy-bundle-docker-0 monitor on undercloud + * Pseudo action: haproxy-bundle_start_0 + * Pseudo action: redis-bundle_promote_0 +- * Pseudo action: redis-bundle-master_promote_0 + * Pseudo action: rabbitmq-bundle_start_0 +- * Resource action: redis promote on redis-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0 ++ * Pseudo action: redis-bundle-master_promote_0 + * Resource action: haproxy-bundle-docker-0 start on undercloud + * Pseudo action: haproxy-bundle_running_0 +- * Pseudo action: redis-bundle-master_promoted_0 +- * Pseudo action: rabbitmq-bundle-clone_start_0 + * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 +- * Resource action: redis monitor=20000 on redis-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_running_0 ++ * Resource action: redis promote on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_promoted_0 + * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud ++ * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 ++ * Pseudo action: redis-bundle-master_post_notify_promoted_0 ++ * Resource action: rabbitmq:0 notify on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0 + * Pseudo action: redis-bundle_promoted_0 +- * Pseudo action: rabbitmq-bundle-clone_running_0 + * Pseudo action: rabbitmq-bundle_running_0 + * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 ++ * Resource action: redis monitor=20000 on redis-bundle-0 + * Pseudo action: galera-bundle_start_0 ++ * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera-bundle-docker-0 start on undercloud + * Resource action: galera-bundle-0 start on undercloud +- * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera:0 start on galera-bundle-0 ++ * Pseudo action: galera-bundle-master_running_0 + * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud + * Resource action: galera-bundle-0 monitor=60000 on undercloud +- * Pseudo action: galera-bundle-master_running_0 + * Pseudo action: galera-bundle_running_0 + * Resource action: galera:0 monitor=30000 on galera-bundle-0 + * Resource action: galera:0 monitor=20000 on galera-bundle-0 +diff --git a/pengine/test10/bundle-order-partial-stop.dot b/pengine/test10/bundle-order-partial-stop.dot +index c0e6616..3fade95 100644 +--- a/pengine/test10/bundle-order-partial-stop.dot ++++ b/pengine/test10/bundle-order-partial-stop.dot +@@ -118,7 +118,29 @@ digraph "g" { + "rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] + "rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold] + "rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] +-"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle-clone_pre_notify_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = dashed] + "rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] +@@ -126,8 +148,8 @@ digraph "g" { + "rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] + "rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] + "rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] +-"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] + "rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] +@@ -140,6 +162,8 @@ digraph "g" { + "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] + "rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] + "rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] +@@ -156,14 +180,52 @@ digraph "g" { + "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" -> "redis-bundle_stopped_0" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_start_0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_stop_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle-master_pre_notify_start_0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle_stopped_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_demote_0" -> "redis-bundle-master_post_notify_demoted_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_post_notify_running_0" [ style = dashed] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_start_0" [ style = dashed] ++"redis-bundle-master_confirmed-pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_stop_0" -> "redis-bundle-master_post_notify_stopped_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] + "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] + "redis-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_post_notify_demoted_0" [ style = bold] + "redis-bundle-master_demoted_0" -> "redis-bundle-master_start_0" [ style = dashed] + "redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold] +-"redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] + "redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] +-"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = dashed] ++"redis-bundle-master_post_notify_demoted_0" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] ++"redis-bundle-master_post_notify_demoted_0" -> "redis_post_notify_demoted_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_running_0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = dashed] ++"redis-bundle-master_post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_post_notify_stopped_0" -> "redis-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] ++"redis-bundle-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_demote_0" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] ++"redis-bundle-master_pre_notify_demote_0" -> "redis_pre_notify_demote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_start_0" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = dashed] ++"redis-bundle-master_pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_pre_notify_stop_0" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] ++"redis-bundle-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" -> "redis-bundle-master_post_notify_running_0" [ style = dashed] + "redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed] + "redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed] +@@ -171,8 +233,8 @@ digraph "g" { + "redis-bundle-master_stop_0" -> "redis-bundle-master_stopped_0" [ style = bold] + "redis-bundle-master_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] + "redis-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_stopped_0" -> "redis-bundle-master_post_notify_stopped_0" [ style = bold] + "redis-bundle-master_stopped_0" -> "redis-bundle-master_start_0" [ style = dashed] +-"redis-bundle-master_stopped_0" -> "redis-bundle_stopped_0" [ style = bold] + "redis-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] + "redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold] +@@ -199,6 +261,12 @@ digraph "g" { + "redis_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis_monitor_45000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] + "redis_monitor_60000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_post_notify_demoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] ++"redis_post_notify_demoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_demote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] ++"redis_pre_notify_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_stop_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] ++"redis_pre_notify_stop_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = dashed] + "redis_start_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] + "redis_start_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] +diff --git a/pengine/test10/bundle-order-partial-stop.exp b/pengine/test10/bundle-order-partial-stop.exp +index 27ae320..937bc59 100644 +--- a/pengine/test10/bundle-order-partial-stop.exp ++++ b/pengine/test10/bundle-order-partial-stop.exp +@@ -1,9 +1,22 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ + + + +@@ -11,11 +24,91 @@ + + + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -31,7 +124,7 @@ + + + +- ++ + + + +@@ -47,28 +140,28 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -78,11 +171,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -91,567 +184,692 @@ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ + + +- ++ + + + +- ++ + +- +- +- +- +- +- +- ++ ++ ++ + + + +- +- +- +- ++ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ + + +- +- +- +- ++ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ + + +- ++ + + + +- +- +- +- +- +- +- +- +- +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ + + +- +- +- +- +- ++ + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- +- +- +- ++ ++ ++ + + + +- +- +- +- +- +- +- ++ + + +- +- +- +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + + +- ++ + +- +- ++ ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- +- +- +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- +- +- +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + +- +- +- +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- +- ++ ++ + + + + +- +- +- +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + + +- ++ + + + +@@ -662,11 +880,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -674,7 +892,7 @@ + + + +- ++ + + + +@@ -691,46 +909,52 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-stop.summary b/pengine/test10/bundle-order-partial-stop.summary +index 7bc24d2..29cf490 100644 +--- a/pengine/test10/bundle-order-partial-stop.summary ++++ b/pengine/test10/bundle-order-partial-stop.summary +@@ -41,41 +41,57 @@ Transition Summary: + * Stop openstack-cinder-volume-docker-0 (undercloud) due to node availability + + Executing cluster transition: ++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_stop_0 + * Resource action: galera cancel=10000 on galera-bundle-0 + * Resource action: redis cancel=20000 on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_pre_notify_demote_0 + * Pseudo action: openstack-cinder-volume_stop_0 + * Pseudo action: redis-bundle_demote_0 +- * Pseudo action: redis-bundle-master_demote_0 + * Pseudo action: galera-bundle_demote_0 +- * Pseudo action: galera-bundle-master_demote_0 + * Pseudo action: rabbitmq-bundle_stop_0 +- * Resource action: galera demote on galera-bundle-0 +- * Resource action: redis demote on redis-bundle-0 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_stop_0 ++ * Pseudo action: rabbitmq-bundle-clone_stop_0 ++ * Pseudo action: galera-bundle-master_demote_0 ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_demote_0 ++ * Pseudo action: redis-bundle-master_demote_0 + * Resource action: openstack-cinder-volume-docker-0 stop on undercloud + * Pseudo action: openstack-cinder-volume_stopped_0 +- * Pseudo action: redis-bundle-master_demoted_0 +- * Pseudo action: galera-bundle-master_demoted_0 +- * Pseudo action: rabbitmq-bundle-clone_stop_0 + * Resource action: rabbitmq stop on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_stopped_0 + * Resource action: rabbitmq-bundle-0 stop on undercloud +- * Pseudo action: redis-bundle_demoted_0 ++ * Resource action: galera demote on galera-bundle-0 ++ * Pseudo action: galera-bundle-master_demoted_0 ++ * Resource action: redis demote on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_demoted_0 + * Pseudo action: galera-bundle_demoted_0 + * Pseudo action: galera-bundle_stop_0 +- * Pseudo action: rabbitmq-bundle-clone_stopped_0 ++ * Pseudo action: rabbitmq-bundle-clone_post_notify_stopped_0 + * Resource action: rabbitmq-bundle-docker-0 stop on undercloud + * Pseudo action: galera-bundle-master_stop_0 +- * Pseudo action: rabbitmq-bundle_stopped_0 ++ * Pseudo action: redis-bundle-master_post_notify_demoted_0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_stopped_0 + * Resource action: galera stop on galera-bundle-0 +- * Resource action: galera-bundle-0 stop on undercloud + * Pseudo action: galera-bundle-master_stopped_0 ++ * Resource action: galera-bundle-0 stop on undercloud ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_demoted_0 ++ * Pseudo action: redis-bundle-master_pre_notify_stop_0 ++ * Pseudo action: redis-bundle_demoted_0 ++ * Pseudo action: rabbitmq-bundle_stopped_0 + * Resource action: galera-bundle-docker-0 stop on undercloud ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_stop_0 + * Pseudo action: galera-bundle_stopped_0 + * Pseudo action: redis-bundle_stop_0 + * Pseudo action: redis-bundle-master_stop_0 + * Resource action: redis stop on redis-bundle-0 +- * Resource action: redis-bundle-0 stop on undercloud + * Pseudo action: redis-bundle-master_stopped_0 ++ * Resource action: redis-bundle-0 stop on undercloud ++ * Pseudo action: redis-bundle-master_post_notify_stopped_0 + * Resource action: redis-bundle-docker-0 stop on undercloud ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_stopped_0 + * Pseudo action: redis-bundle_stopped_0 + * Pseudo action: haproxy-bundle_stop_0 + * Resource action: haproxy-bundle-docker-0 stop on undercloud +diff --git a/pengine/test10/bundle-order-startup-clone-2.dot b/pengine/test10/bundle-order-startup-clone-2.dot +index af90261..8726909 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.dot ++++ b/pengine/test10/bundle-order-startup-clone-2.dot +@@ -233,14 +233,48 @@ digraph "g" { + "redis-bundle-docker-2_start_0 metal-3" -> "redis:2_promote_0 redis-bundle-2" [ style = bold] + "redis-bundle-docker-2_start_0 metal-3" -> "redis:2_start_0 redis-bundle-2" [ style = bold] + "redis-bundle-docker-2_start_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_start_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" -> "redis:0_post_notify_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" -> "redis:1_post_notify_promote_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" -> "redis:2_post_notify_promote_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_running_0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis-bundle-master_post_notify_running_0" -> "redis:0_post_notify_start_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_running_0" -> "redis:1_post_notify_start_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_post_notify_running_0" -> "redis:2_post_notify_start_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" -> "redis:0_pre_notify_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" -> "redis:1_pre_notify_promote_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" -> "redis:2_pre_notify_promote_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_start_0" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold] ++"redis-bundle-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle-master_promote_0" -> "redis:0_promote_0 redis-bundle-0" [ style = bold] + "redis-bundle-master_promote_0" -> "redis:1_promote_0 redis-bundle-1" [ style = bold] + "redis-bundle-master_promote_0" -> "redis:2_promote_0 redis-bundle-2" [ style = bold] + "redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] +-"redis-bundle-master_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_promoted_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] + "redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] + "redis-bundle-master_running_0" -> "redis-bundle-master_promote_0" [ style = bold] +-"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold] + "redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold] + "redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold] +@@ -259,6 +293,12 @@ digraph "g" { + "redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] + "redis:0_monitor_20000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:0_post_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis:0_post_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:0_post_notify_start_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis:0_post_notify_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:0_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis:0_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis:0_promote_0 redis-bundle-0" -> "redis-bundle-master_promoted_0" [ style = bold] + "redis:0_promote_0 redis-bundle-0" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold] + "redis:0_promote_0 redis-bundle-0" -> "redis:1_promote_0 redis-bundle-1" [ style = bold] +@@ -269,6 +309,12 @@ digraph "g" { + "redis:0_start_0 redis-bundle-0" -> "redis:1_start_0 redis-bundle-1" [ style = bold] + "redis:0_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis:1_monitor_20000 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis:1_post_notify_promote_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis:1_post_notify_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis:1_post_notify_start_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis:1_post_notify_start_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis:1_pre_notify_promote_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis:1_pre_notify_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] + "redis:1_promote_0 redis-bundle-1" -> "redis-bundle-master_promoted_0" [ style = bold] + "redis:1_promote_0 redis-bundle-1" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold] + "redis:1_promote_0 redis-bundle-1" -> "redis:2_promote_0 redis-bundle-2" [ style = bold] +@@ -279,6 +325,12 @@ digraph "g" { + "redis:1_start_0 redis-bundle-1" -> "redis:2_start_0 redis-bundle-2" [ style = bold] + "redis:1_start_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] + "redis:2_monitor_20000 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis:2_post_notify_promote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis:2_post_notify_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis:2_post_notify_start_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis:2_post_notify_start_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis:2_pre_notify_promote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis:2_pre_notify_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] + "redis:2_promote_0 redis-bundle-2" -> "redis-bundle-master_promoted_0" [ style = bold] + "redis:2_promote_0 redis-bundle-2" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold] + "redis:2_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-order-startup-clone-2.exp b/pengine/test10/bundle-order-startup-clone-2.exp +index d05eb96..c0910a7 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.exp ++++ b/pengine/test10/bundle-order-startup-clone-2.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -103,7 +103,7 @@ + + + +- ++ + + + +@@ -163,13 +163,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +@@ -252,7 +252,7 @@ + + + +- ++ + + + +@@ -309,6 +309,147 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -320,7 +461,7 @@ + + + +- ++ + + + +@@ -360,7 +501,7 @@ + + + +- ++ + + + +@@ -369,7 +510,7 @@ + + + +- ++ + + + +@@ -378,7 +519,7 @@ + + + +- ++ + + + +@@ -387,7 +528,7 @@ + + + +- ++ + + + +@@ -400,7 +541,7 @@ + + + +- ++ + + + +@@ -413,61 +554,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -480,7 +567,7 @@ + + + +- ++ + + + +@@ -520,7 +607,7 @@ + + + +- ++ + + + +@@ -529,7 +616,7 @@ + + + +- ++ + + + +@@ -538,7 +625,7 @@ + + + +- ++ + + + +@@ -547,7 +634,7 @@ + + + +- ++ + + + +@@ -560,7 +647,7 @@ + + + +- ++ + + + +@@ -573,89 +660,35 @@ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- +- +- +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + +- ++ + +- +- +- +- +- +- +- +- +- +- + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -680,7 +713,7 @@ + + + +- ++ + + + +@@ -689,7 +722,7 @@ + + + +- ++ + + + +@@ -698,7 +731,7 @@ + + + +- ++ + + + +@@ -707,7 +740,7 @@ + + + +- ++ + + + +@@ -720,7 +753,7 @@ + + + +- ++ + + + +@@ -733,7 +766,7 @@ + + + +- ++ + + + +@@ -746,7 +779,7 @@ + + + +- ++ + + + +@@ -786,7 +819,7 @@ + + + +- ++ + + + +@@ -795,7 +828,7 @@ + + + +- ++ + + + +@@ -804,7 +837,7 @@ + + + +- ++ + + + +@@ -813,7 +846,7 @@ + + + +- ++ + + + +@@ -826,7 +859,7 @@ + + + +- ++ + + + +@@ -866,7 +899,7 @@ + + + +- ++ + + + +@@ -875,7 +908,7 @@ + + + +- ++ + + + +@@ -884,7 +917,7 @@ + + + +- ++ + + + +@@ -893,7 +926,7 @@ + + + +- ++ + + + +@@ -906,7 +939,7 @@ + + + +- ++ + + + +@@ -946,7 +979,7 @@ + + + +- ++ + + + +@@ -955,7 +988,7 @@ + + + +- ++ + + + +@@ -964,7 +997,7 @@ + + + +- ++ + + + +@@ -973,11 +1006,50 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + + +@@ -990,13 +1062,19 @@ + + + ++ ++ ++ ++ ++ ++ + + +- ++ + + + +- ++ + + + +@@ -1007,33 +1085,452 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- +- +- +- ++ + + +- ++ + + + +- ++ + + + +@@ -1046,7 +1543,7 @@ + + + +- ++ + + + +@@ -1086,7 +1583,7 @@ + + + +- ++ + + + +@@ -1095,7 +1592,7 @@ + + + +- ++ + + + +@@ -1104,7 +1601,7 @@ + + + +- ++ + + + +@@ -1113,7 +1610,7 @@ + + + +- ++ + + + +@@ -1126,7 +1623,7 @@ + + + +- ++ + + + +@@ -1139,73 +1636,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -1218,7 +1649,7 @@ + + + +- ++ + + + +@@ -1258,7 +1689,7 @@ + + + +- ++ + + + +@@ -1267,7 +1698,7 @@ + + + +- ++ + + + +@@ -1276,7 +1707,7 @@ + + + +- ++ + + + +@@ -1285,7 +1716,7 @@ + + + +- ++ + + + +@@ -1298,7 +1729,7 @@ + + + +- ++ + + + +@@ -1311,73 +1742,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -1390,7 +1755,7 @@ + + + +- ++ + + + +@@ -1430,7 +1795,7 @@ + + + +- ++ + + + +@@ -1439,7 +1804,7 @@ + + + +- ++ + + + +@@ -1448,7 +1813,7 @@ + + + +- ++ + + + +@@ -1457,7 +1822,7 @@ + + + +- ++ + + + +@@ -1470,7 +1835,7 @@ + + + +- ++ + + + +@@ -1483,21 +1848,21 @@ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -1507,73 +1872,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -1590,11 +1889,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -1602,7 +1901,7 @@ + + + +- ++ + + + +@@ -1620,7 +1919,7 @@ + + + +- ++ + + + +@@ -1628,40 +1927,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -1682,7 +1948,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-startup-clone-2.summary b/pengine/test10/bundle-order-startup-clone-2.summary +index 9b0eeaa..a2c2146 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.summary ++++ b/pengine/test10/bundle-order-startup-clone-2.summary +@@ -67,6 +67,7 @@ Executing cluster transition: + * Resource action: haproxy-bundle-docker-2 monitor on metal-3 + * Resource action: haproxy-bundle-docker-2 monitor on metal-2 + * Resource action: haproxy-bundle-docker-2 monitor on metal-1 ++ * Pseudo action: redis-bundle-master_pre_notify_start_0 + * Resource action: redis-bundle-docker-0 monitor on metal-3 + * Resource action: redis-bundle-docker-0 monitor on metal-2 + * Resource action: redis-bundle-docker-0 monitor on metal-1 +@@ -82,39 +83,55 @@ Executing cluster transition: + * Resource action: haproxy-bundle-docker-0 start on metal-1 + * Resource action: haproxy-bundle-docker-1 start on metal-2 + * Resource action: haproxy-bundle-docker-2 start on metal-3 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_start_0 ++ * Pseudo action: redis-bundle-master_start_0 + * Resource action: redis-bundle-docker-0 start on metal-1 + * Resource action: redis-bundle-0 start on metal-1 + * Resource action: redis-bundle-docker-1 start on metal-2 + * Resource action: redis-bundle-1 start on metal-2 + * Resource action: redis-bundle-docker-2 start on metal-3 + * Resource action: redis-bundle-2 start on metal-3 +- * Pseudo action: redis-bundle-master_start_0 + * Pseudo action: haproxy-bundle_running_0 + * Resource action: haproxy-bundle-docker-0 monitor=60000 on metal-1 + * Resource action: haproxy-bundle-docker-1 monitor=60000 on metal-2 + * Resource action: haproxy-bundle-docker-2 monitor=60000 on metal-3 + * Resource action: redis:0 start on redis-bundle-0 ++ * Resource action: redis:1 start on redis-bundle-1 ++ * Resource action: redis:2 start on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_running_0 + * Resource action: redis-bundle-docker-0 monitor=60000 on metal-1 + * Resource action: redis-bundle-0 monitor=60000 on metal-1 +- * Resource action: redis:1 start on redis-bundle-1 + * Resource action: redis-bundle-docker-1 monitor=60000 on metal-2 + * Resource action: redis-bundle-1 monitor=60000 on metal-2 +- * Resource action: redis:2 start on redis-bundle-2 + * Resource action: redis-bundle-docker-2 monitor=60000 on metal-3 + * Resource action: redis-bundle-2 monitor=60000 on metal-3 +- * Pseudo action: redis-bundle-master_running_0 ++ * Pseudo action: redis-bundle-master_post_notify_running_0 ++ * Resource action: redis:0 notify on redis-bundle-0 ++ * Resource action: redis:1 notify on redis-bundle-1 ++ * Resource action: redis:2 notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_running_0 + * Pseudo action: redis-bundle_running_0 ++ * Pseudo action: redis-bundle-master_pre_notify_promote_0 + * Pseudo action: redis-bundle_promote_0 ++ * Resource action: redis:0 notify on redis-bundle-0 ++ * Resource action: redis:1 notify on redis-bundle-1 ++ * Resource action: redis:2 notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0 + * Pseudo action: redis-bundle-master_promote_0 + * Resource action: redis:0 promote on redis-bundle-0 + * Resource action: redis:1 promote on redis-bundle-1 + * Resource action: redis:2 promote on redis-bundle-2 + * Pseudo action: redis-bundle-master_promoted_0 ++ * Pseudo action: redis-bundle-master_post_notify_promoted_0 ++ * Resource action: redis:0 notify on redis-bundle-0 ++ * Resource action: redis:1 notify on redis-bundle-1 ++ * Resource action: redis:2 notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0 ++ * Pseudo action: redis-bundle_promoted_0 ++ * Pseudo action: storage-clone_start_0 + * Resource action: redis:0 monitor=20000 on redis-bundle-0 + * Resource action: redis:1 monitor=20000 on redis-bundle-1 + * Resource action: redis:2 monitor=20000 on redis-bundle-2 +- * Pseudo action: redis-bundle_promoted_0 +- * Pseudo action: storage-clone_start_0 + * Resource action: storage:0 start on metal-1 + * Resource action: storage:1 start on metal-2 + * Resource action: storage:2 start on metal-3 +@@ -128,23 +145,23 @@ Executing cluster transition: + * Resource action: storage:0 monitor=30000 on metal-1 + * Resource action: storage:1 monitor=30000 on metal-2 + * Resource action: storage:2 monitor=30000 on metal-3 ++ * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera-bundle-docker-0 start on metal-1 + * Resource action: galera-bundle-0 start on metal-1 + * Resource action: galera-bundle-docker-1 start on metal-2 + * Resource action: galera-bundle-1 start on metal-2 + * Resource action: galera-bundle-docker-2 start on metal-3 + * Resource action: galera-bundle-2 start on metal-3 +- * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera:0 start on galera-bundle-0 ++ * Resource action: galera:1 start on galera-bundle-1 ++ * Resource action: galera:2 start on galera-bundle-2 ++ * Pseudo action: galera-bundle-master_running_0 + * Resource action: galera-bundle-docker-0 monitor=60000 on metal-1 + * Resource action: galera-bundle-0 monitor=60000 on metal-1 +- * Resource action: galera:1 start on galera-bundle-1 + * Resource action: galera-bundle-docker-1 monitor=60000 on metal-2 + * Resource action: galera-bundle-1 monitor=60000 on metal-2 +- * Resource action: galera:2 start on galera-bundle-2 + * Resource action: galera-bundle-docker-2 monitor=60000 on metal-3 + * Resource action: galera-bundle-2 monitor=60000 on metal-3 +- * Pseudo action: galera-bundle-master_running_0 + * Pseudo action: galera-bundle_running_0 + * Resource action: galera:0 monitor=30000 on galera-bundle-0 + * Resource action: galera:0 monitor=20000 on galera-bundle-0 +diff --git a/pengine/test10/bundle-order-startup-clone.dot b/pengine/test10/bundle-order-startup-clone.dot +index 92f019f..219b676 100644 +--- a/pengine/test10/bundle-order-startup-clone.dot ++++ b/pengine/test10/bundle-order-startup-clone.dot +@@ -64,7 +64,19 @@ digraph "g" { + "redis-bundle-docker-0_start_0 metal-2" -> "redis-bundle_running_0" [ style = bold] + "redis-bundle-docker-0_start_0 metal-2" -> "redis:0_start_0 redis-bundle-0" [ style = bold] + "redis-bundle-docker-0_start_0 metal-2" [ style=bold color="green" fontcolor="black"] +-"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_start_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_running_0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis-bundle-master_post_notify_running_0" -> "redis:0_post_notify_start_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_start_0" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold] ++"redis-bundle-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] + "redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold] + "redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold] +@@ -75,6 +87,8 @@ digraph "g" { + "redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] + "redis:0_monitor_45000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis:0_monitor_60000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:0_post_notify_start_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis:0_post_notify_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis:0_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = bold] + "redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold] + "redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup-clone.exp b/pengine/test10/bundle-order-startup-clone.exp +index c736cb9..197529f 100644 +--- a/pengine/test10/bundle-order-startup-clone.exp ++++ b/pengine/test10/bundle-order-startup-clone.exp +@@ -115,11 +115,24 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + + +@@ -129,13 +142,16 @@ + + + ++ ++ ++ + + +- ++ + + + +- ++ + + + +@@ -145,13 +161,16 @@ + + + ++ ++ ++ + + +- ++ + + + +- ++ + + + +@@ -166,7 +185,87 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -179,7 +278,7 @@ + + + +- ++ + + + +@@ -201,7 +300,7 @@ + + + +- ++ + + + +@@ -210,7 +309,7 @@ + + + +- ++ + + + +@@ -219,7 +318,7 @@ + + + +- ++ + + + +@@ -228,7 +327,7 @@ + + + +- ++ + + + +@@ -241,7 +340,7 @@ + + + +- ++ + + + +@@ -254,34 +353,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -292,11 +364,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -304,7 +376,7 @@ + + + +- ++ + + + +@@ -316,7 +388,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-startup-clone.summary b/pengine/test10/bundle-order-startup-clone.summary +index 5e826dc..c2b46de 100644 +--- a/pengine/test10/bundle-order-startup-clone.summary ++++ b/pengine/test10/bundle-order-startup-clone.summary +@@ -34,21 +34,26 @@ Executing cluster transition: + * Resource action: haproxy-bundle-docker-0 monitor on metal-3 + * Resource action: haproxy-bundle-docker-0 monitor on metal-2 + * Resource action: haproxy-bundle-docker-0 monitor on metal-1 ++ * Pseudo action: redis-bundle-master_pre_notify_start_0 + * Resource action: redis-bundle-docker-0 monitor on metal-3 + * Resource action: redis-bundle-docker-0 monitor on metal-2 + * Resource action: redis-bundle-docker-0 monitor on metal-1 + * Pseudo action: redis-bundle_start_0 + * Pseudo action: haproxy-bundle_start_0 + * Resource action: haproxy-bundle-docker-0 start on metal-2 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_start_0 ++ * Pseudo action: redis-bundle-master_start_0 + * Resource action: redis-bundle-docker-0 start on metal-2 + * Resource action: redis-bundle-0 start on metal-2 +- * Pseudo action: redis-bundle-master_start_0 + * Pseudo action: haproxy-bundle_running_0 + * Resource action: haproxy-bundle-docker-0 monitor=60000 on metal-2 + * Resource action: redis:0 start on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_running_0 + * Resource action: redis-bundle-docker-0 monitor=60000 on metal-2 + * Resource action: redis-bundle-0 monitor=60000 on metal-2 +- * Pseudo action: redis-bundle-master_running_0 ++ * Pseudo action: redis-bundle-master_post_notify_running_0 ++ * Resource action: redis:0 notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_running_0 + * Pseudo action: redis-bundle_running_0 + * Resource action: redis:0 monitor=60000 on redis-bundle-0 + * Resource action: redis:0 monitor=45000 on redis-bundle-0 +diff --git a/pengine/test10/bundle-order-startup.dot b/pengine/test10/bundle-order-startup.dot +index 66053b9..3ec10fa 100644 +--- a/pengine/test10/bundle-order-startup.dot ++++ b/pengine/test10/bundle-order-startup.dot +@@ -88,7 +88,18 @@ digraph "g" { + "rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] +-"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] + "rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] +@@ -106,6 +117,8 @@ digraph "g" { + "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-0_start_0 undercloud" [ style = bold] + "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] +@@ -123,7 +136,19 @@ digraph "g" { + "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle_running_0" [ style = bold] + "redis-bundle-docker-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold] + "redis-bundle-docker-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] +-"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_start_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_running_0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis-bundle-master_post_notify_running_0" -> "redis:0_post_notify_start_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_start_0" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold] ++"redis-bundle-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] + "redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold] + "redis-bundle-master_start_0" -> "redis:0_start_0 redis-bundle-0" [ style = bold] +@@ -135,6 +160,8 @@ digraph "g" { + "redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] + "redis:0_monitor_45000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis:0_monitor_60000 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis:0_post_notify_start_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis:0_post_notify_start_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis:0_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = bold] + "redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold] + "redis:0_start_0 redis-bundle-0" -> "redis:0_monitor_60000 redis-bundle-0" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp +index 2128d7c..161c1ae 100644 +--- a/pengine/test10/bundle-order-startup.exp ++++ b/pengine/test10/bundle-order-startup.exp +@@ -1,9 +1,22 @@ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + + +@@ -13,13 +26,16 @@ + + + ++ ++ ++ + + +- ++ + + + +- ++ + + + +@@ -34,7 +50,87 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -47,7 +143,7 @@ + + + +- ++ + + + +@@ -63,7 +159,7 @@ + + + +- ++ + + + +@@ -72,7 +168,7 @@ + + + +- ++ + + + +@@ -85,7 +181,7 @@ + + + +- ++ + + + +@@ -98,73 +194,100 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -174,11 +297,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -187,99 +310,198 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + +- ++ + + + + +- ++ ++ ++ ++ + + +- ++ + + + +- ++ + +- ++ + +- ++ + + + + +- ++ ++ ++ ++ + + +- ++ + + + +- ++ + +- ++ + +- ++ + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -289,11 +511,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -302,48 +524,48 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -354,7 +576,7 @@ + + + +- ++ + + + +@@ -363,22 +585,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -389,7 +611,7 @@ + + + +- ++ + + + +@@ -398,22 +620,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -424,7 +646,7 @@ + + + +- ++ + + + +@@ -433,22 +655,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -459,7 +681,7 @@ + + + +- ++ + + + +@@ -468,22 +690,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -494,7 +716,7 @@ + + + +- ++ + + + +@@ -503,22 +725,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -529,7 +751,7 @@ + + + +- ++ + + + +@@ -538,22 +760,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -563,11 +785,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -576,22 +798,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -601,11 +823,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -614,201 +836,120 @@ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- +- +- +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + +- +- +- +- +- +- +- +- +- + +- +- +- +- ++ + + + +- ++ + +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ + + + +@@ -819,11 +960,11 @@ + + + +- ++ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-startup.summary b/pengine/test10/bundle-order-startup.summary +index b5c2091..8da71d0 100644 +--- a/pengine/test10/bundle-order-startup.summary ++++ b/pengine/test10/bundle-order-startup.summary +@@ -39,8 +39,10 @@ Transition Summary: + * Start openstack-cinder-volume-docker-0 (undercloud) + + Executing cluster transition: ++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 + * Resource action: rabbitmq-bundle-docker-0 monitor on undercloud + * Resource action: galera-bundle-docker-0 monitor on undercloud ++ * Pseudo action: redis-bundle-master_pre_notify_start_0 + * Resource action: redis-bundle-docker-0 monitor on undercloud + * Resource action: ip-192.168.122.254 monitor on undercloud + * Resource action: ip-192.168.122.250 monitor on undercloud +@@ -52,8 +54,11 @@ Executing cluster transition: + * Resource action: openstack-cinder-volume-docker-0 monitor on undercloud + * Pseudo action: openstack-cinder-volume_start_0 + * Pseudo action: rabbitmq-bundle_start_0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 + * Resource action: rabbitmq-bundle-docker-0 start on undercloud + * Resource action: rabbitmq-bundle-0 start on undercloud ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_start_0 + * Resource action: ip-192.168.122.254 start on undercloud + * Resource action: ip-192.168.122.250 start on undercloud + * Resource action: ip-192.168.122.249 start on undercloud +@@ -63,8 +68,8 @@ Executing cluster transition: + * Resource action: openstack-cinder-volume-docker-0 start on undercloud + * Pseudo action: openstack-cinder-volume_running_0 + * Pseudo action: haproxy-bundle_start_0 +- * Pseudo action: rabbitmq-bundle-clone_start_0 + * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_running_0 + * Resource action: rabbitmq-bundle-docker-0 monitor=60000 on undercloud + * Resource action: rabbitmq-bundle-0 monitor=60000 on undercloud + * Resource action: ip-192.168.122.254 monitor=10000 on undercloud +@@ -77,28 +82,33 @@ Executing cluster transition: + * Resource action: openstack-cinder-volume-docker-0 monitor=60000 on undercloud + * Pseudo action: haproxy-bundle_running_0 + * Pseudo action: redis-bundle_start_0 +- * Pseudo action: rabbitmq-bundle-clone_running_0 +- * Pseudo action: rabbitmq-bundle_running_0 +- * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 ++ * Pseudo action: redis-bundle-master_start_0 + * Resource action: redis-bundle-docker-0 start on undercloud + * Resource action: redis-bundle-0 start on undercloud + * Resource action: haproxy-bundle-docker-0 monitor=60000 on undercloud +- * Pseudo action: redis-bundle-master_start_0 ++ * Resource action: rabbitmq:0 notify on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 + * Resource action: redis:0 start on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_running_0 + * Resource action: redis-bundle-docker-0 monitor=60000 on undercloud + * Resource action: redis-bundle-0 monitor=60000 on undercloud +- * Pseudo action: redis-bundle-master_running_0 ++ * Pseudo action: rabbitmq-bundle_running_0 ++ * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 ++ * Pseudo action: redis-bundle-master_post_notify_running_0 ++ * Resource action: redis:0 notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_running_0 + * Pseudo action: redis-bundle_running_0 + * Pseudo action: galera-bundle_start_0 ++ * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera-bundle-docker-0 start on undercloud + * Resource action: galera-bundle-0 start on undercloud + * Resource action: redis:0 monitor=60000 on redis-bundle-0 + * Resource action: redis:0 monitor=45000 on redis-bundle-0 +- * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera:0 start on galera-bundle-0 ++ * Pseudo action: galera-bundle-master_running_0 + * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud + * Resource action: galera-bundle-0 monitor=60000 on undercloud +- * Pseudo action: galera-bundle-master_running_0 + * Pseudo action: galera-bundle_running_0 + * Resource action: galera:0 monitor=30000 on galera-bundle-0 + * Resource action: galera:0 monitor=20000 on galera-bundle-0 +diff --git a/pengine/test10/bundle-order-stop-clone.exp b/pengine/test10/bundle-order-stop-clone.exp +index ac0ae05..c359e00 100644 +--- a/pengine/test10/bundle-order-stop-clone.exp ++++ b/pengine/test10/bundle-order-stop-clone.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +@@ -40,7 +40,7 @@ + + + +- ++ + + + +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -66,7 +66,7 @@ + + + +- ++ + + + +@@ -88,10 +88,10 @@ + + + +- ++ + + +- ++ + + + +@@ -121,13 +121,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +@@ -185,89 +185,89 @@ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ + + +- ++ + + + + + +- +- +- +- +- +- +- ++ ++ ++ + + + +- ++ + + + + + +- ++ + + + + + +- +- +- +- ++ + + + + + +- ++ + + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ + + + + + +- +- +- ++ ++ ++ ++ ++ ++ ++ + + + +- +- +- +- ++ + + + +diff --git a/pengine/test10/bundle-order-stop-clone.summary b/pengine/test10/bundle-order-stop-clone.summary +index 66cb82a..70e0f21 100644 +--- a/pengine/test10/bundle-order-stop-clone.summary ++++ b/pengine/test10/bundle-order-stop-clone.summary +@@ -35,8 +35,8 @@ Executing cluster transition: + * Pseudo action: storage-clone_confirmed-pre_notify_stop_0 + * Pseudo action: galera-bundle-master_stop_0 + * Resource action: galera:0 stop on galera-bundle-0 +- * Resource action: galera-bundle-0 stop on metal-1 + * Pseudo action: galera-bundle-master_stopped_0 ++ * Resource action: galera-bundle-0 stop on metal-1 + * Resource action: galera-bundle-docker-0 stop on metal-1 + * Pseudo action: galera-bundle_stopped_0 + * Pseudo action: galera-bundle_start_0 +diff --git a/pengine/test10/bundle-order-stop.dot b/pengine/test10/bundle-order-stop.dot +index c0e6616..3fade95 100644 +--- a/pengine/test10/bundle-order-stop.dot ++++ b/pengine/test10/bundle-order-stop.dot +@@ -118,7 +118,29 @@ digraph "g" { + "rabbitmq-bundle-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] + "rabbitmq-bundle-0_stop_0 undercloud" -> "rabbitmq-bundle-docker-0_stop_0 undercloud" [ style = bold] + "rabbitmq-bundle-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] +-"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle-clone_pre_notify_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = dashed] ++"rabbitmq-bundle-clone_post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = dashed] ++"rabbitmq-bundle-clone_pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = dashed] + "rabbitmq-bundle-clone_running_0" [ style=dashed color="red" fontcolor="orange"] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] +@@ -126,8 +148,8 @@ digraph "g" { + "rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] + "rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] + "rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = dashed] +-"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] + "rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "all_stopped" [ style = bold] + "rabbitmq-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] +@@ -140,6 +162,8 @@ digraph "g" { + "rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] + "rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] + "rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] +@@ -156,14 +180,52 @@ digraph "g" { + "redis-bundle-docker-0_stop_0 undercloud" -> "do_shutdown undercloud" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" -> "redis-bundle_stopped_0" [ style = bold] + "redis-bundle-docker-0_stop_0 undercloud" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_start_0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_stop_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle-master_pre_notify_start_0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle_stopped_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_demote_0" -> "redis-bundle-master_post_notify_demoted_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_post_notify_running_0" [ style = dashed] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_start_0" [ style = dashed] ++"redis-bundle-master_confirmed-pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_stop_0" -> "redis-bundle-master_post_notify_stopped_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] + "redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] + "redis-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_post_notify_demoted_0" [ style = bold] + "redis-bundle-master_demoted_0" -> "redis-bundle-master_start_0" [ style = dashed] + "redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold] +-"redis-bundle-master_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] + "redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] +-"redis-bundle-master_running_0" -> "redis-bundle_running_0" [ style = dashed] ++"redis-bundle-master_post_notify_demoted_0" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] ++"redis-bundle-master_post_notify_demoted_0" -> "redis_post_notify_demoted_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_running_0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = dashed] ++"redis-bundle-master_post_notify_running_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_post_notify_stopped_0" -> "redis-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] ++"redis-bundle-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_demote_0" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] ++"redis-bundle-master_pre_notify_demote_0" -> "redis_pre_notify_demote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_start_0" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = dashed] ++"redis-bundle-master_pre_notify_start_0" [ style=dashed color="red" fontcolor="orange"] ++"redis-bundle-master_pre_notify_stop_0" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] ++"redis-bundle-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" -> "redis-bundle-master_post_notify_running_0" [ style = dashed] + "redis-bundle-master_running_0" [ style=dashed color="red" fontcolor="orange"] + "redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = dashed] + "redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed] +@@ -171,8 +233,8 @@ digraph "g" { + "redis-bundle-master_stop_0" -> "redis-bundle-master_stopped_0" [ style = bold] + "redis-bundle-master_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] + "redis-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_stopped_0" -> "redis-bundle-master_post_notify_stopped_0" [ style = bold] + "redis-bundle-master_stopped_0" -> "redis-bundle-master_start_0" [ style = dashed] +-"redis-bundle-master_stopped_0" -> "redis-bundle_stopped_0" [ style = bold] + "redis-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] + "redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] + "redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold] +@@ -199,6 +261,12 @@ digraph "g" { + "redis_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis_monitor_45000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] + "redis_monitor_60000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_post_notify_demoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] ++"redis_post_notify_demoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_demote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] ++"redis_pre_notify_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_stop_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] ++"redis_pre_notify_stop_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] + "redis_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = dashed] + "redis_start_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] + "redis_start_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] +diff --git a/pengine/test10/bundle-order-stop.exp b/pengine/test10/bundle-order-stop.exp +index 27ae320..937bc59 100644 +--- a/pengine/test10/bundle-order-stop.exp ++++ b/pengine/test10/bundle-order-stop.exp +@@ -1,9 +1,22 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ + + + +@@ -11,11 +24,91 @@ + + + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -31,7 +124,7 @@ + + + +- ++ + + + +@@ -47,28 +140,28 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -78,11 +171,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -91,567 +184,692 @@ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ + + +- ++ + + + +- ++ + +- +- +- +- +- +- +- ++ ++ ++ + + + +- +- +- +- ++ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ + + +- +- +- +- ++ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ + + +- ++ + + + +- +- +- +- +- +- +- +- +- +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ + + +- +- +- +- +- ++ + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- +- ++ ++ ++ + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- +- +- +- ++ ++ ++ + + + +- +- +- +- +- +- +- ++ + + +- +- +- +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + + +- ++ + +- +- ++ ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + +- +- ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- +- +- +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- +- +- +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ + + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + +- +- +- +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- +- ++ ++ + + + + +- +- +- +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ + + + +- ++ + +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + +- ++ + + +- ++ + + + +- ++ + +- +- ++ ++ + + + + +- ++ + + + +- ++ + + + +@@ -662,11 +880,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -674,7 +892,7 @@ + + + +- ++ + + + +@@ -691,46 +909,52 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-stop.summary b/pengine/test10/bundle-order-stop.summary +index 7bc24d2..29cf490 100644 +--- a/pengine/test10/bundle-order-stop.summary ++++ b/pengine/test10/bundle-order-stop.summary +@@ -41,41 +41,57 @@ Transition Summary: + * Stop openstack-cinder-volume-docker-0 (undercloud) due to node availability + + Executing cluster transition: ++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_stop_0 + * Resource action: galera cancel=10000 on galera-bundle-0 + * Resource action: redis cancel=20000 on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_pre_notify_demote_0 + * Pseudo action: openstack-cinder-volume_stop_0 + * Pseudo action: redis-bundle_demote_0 +- * Pseudo action: redis-bundle-master_demote_0 + * Pseudo action: galera-bundle_demote_0 +- * Pseudo action: galera-bundle-master_demote_0 + * Pseudo action: rabbitmq-bundle_stop_0 +- * Resource action: galera demote on galera-bundle-0 +- * Resource action: redis demote on redis-bundle-0 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_stop_0 ++ * Pseudo action: rabbitmq-bundle-clone_stop_0 ++ * Pseudo action: galera-bundle-master_demote_0 ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_demote_0 ++ * Pseudo action: redis-bundle-master_demote_0 + * Resource action: openstack-cinder-volume-docker-0 stop on undercloud + * Pseudo action: openstack-cinder-volume_stopped_0 +- * Pseudo action: redis-bundle-master_demoted_0 +- * Pseudo action: galera-bundle-master_demoted_0 +- * Pseudo action: rabbitmq-bundle-clone_stop_0 + * Resource action: rabbitmq stop on rabbitmq-bundle-0 ++ * Pseudo action: rabbitmq-bundle-clone_stopped_0 + * Resource action: rabbitmq-bundle-0 stop on undercloud +- * Pseudo action: redis-bundle_demoted_0 ++ * Resource action: galera demote on galera-bundle-0 ++ * Pseudo action: galera-bundle-master_demoted_0 ++ * Resource action: redis demote on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_demoted_0 + * Pseudo action: galera-bundle_demoted_0 + * Pseudo action: galera-bundle_stop_0 +- * Pseudo action: rabbitmq-bundle-clone_stopped_0 ++ * Pseudo action: rabbitmq-bundle-clone_post_notify_stopped_0 + * Resource action: rabbitmq-bundle-docker-0 stop on undercloud + * Pseudo action: galera-bundle-master_stop_0 +- * Pseudo action: rabbitmq-bundle_stopped_0 ++ * Pseudo action: redis-bundle-master_post_notify_demoted_0 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_stopped_0 + * Resource action: galera stop on galera-bundle-0 +- * Resource action: galera-bundle-0 stop on undercloud + * Pseudo action: galera-bundle-master_stopped_0 ++ * Resource action: galera-bundle-0 stop on undercloud ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_demoted_0 ++ * Pseudo action: redis-bundle-master_pre_notify_stop_0 ++ * Pseudo action: redis-bundle_demoted_0 ++ * Pseudo action: rabbitmq-bundle_stopped_0 + * Resource action: galera-bundle-docker-0 stop on undercloud ++ * Resource action: redis notify on redis-bundle-0 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_stop_0 + * Pseudo action: galera-bundle_stopped_0 + * Pseudo action: redis-bundle_stop_0 + * Pseudo action: redis-bundle-master_stop_0 + * Resource action: redis stop on redis-bundle-0 +- * Resource action: redis-bundle-0 stop on undercloud + * Pseudo action: redis-bundle-master_stopped_0 ++ * Resource action: redis-bundle-0 stop on undercloud ++ * Pseudo action: redis-bundle-master_post_notify_stopped_0 + * Resource action: redis-bundle-docker-0 stop on undercloud ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_stopped_0 + * Pseudo action: redis-bundle_stopped_0 + * Pseudo action: haproxy-bundle_stop_0 + * Resource action: haproxy-bundle-docker-0 stop on undercloud +diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp +index 5bf7eb3..b7bb735 100644 +--- a/pengine/test10/remote-fence-unclean-3.exp ++++ b/pengine/test10/remote-fence-unclean-3.exp +@@ -75,7 +75,7 @@ + + + +- ++ + + + +@@ -107,7 +107,7 @@ + + + +- ++ + + + +-- +1.8.3.1 + diff --git a/SOURCES/090-remote-fence-loop.patch b/SOURCES/090-remote-fence-loop.patch new file mode 100644 index 0000000..5f81e6f --- /dev/null +++ b/SOURCES/090-remote-fence-loop.patch @@ -0,0 +1,128 @@ +From 00fed62ed4899412f65644e6b84e49fbf89d09bb Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 2 Aug 2017 19:18:25 -0500 +Subject: [PATCH] Fix: pengine: avoid fence loop for remote nodes + +This leaves the original issue unaddressed, for investigation later. +--- + pengine/allocate.c | 3 +++ + pengine/test10/remote-fence-unclean-3.dot | 5 ----- + pengine/test10/remote-fence-unclean-3.exp | 29 --------------------------- + pengine/test10/remote-fence-unclean-3.summary | 3 --- + 4 files changed, 3 insertions(+), 37 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 3a883ad..e3cb4cc 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -905,10 +905,13 @@ probe_resources(pe_working_set_t * data_set) + continue; + + } else if (node->details->online == FALSE && node->details->remote_rsc) { ++ // TODO figure out why this results in fence loop ++ /* + enum remote_connection_state state = get_remote_node_state(node); + if(state == remote_state_failed) { + pe_fence_node(data_set, node, "the connection is unrecoverable"); + } ++ */ + continue; + + } else if(node->details->online == FALSE) { +diff --git a/pengine/test10/remote-fence-unclean-3.dot b/pengine/test10/remote-fence-unclean-3.dot +index b32b77e..14adaef 100644 +--- a/pengine/test10/remote-fence-unclean-3.dot ++++ b/pengine/test10/remote-fence-unclean-3.dot +@@ -1,5 +1,4 @@ + digraph "g" { +-"all_stopped" -> "fence1_start_0 overcloud-controller-0" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "fence1_monitor_0 overcloud-controller-0" -> "fence1_start_0 overcloud-controller-0" [ style = bold] + "fence1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] +@@ -12,8 +11,4 @@ digraph "g" { + "fence1_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] + "overcloud-novacompute-0_stop_0 overcloud-controller-0" -> "all_stopped" [ style = bold] + "overcloud-novacompute-0_stop_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] +-"stonith 'reboot' overcloud-novacompute-0" -> "stonith_complete" [ style = bold] +-"stonith 'reboot' overcloud-novacompute-0" [ style=bold color="green" fontcolor="black"] +-"stonith_complete" -> "all_stopped" [ style = bold] +-"stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp +index 2e341bd..64e5a62 100644 +--- a/pengine/test10/remote-fence-unclean-3.exp ++++ b/pengine/test10/remote-fence-unclean-3.exp +@@ -21,9 +21,6 @@ + + + +- +- +- + + + +@@ -75,29 +72,6 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- + + + +@@ -106,9 +80,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary +index ec24500..ec54d8e 100644 +--- a/pengine/test10/remote-fence-unclean-3.summary ++++ b/pengine/test10/remote-fence-unclean-3.summary +@@ -34,7 +34,6 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund + openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1 + + Transition Summary: +- * Fence (reboot) overcloud-novacompute-0 'the connection is unrecoverable' + * Start fence1 (overcloud-controller-0) + * Stop overcloud-novacompute-0 (overcloud-controller-0) + +@@ -43,8 +42,6 @@ Executing cluster transition: + * Resource action: fence1 monitor on overcloud-controller-1 + * Resource action: fence1 monitor on overcloud-controller-0 + * Resource action: overcloud-novacompute-0 stop on overcloud-controller-0 +- * Fencing overcloud-novacompute-0 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: all_stopped + * Resource action: fence1 start on overcloud-controller-0 + * Resource action: fence1 monitor=60000 on overcloud-controller-0 +-- +1.8.3.1 + diff --git a/SOURCES/091-compat.patch b/SOURCES/091-compat.patch new file mode 100644 index 0000000..789b933 --- /dev/null +++ b/SOURCES/091-compat.patch @@ -0,0 +1,316 @@ +From cef0ff51879e965c3b7d214a23b61e6bf61aa33d Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 17 Aug 2017 15:53:27 -0500 +Subject: [PATCH 1/3] Refactor: crmd,pengine: prefix internal node attributes + with '#' + +so crm_mon will hide them. Also, define constants for them. + +As a positive side effect, node_unfenced is no longer added as a +meta-attribute to stonith "on" operations. +--- + crmd/te_callbacks.c | 14 +++++--------- + include/crm/crm.h | 5 +++++ + include/crm/msg_xml.h | 3 ++- + lib/pengine/utils.c | 12 ++++++++---- + pengine/native.c | 4 ++-- + pengine/test10/start-then-stop-with-unfence.xml | 2 +- + pengine/test10/unfence-definition.exp | 2 +- + pengine/test10/unfence-definition.xml | 4 ++-- + pengine/test10/unfence-parameters.exp | 2 +- + pengine/test10/unfence-parameters.xml | 4 ++-- + pengine/test10/unfence-startup.xml | 4 ++-- + 11 files changed, 31 insertions(+), 25 deletions(-) + +diff --git a/crmd/te_callbacks.c b/crmd/te_callbacks.c +index 4476d4a..2273b86 100644 +--- a/crmd/te_callbacks.c ++++ b/crmd/te_callbacks.c +@@ -798,21 +797,17 @@ tengine_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data) + if (action->confirmed == FALSE) { + te_action_confirmed(action); + if (safe_str_eq("on", op)) { +- const char *key = NULL; + const char *value = NULL; + char *now = crm_itoa(time(NULL)); + +- key = XML_NODE_IS_UNFENCED; +- update_attrd(target, key, now, NULL, FALSE); ++ update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, FALSE); + free(now); + +- key = "digests-all"; +- value = crm_meta_value(action->params, key); +- update_attrd(target, key, value, NULL, FALSE); ++ value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL); ++ update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, FALSE); + +- key = "digests-secure"; +- value = crm_meta_value(action->params, key); +- update_attrd(target, key, value, NULL, FALSE); ++ value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE); ++ update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, FALSE); + + } else if (action->sent_update == FALSE) { + send_stonith_update(action, target, uuid); +diff --git a/include/crm/crm.h b/include/crm/crm.h +index 0da6bfe..35ab6ca 100644 +--- a/include/crm/crm.h ++++ b/include/crm/crm.h +@@ -94,6 +94,11 @@ extern char *crm_system_name; + # define CRM_SYSTEM_STONITHD "stonithd" + # define CRM_SYSTEM_MCP "pacemakerd" + ++// Names of internally generated node attributes ++# define CRM_ATTR_UNFENCED "#node-unfenced" ++# define CRM_ATTR_DIGESTS_ALL "#digests-all" ++# define CRM_ATTR_DIGESTS_SECURE "#digests-secure" ++ + /* Valid operations */ + # define CRM_OP_NOOP "noop" + +diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h +index 8cf22f3..38183b5 100644 +--- a/include/crm/msg_xml.h ++++ b/include/crm/msg_xml.h +@@ -228,6 +234,8 @@ + # define XML_OP_ATTR_DEPENDENT "dependent-on" + # define XML_OP_ATTR_ORIGIN "interval-origin" + # define XML_OP_ATTR_PENDING "record-pending" ++# define XML_OP_ATTR_DIGESTS_ALL "digests-all" ++# define XML_OP_ATTR_DIGESTS_SECURE "digests-secure" + + # define XML_CIB_TAG_LRM "lrm" + # define XML_LRM_TAG_RESOURCES "lrm_resources" +@@ -254,7 +264,6 @@ + # define XML_NODE_IS_PEER "crmd" + # define XML_NODE_IS_REMOTE "remote_node" + # define XML_NODE_IS_FENCED "node_fenced" +-# define XML_NODE_IS_UNFENCED "node_unfenced" + # define XML_NODE_IS_MAINTENANCE "node_in_maintenance" + + # define XML_CIB_ATTR_SHUTDOWN "shutdown" +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index cb1ed08..848b311 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -2115,8 +1987,10 @@ fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * da + char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); + op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, node, NULL, data_set); + +- const char *digest_all = g_hash_table_lookup(node->details->attrs, "digests-all"); +- const char *digest_secure = g_hash_table_lookup(node->details->attrs, "digests-secure"); ++ const char *digest_all = g_hash_table_lookup(node->details->attrs, ++ CRM_ATTR_DIGESTS_ALL); ++ const char *digest_secure = g_hash_table_lookup(node->details->attrs, ++ CRM_ATTR_DIGESTS_SECURE); + + /* No 'reloads' for fencing device changes + * +@@ -2269,8 +2146,10 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + digests_secure+digests_secure_offset, max-digests_secure_offset, + "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_secure_calc); + } +- add_hash_param(stonith_op->meta, strdup("digests-all"), digests_all); +- add_hash_param(stonith_op->meta, strdup("digests-secure"), digests_secure); ++ add_hash_param(stonith_op->meta, strdup(XML_OP_ATTR_DIGESTS_ALL), ++ digests_all); ++ add_hash_param(stonith_op->meta, strdup(XML_OP_ATTR_DIGESTS_SECURE), ++ digests_secure); + } + + } else { +diff --git a/pengine/native.c b/pengine/native.c +index fdbbf0c..18efa1d 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -2490,7 +2501,7 @@ StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * d + + if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { + action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, data_set); +- const char *unfenced = g_hash_table_lookup(current->details->attrs, XML_NODE_IS_UNFENCED); ++ const char *unfenced = g_hash_table_lookup(current->details->attrs, CRM_ATTR_UNFENCED); + + order_actions(stop, unfence, pe_order_implies_first); + if (unfenced == NULL || safe_str_eq("0", unfenced)) { +@@ -2513,7 +2524,7 @@ StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * + + if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { + action_t *unfence = pe_fence_op(next, "on", TRUE, NULL, data_set); +- const char *unfenced = g_hash_table_lookup(next->details->attrs, XML_NODE_IS_UNFENCED); ++ const char *unfenced = g_hash_table_lookup(next->details->attrs, CRM_ATTR_UNFENCED); + + order_actions(unfence, start, pe_order_implies_then); + +diff --git a/pengine/test10/start-then-stop-with-unfence.xml b/pengine/test10/start-then-stop-with-unfence.xml +index b856250..587b05a 100644 +--- a/pengine/test10/start-then-stop-with-unfence.xml ++++ b/pengine/test10/start-then-stop-with-unfence.xml +@@ -95,7 +95,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unfence-definition.exp b/pengine/test10/unfence-definition.exp +index f9ad4c1..4e619fe 100644 +--- a/pengine/test10/unfence-definition.exp ++++ b/pengine/test10/unfence-definition.exp +@@ -389,7 +389,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unfence-definition.xml b/pengine/test10/unfence-definition.xml +index cb189cd..90aa3da 100644 +--- a/pengine/test10/unfence-definition.xml ++++ b/pengine/test10/unfence-definition.xml +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -63,7 +63,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unfence-parameters.exp b/pengine/test10/unfence-parameters.exp +index 64286a8..8022591 100644 +--- a/pengine/test10/unfence-parameters.exp ++++ b/pengine/test10/unfence-parameters.exp +@@ -373,7 +373,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unfence-parameters.xml b/pengine/test10/unfence-parameters.xml +index 26588bf..e1975b9 100644 +--- a/pengine/test10/unfence-parameters.xml ++++ b/pengine/test10/unfence-parameters.xml +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -63,7 +63,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unfence-startup.xml b/pengine/test10/unfence-startup.xml +index 28637e3..69e2df7 100644 +--- a/pengine/test10/unfence-startup.xml ++++ b/pengine/test10/unfence-startup.xml +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -63,7 +63,7 @@ + + + +- ++ + + + +-- +1.8.3.1 + + +From 8440151215e39a69eea55aae996329a11d7800a5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 17 Aug 2017 17:38:30 -0500 +Subject: [PATCH 2/3] Refactor: libpe_status: remove redundant constant + definition + +--- + lib/pengine/utils.c | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 848b311..b5794fb 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -2211,8 +2083,6 @@ find_unfencing_devices(GListPtr candidates, GListPtr matches) + } + + +-#define STONITH_DIGEST_TASK "stonith-on" +- + action_t * + pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe_working_set_t * data_set) + { +-- +1.8.3.1 + + +From 04656184bcd4598a9b9d83b344c6f9f9cafe4df7 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 17 Aug 2017 17:42:55 -0500 +Subject: [PATCH 3/3] Refactor: libpe_status: preserve backward API + compatibility + +There isn't a well-defined line in the pengine libraries as to what's part of +the public API, but be on the safe side ... +--- + include/crm/pengine/status.h | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index fcd5226..b1f4d30 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -329,7 +329,6 @@ struct pe_action_s { + + char *task; + char *uuid; +- char *reason; + char *cancel_task; + + enum pe_action_flags flags; +@@ -374,6 +373,8 @@ struct pe_action_s { + + GListPtr actions_before; /* action_wrapper_t* */ + GListPtr actions_after; /* action_wrapper_t* */ ++ ++ char *reason; + }; + + struct ticket_s { +-- +1.8.3.1 + diff --git a/SOURCES/092-remote-fencing.patch b/SOURCES/092-remote-fencing.patch new file mode 100644 index 0000000..793f434 --- /dev/null +++ b/SOURCES/092-remote-fencing.patch @@ -0,0 +1,1082 @@ +From a72ef2be2240aac04b784710ae81c4671a8189cd Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Sat, 19 Aug 2017 15:45:27 -0500 +Subject: [PATCH 1/2] Fix: pengine: re-enable unrecoverable remote fencing + +The fence loop referenced by 00fed62 was due to get_remote_node_state() +considering a failed remote connection resource unrecoverable, even if it was +waiting for its reconnect interval to expire before attempting reconnection. +--- + pengine/allocate.c | 43 +++++++++++++++++---------- + pengine/test10/remote-fence-unclean-3.dot | 5 ++++ + pengine/test10/remote-fence-unclean-3.exp | 35 ++++++++++++++++++++-- + pengine/test10/remote-fence-unclean-3.summary | 3 ++ + 4 files changed, 68 insertions(+), 18 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 01b5b98..7956916 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -905,13 +905,10 @@ probe_resources(pe_working_set_t * data_set) + continue; + + } else if (node->details->online == FALSE && node->details->remote_rsc) { +- // TODO figure out why this results in fence loop +- /* + enum remote_connection_state state = get_remote_node_state(node); + if(state == remote_state_failed) { + pe_fence_node(data_set, node, "the connection is unrecoverable"); + } +- */ + continue; + + } else if(node->details->online == FALSE) { +@@ -1904,22 +1901,38 @@ get_remote_node_state(pe_node_t *node) + * on that remote node until after it starts elsewhere. + */ + if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) { +- /* There is nowhere left to run the connection resource, +- * and the resource is in a failed state (either directly +- * or because it is located on a failed node). +- * +- * If there are any resources known to be active on it (stop), +- * or if there are resources in an unknown state (probe), we +- * must assume the worst and fence it. +- */ +- if (is_set(remote_rsc->flags, pe_rsc_failed)) { +- return remote_state_failed; +- } else if(cluster_node && cluster_node->details->unclean) { ++ /* The connection resource is not going to run anywhere */ ++ ++ if (cluster_node && cluster_node->details->unclean) { ++ /* The remote connection is failed because its resource is on a ++ * failed node and can't be recovered elsewhere, so we must fence. ++ */ + return remote_state_failed; +- } else { ++ } ++ ++ if (is_not_set(remote_rsc->flags, pe_rsc_failed)) { ++ /* Connection resource is cleanly stopped */ + return remote_state_stopped; + } + ++ /* Connection resource is failed */ ++ ++ if ((remote_rsc->next_role == RSC_ROLE_STOPPED) ++ && remote_rsc->remote_reconnect_interval ++ && node->details->remote_was_fenced) { ++ ++ /* We won't know whether the connection is recoverable until the ++ * reconnect interval expires and we reattempt connection. ++ */ ++ return remote_state_unknown; ++ } ++ ++ /* The remote connection is in a failed state. If there are any ++ * resources known to be active on it (stop) or in an unknown state ++ * (probe), we must assume the worst and fence it. ++ */ ++ return remote_state_failed; ++ + } else if (cluster_node == NULL) { + /* Connection is recoverable but not currently running anywhere, see if we can recover it first */ + return remote_state_unknown; +diff --git a/pengine/test10/remote-fence-unclean-3.dot b/pengine/test10/remote-fence-unclean-3.dot +index 14adaef..b32b77e 100644 +--- a/pengine/test10/remote-fence-unclean-3.dot ++++ b/pengine/test10/remote-fence-unclean-3.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "fence1_start_0 overcloud-controller-0" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "fence1_monitor_0 overcloud-controller-0" -> "fence1_start_0 overcloud-controller-0" [ style = bold] + "fence1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] +@@ -11,4 +12,8 @@ digraph "g" { + "fence1_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] + "overcloud-novacompute-0_stop_0 overcloud-controller-0" -> "all_stopped" [ style = bold] + "overcloud-novacompute-0_stop_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' overcloud-novacompute-0" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' overcloud-novacompute-0" [ style=bold color="green" fontcolor="black"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp +index 64e5a62..b7bb735 100644 +--- a/pengine/test10/remote-fence-unclean-3.exp ++++ b/pengine/test10/remote-fence-unclean-3.exp +@@ -21,6 +21,9 @@ + + + ++ ++ ++ + + + +@@ -72,6 +75,29 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -80,6 +106,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary +index ec54d8e..6d15598 100644 +--- a/pengine/test10/remote-fence-unclean-3.summary ++++ b/pengine/test10/remote-fence-unclean-3.summary +@@ -34,6 +34,7 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund + openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1 + + Transition Summary: ++ * Fence (reboot) overcloud-novacompute-0 'the connection is unrecoverable' + * Start fence1 (overcloud-controller-0) + * Stop overcloud-novacompute-0 (overcloud-controller-0) + +@@ -42,6 +43,8 @@ Executing cluster transition: + * Resource action: fence1 monitor on overcloud-controller-1 + * Resource action: fence1 monitor on overcloud-controller-0 + * Resource action: overcloud-novacompute-0 stop on overcloud-controller-0 ++ * Fencing overcloud-novacompute-0 (reboot) ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + * Resource action: fence1 start on overcloud-controller-0 + * Resource action: fence1 monitor=60000 on overcloud-controller-0 +-- +1.8.3.1 + + +From c700136f437ca74d3981a419dc07bbf9a7f692fb Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 21 Aug 2017 15:23:04 -0500 +Subject: [PATCH 2/2] Test: pengine: add regression test for remote connect + interval wait + +--- + pengine/regression.sh | 1 + + pengine/test10/remote-reconnect-delay.dot | 9 + + pengine/test10/remote-reconnect-delay.exp | 49 +++ + pengine/test10/remote-reconnect-delay.scores | 207 +++++++++++ + pengine/test10/remote-reconnect-delay.summary | 66 ++++ + pengine/test10/remote-reconnect-delay.xml | 504 ++++++++++++++++++++++++++ + 6 files changed, 836 insertions(+) + create mode 100644 pengine/test10/remote-reconnect-delay.dot + create mode 100644 pengine/test10/remote-reconnect-delay.exp + create mode 100644 pengine/test10/remote-reconnect-delay.scores + create mode 100644 pengine/test10/remote-reconnect-delay.summary + create mode 100644 pengine/test10/remote-reconnect-delay.xml + +diff --git a/pengine/regression.sh b/pengine/regression.sh +index d1a8a3f..7dc3a04 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -853,6 +853,7 @@ do_test remote-recover-connection "Optimistically recovery of only the connectio + do_test remote-recover-all "Fencing when the connection has no home" + do_test remote-recover-no-resources "Fencing when the connection has no home and no active resources" + do_test remote-recover-unknown "Fencing when the connection has no home and the remote has no operation history" ++do_test remote-reconnect-delay "Waiting for remote reconnect interval to expire" + + echo "" + do_test resource-discovery "Exercises resource-discovery location constraint option." +diff --git a/pengine/test10/remote-reconnect-delay.dot b/pengine/test10/remote-reconnect-delay.dot +new file mode 100644 +index 0000000..42c2cef +--- /dev/null ++++ b/pengine/test10/remote-reconnect-delay.dot +@@ -0,0 +1,9 @@ ++digraph "g" { ++"Fencing_monitor_120000 rhel7-2" [ style=bold color="green" fontcolor="black"] ++"Fencing_start_0 rhel7-2" -> "Fencing_monitor_120000 rhel7-2" [ style = bold] ++"Fencing_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] ++"Fencing_stop_0 rhel7-2" -> "Fencing_start_0 rhel7-2" [ style = bold] ++"Fencing_stop_0 rhel7-2" -> "all_stopped" [ style = bold] ++"Fencing_stop_0 rhel7-2" [ style=bold color="green" fontcolor="black"] ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++} +diff --git a/pengine/test10/remote-reconnect-delay.exp b/pengine/test10/remote-reconnect-delay.exp +new file mode 100644 +index 0000000..fc616c2 +--- /dev/null ++++ b/pengine/test10/remote-reconnect-delay.exp +@@ -0,0 +1,49 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/remote-reconnect-delay.scores b/pengine/test10/remote-reconnect-delay.scores +new file mode 100644 +index 0000000..411af28 +--- /dev/null ++++ b/pengine/test10/remote-reconnect-delay.scores +@@ -0,0 +1,207 @@ ++Allocation scores: ++Using the original execution date of: 2017-08-21 17:12:54Z ++clone_color: Connectivity allocation score on remote-rhel7-3: 0 ++clone_color: Connectivity allocation score on rhel7-1: 0 ++clone_color: Connectivity allocation score on rhel7-2: 0 ++clone_color: Connectivity allocation score on rhel7-4: 0 ++clone_color: Connectivity allocation score on rhel7-5: 0 ++clone_color: master-1 allocation score on remote-rhel7-3: -INFINITY ++clone_color: master-1 allocation score on rhel7-1: 0 ++clone_color: master-1 allocation score on rhel7-2: 0 ++clone_color: master-1 allocation score on rhel7-4: 0 ++clone_color: master-1 allocation score on rhel7-5: 0 ++clone_color: ping-1:0 allocation score on remote-rhel7-3: 0 ++clone_color: ping-1:0 allocation score on rhel7-1: 1 ++clone_color: ping-1:0 allocation score on rhel7-2: 0 ++clone_color: ping-1:0 allocation score on rhel7-4: 0 ++clone_color: ping-1:0 allocation score on rhel7-5: 0 ++clone_color: ping-1:1 allocation score on remote-rhel7-3: 0 ++clone_color: ping-1:1 allocation score on rhel7-1: 0 ++clone_color: ping-1:1 allocation score on rhel7-2: 1 ++clone_color: ping-1:1 allocation score on rhel7-4: 0 ++clone_color: ping-1:1 allocation score on rhel7-5: 0 ++clone_color: ping-1:2 allocation score on remote-rhel7-3: 0 ++clone_color: ping-1:2 allocation score on rhel7-1: 0 ++clone_color: ping-1:2 allocation score on rhel7-2: 0 ++clone_color: ping-1:2 allocation score on rhel7-4: 1 ++clone_color: ping-1:2 allocation score on rhel7-5: 0 ++clone_color: ping-1:3 allocation score on remote-rhel7-3: 0 ++clone_color: ping-1:3 allocation score on rhel7-1: 0 ++clone_color: ping-1:3 allocation score on rhel7-2: 0 ++clone_color: ping-1:3 allocation score on rhel7-4: 0 ++clone_color: ping-1:3 allocation score on rhel7-5: 1 ++clone_color: ping-1:4 allocation score on remote-rhel7-3: 0 ++clone_color: ping-1:4 allocation score on rhel7-1: 0 ++clone_color: ping-1:4 allocation score on rhel7-2: 0 ++clone_color: ping-1:4 allocation score on rhel7-4: 0 ++clone_color: ping-1:4 allocation score on rhel7-5: 0 ++clone_color: stateful-1:0 allocation score on remote-rhel7-3: -INFINITY ++clone_color: stateful-1:0 allocation score on rhel7-1: 6 ++clone_color: stateful-1:0 allocation score on rhel7-2: 0 ++clone_color: stateful-1:0 allocation score on rhel7-4: 0 ++clone_color: stateful-1:0 allocation score on rhel7-5: 0 ++clone_color: stateful-1:1 allocation score on remote-rhel7-3: -INFINITY ++clone_color: stateful-1:1 allocation score on rhel7-1: 0 ++clone_color: stateful-1:1 allocation score on rhel7-2: 11 ++clone_color: stateful-1:1 allocation score on rhel7-4: 0 ++clone_color: stateful-1:1 allocation score on rhel7-5: 0 ++clone_color: stateful-1:2 allocation score on remote-rhel7-3: -INFINITY ++clone_color: stateful-1:2 allocation score on rhel7-1: 0 ++clone_color: stateful-1:2 allocation score on rhel7-2: 0 ++clone_color: stateful-1:2 allocation score on rhel7-4: 6 ++clone_color: stateful-1:2 allocation score on rhel7-5: 0 ++clone_color: stateful-1:3 allocation score on remote-rhel7-3: -INFINITY ++clone_color: stateful-1:3 allocation score on rhel7-1: 0 ++clone_color: stateful-1:3 allocation score on rhel7-2: 0 ++clone_color: stateful-1:3 allocation score on rhel7-4: 0 ++clone_color: stateful-1:3 allocation score on rhel7-5: 6 ++clone_color: stateful-1:4 allocation score on remote-rhel7-3: -INFINITY ++clone_color: stateful-1:4 allocation score on rhel7-1: 0 ++clone_color: stateful-1:4 allocation score on rhel7-2: 0 ++clone_color: stateful-1:4 allocation score on rhel7-4: 0 ++clone_color: stateful-1:4 allocation score on rhel7-5: 0 ++group_color: group-1 allocation score on remote-rhel7-3: 0 ++group_color: group-1 allocation score on rhel7-1: 0 ++group_color: group-1 allocation score on rhel7-2: 0 ++group_color: group-1 allocation score on rhel7-4: 0 ++group_color: group-1 allocation score on rhel7-5: 0 ++group_color: petulant allocation score on remote-rhel7-3: 0 ++group_color: petulant allocation score on rhel7-1: 0 ++group_color: petulant allocation score on rhel7-2: 0 ++group_color: petulant allocation score on rhel7-4: 0 ++group_color: petulant allocation score on rhel7-5: 0 ++group_color: r192.168.122.207 allocation score on remote-rhel7-3: 0 ++group_color: r192.168.122.207 allocation score on rhel7-1: 0 ++group_color: r192.168.122.207 allocation score on rhel7-2: 0 ++group_color: r192.168.122.207 allocation score on rhel7-4: 0 ++group_color: r192.168.122.207 allocation score on rhel7-5: 0 ++group_color: r192.168.122.208 allocation score on remote-rhel7-3: 0 ++group_color: r192.168.122.208 allocation score on rhel7-1: 0 ++group_color: r192.168.122.208 allocation score on rhel7-2: 0 ++group_color: r192.168.122.208 allocation score on rhel7-4: 0 ++group_color: r192.168.122.208 allocation score on rhel7-5: 0 ++native_color: Fencing allocation score on remote-rhel7-3: -INFINITY ++native_color: Fencing allocation score on rhel7-1: 0 ++native_color: Fencing allocation score on rhel7-2: 0 ++native_color: Fencing allocation score on rhel7-4: 0 ++native_color: Fencing allocation score on rhel7-5: 0 ++native_color: FencingFail allocation score on remote-rhel7-3: -INFINITY ++native_color: FencingFail allocation score on rhel7-1: 0 ++native_color: FencingFail allocation score on rhel7-2: 0 ++native_color: FencingFail allocation score on rhel7-4: 0 ++native_color: FencingFail allocation score on rhel7-5: 0 ++native_color: lsb-dummy allocation score on remote-rhel7-3: -INFINITY ++native_color: lsb-dummy allocation score on rhel7-1: -INFINITY ++native_color: lsb-dummy allocation score on rhel7-2: 0 ++native_color: lsb-dummy allocation score on rhel7-4: -INFINITY ++native_color: lsb-dummy allocation score on rhel7-5: -INFINITY ++native_color: migrator allocation score on remote-rhel7-3: 0 ++native_color: migrator allocation score on rhel7-1: 0 ++native_color: migrator allocation score on rhel7-2: 0 ++native_color: migrator allocation score on rhel7-4: 0 ++native_color: migrator allocation score on rhel7-5: 1 ++native_color: petulant allocation score on remote-rhel7-3: -INFINITY ++native_color: petulant allocation score on rhel7-1: -INFINITY ++native_color: petulant allocation score on rhel7-2: 0 ++native_color: petulant allocation score on rhel7-4: -INFINITY ++native_color: petulant allocation score on rhel7-5: -INFINITY ++native_color: ping-1:0 allocation score on remote-rhel7-3: -INFINITY ++native_color: ping-1:0 allocation score on rhel7-1: 1 ++native_color: ping-1:0 allocation score on rhel7-2: 0 ++native_color: ping-1:0 allocation score on rhel7-4: 0 ++native_color: ping-1:0 allocation score on rhel7-5: 0 ++native_color: ping-1:1 allocation score on remote-rhel7-3: -INFINITY ++native_color: ping-1:1 allocation score on rhel7-1: -INFINITY ++native_color: ping-1:1 allocation score on rhel7-2: 1 ++native_color: ping-1:1 allocation score on rhel7-4: 0 ++native_color: ping-1:1 allocation score on rhel7-5: 0 ++native_color: ping-1:2 allocation score on remote-rhel7-3: -INFINITY ++native_color: ping-1:2 allocation score on rhel7-1: -INFINITY ++native_color: ping-1:2 allocation score on rhel7-2: -INFINITY ++native_color: ping-1:2 allocation score on rhel7-4: 1 ++native_color: ping-1:2 allocation score on rhel7-5: 0 ++native_color: ping-1:3 allocation score on remote-rhel7-3: -INFINITY ++native_color: ping-1:3 allocation score on rhel7-1: -INFINITY ++native_color: ping-1:3 allocation score on rhel7-2: -INFINITY ++native_color: ping-1:3 allocation score on rhel7-4: -INFINITY ++native_color: ping-1:3 allocation score on rhel7-5: 1 ++native_color: ping-1:4 allocation score on remote-rhel7-3: -INFINITY ++native_color: ping-1:4 allocation score on rhel7-1: -INFINITY ++native_color: ping-1:4 allocation score on rhel7-2: -INFINITY ++native_color: ping-1:4 allocation score on rhel7-4: -INFINITY ++native_color: ping-1:4 allocation score on rhel7-5: -INFINITY ++native_color: r192.168.122.207 allocation score on remote-rhel7-3: -INFINITY ++native_color: r192.168.122.207 allocation score on rhel7-1: -INFINITY ++native_color: r192.168.122.207 allocation score on rhel7-2: 11 ++native_color: r192.168.122.207 allocation score on rhel7-4: -INFINITY ++native_color: r192.168.122.207 allocation score on rhel7-5: -INFINITY ++native_color: r192.168.122.208 allocation score on remote-rhel7-3: -INFINITY ++native_color: r192.168.122.208 allocation score on rhel7-1: -INFINITY ++native_color: r192.168.122.208 allocation score on rhel7-2: 0 ++native_color: r192.168.122.208 allocation score on rhel7-4: -INFINITY ++native_color: r192.168.122.208 allocation score on rhel7-5: -INFINITY ++native_color: remote-rhel7-3 allocation score on remote-rhel7-3: -INFINITY ++native_color: remote-rhel7-3 allocation score on rhel7-1: -INFINITY ++native_color: remote-rhel7-3 allocation score on rhel7-2: -INFINITY ++native_color: remote-rhel7-3 allocation score on rhel7-4: -INFINITY ++native_color: remote-rhel7-3 allocation score on rhel7-5: -INFINITY ++native_color: remote-rsc allocation score on remote-rhel7-3: INFINITY ++native_color: remote-rsc allocation score on rhel7-1: 0 ++native_color: remote-rsc allocation score on rhel7-2: 0 ++native_color: remote-rsc allocation score on rhel7-4: 0 ++native_color: remote-rsc allocation score on rhel7-5: 0 ++native_color: rsc_rhel7-1 allocation score on remote-rhel7-3: 0 ++native_color: rsc_rhel7-1 allocation score on rhel7-1: 100 ++native_color: rsc_rhel7-1 allocation score on rhel7-2: 0 ++native_color: rsc_rhel7-1 allocation score on rhel7-4: 0 ++native_color: rsc_rhel7-1 allocation score on rhel7-5: 0 ++native_color: rsc_rhel7-2 allocation score on remote-rhel7-3: 0 ++native_color: rsc_rhel7-2 allocation score on rhel7-1: 0 ++native_color: rsc_rhel7-2 allocation score on rhel7-2: 100 ++native_color: rsc_rhel7-2 allocation score on rhel7-4: 0 ++native_color: rsc_rhel7-2 allocation score on rhel7-5: 0 ++native_color: rsc_rhel7-3 allocation score on remote-rhel7-3: 0 ++native_color: rsc_rhel7-3 allocation score on rhel7-1: 0 ++native_color: rsc_rhel7-3 allocation score on rhel7-2: 0 ++native_color: rsc_rhel7-3 allocation score on rhel7-4: 0 ++native_color: rsc_rhel7-3 allocation score on rhel7-5: 0 ++native_color: rsc_rhel7-4 allocation score on remote-rhel7-3: 0 ++native_color: rsc_rhel7-4 allocation score on rhel7-1: 0 ++native_color: rsc_rhel7-4 allocation score on rhel7-2: 0 ++native_color: rsc_rhel7-4 allocation score on rhel7-4: 100 ++native_color: rsc_rhel7-4 allocation score on rhel7-5: 0 ++native_color: rsc_rhel7-5 allocation score on remote-rhel7-3: 0 ++native_color: rsc_rhel7-5 allocation score on rhel7-1: 0 ++native_color: rsc_rhel7-5 allocation score on rhel7-2: 0 ++native_color: rsc_rhel7-5 allocation score on rhel7-4: 0 ++native_color: rsc_rhel7-5 allocation score on rhel7-5: 100 ++native_color: stateful-1:0 allocation score on remote-rhel7-3: -INFINITY ++native_color: stateful-1:0 allocation score on rhel7-1: 6 ++native_color: stateful-1:0 allocation score on rhel7-2: -INFINITY ++native_color: stateful-1:0 allocation score on rhel7-4: 0 ++native_color: stateful-1:0 allocation score on rhel7-5: 0 ++native_color: stateful-1:1 allocation score on remote-rhel7-3: -INFINITY ++native_color: stateful-1:1 allocation score on rhel7-1: 0 ++native_color: stateful-1:1 allocation score on rhel7-2: 11 ++native_color: stateful-1:1 allocation score on rhel7-4: 0 ++native_color: stateful-1:1 allocation score on rhel7-5: 0 ++native_color: stateful-1:2 allocation score on remote-rhel7-3: -INFINITY ++native_color: stateful-1:2 allocation score on rhel7-1: -INFINITY ++native_color: stateful-1:2 allocation score on rhel7-2: -INFINITY ++native_color: stateful-1:2 allocation score on rhel7-4: 6 ++native_color: stateful-1:2 allocation score on rhel7-5: 0 ++native_color: stateful-1:3 allocation score on remote-rhel7-3: -INFINITY ++native_color: stateful-1:3 allocation score on rhel7-1: -INFINITY ++native_color: stateful-1:3 allocation score on rhel7-2: -INFINITY ++native_color: stateful-1:3 allocation score on rhel7-4: -INFINITY ++native_color: stateful-1:3 allocation score on rhel7-5: 6 ++native_color: stateful-1:4 allocation score on remote-rhel7-3: -INFINITY ++native_color: stateful-1:4 allocation score on rhel7-1: -INFINITY ++native_color: stateful-1:4 allocation score on rhel7-2: -INFINITY ++native_color: stateful-1:4 allocation score on rhel7-4: -INFINITY ++native_color: stateful-1:4 allocation score on rhel7-5: -INFINITY ++stateful-1:0 promotion score on rhel7-1: 5 ++stateful-1:1 promotion score on rhel7-2: 10 ++stateful-1:2 promotion score on rhel7-4: 5 ++stateful-1:3 promotion score on rhel7-5: 5 ++stateful-1:4 promotion score on none: 0 +diff --git a/pengine/test10/remote-reconnect-delay.summary b/pengine/test10/remote-reconnect-delay.summary +new file mode 100644 +index 0000000..ea11483 +--- /dev/null ++++ b/pengine/test10/remote-reconnect-delay.summary +@@ -0,0 +1,66 @@ ++Using the original execution date of: 2017-08-21 17:12:54Z ++ ++Current cluster status: ++Online: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] ++RemoteOFFLINE: [ remote-rhel7-3 ] ++ ++ Fencing (stonith:fence_xvm): Started rhel7-2 ++ FencingFail (stonith:fence_dummy): Started rhel7-4 ++ rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-1 ++ rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Started rhel7-2 ++ rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Started rhel7-5 ++ rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Started rhel7-4 ++ rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Started rhel7-5 ++ migrator (ocf::pacemaker:Dummy): Started rhel7-5 ++ Clone Set: Connectivity [ping-1] ++ Started: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] ++ Stopped: [ remote-rhel7-3 ] ++ Master/Slave Set: master-1 [stateful-1] ++ Masters: [ rhel7-2 ] ++ Slaves: [ rhel7-1 rhel7-4 rhel7-5 ] ++ Stopped: [ remote-rhel7-3 ] ++ Resource Group: group-1 ++ r192.168.122.207 (ocf::heartbeat:IPaddr2): Started rhel7-2 ++ petulant (service:DummySD): Started rhel7-2 ++ r192.168.122.208 (ocf::heartbeat:IPaddr2): Started rhel7-2 ++ lsb-dummy (lsb:/usr/share/pacemaker/tests/cts/LSBDummy): Started rhel7-2 ++ remote-rhel7-3 (ocf::pacemaker:remote): FAILED ++ remote-rsc (ocf::heartbeat:Dummy): Started rhel7-1 ++ ++Transition Summary: ++ * Restart Fencing ( rhel7-2 ) ++ ++Executing cluster transition: ++ * Resource action: Fencing stop on rhel7-2 ++ * Resource action: Fencing start on rhel7-2 ++ * Resource action: Fencing monitor=120000 on rhel7-2 ++ * Pseudo action: all_stopped ++Using the original execution date of: 2017-08-21 17:12:54Z ++ ++Revised cluster status: ++Online: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] ++RemoteOFFLINE: [ remote-rhel7-3 ] ++ ++ Fencing (stonith:fence_xvm): Started rhel7-2 ++ FencingFail (stonith:fence_dummy): Started rhel7-4 ++ rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-1 ++ rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Started rhel7-2 ++ rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Started rhel7-5 ++ rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Started rhel7-4 ++ rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Started rhel7-5 ++ migrator (ocf::pacemaker:Dummy): Started rhel7-5 ++ Clone Set: Connectivity [ping-1] ++ Started: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] ++ Stopped: [ remote-rhel7-3 ] ++ Master/Slave Set: master-1 [stateful-1] ++ Masters: [ rhel7-2 ] ++ Slaves: [ rhel7-1 rhel7-4 rhel7-5 ] ++ Stopped: [ remote-rhel7-3 ] ++ Resource Group: group-1 ++ r192.168.122.207 (ocf::heartbeat:IPaddr2): Started rhel7-2 ++ petulant (service:DummySD): Started rhel7-2 ++ r192.168.122.208 (ocf::heartbeat:IPaddr2): Started rhel7-2 ++ lsb-dummy (lsb:/usr/share/pacemaker/tests/cts/LSBDummy): Started rhel7-2 ++ remote-rhel7-3 (ocf::pacemaker:remote): FAILED ++ remote-rsc (ocf::heartbeat:Dummy): Started rhel7-1 ++ +diff --git a/pengine/test10/remote-reconnect-delay.xml b/pengine/test10/remote-reconnect-delay.xml +new file mode 100644 +index 0000000..e9ed3e6 +--- /dev/null ++++ b/pengine/test10/remote-reconnect-delay.xml +@@ -0,0 +1,504 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +-- +1.8.3.1 + diff --git a/SOURCES/093-bundle-fixes.patch b/SOURCES/093-bundle-fixes.patch new file mode 100644 index 0000000..2ca58d6 --- /dev/null +++ b/SOURCES/093-bundle-fixes.patch @@ -0,0 +1,29100 @@ +From cf34f4c950704f5ac05ed9553b3b900ea33e1190 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Thu, 17 Aug 2017 11:48:17 +1000 +Subject: [PATCH 01/21] PE/attrd: Allow bundle resources to read/write + attributes based on the physical host rather than the container name + +--- + include/crm/msg_xml.h | 1 + + include/crm/pengine/internal.h | 2 + + include/crm_internal.h | 3 ++ + lib/pengine/common.c | 48 ++++++++++++++++++++++ + lib/pengine/container.c | 5 ++- + lib/pengine/unpack.c | 22 +++++----- + lib/pengine/utils.c | 4 +- + lib/pengine/variant.h | 1 + + pengine/allocate.c | 3 +- + pengine/constraints.c | 8 ++-- + pengine/graph.c | 39 ++++++++++++++++++ + pengine/master.c | 16 +++----- + pengine/native.c | 16 ++++---- + pengine/test10/bug-cl-5247.exp | 26 ++++++------ + pengine/test10/bug-rh-1097457.exp | 22 +++++----- + pengine/test10/bundle-nested-colocation.exp | 18 ++++----- + pengine/test10/bundle-order-partial-start-2.exp | 20 ++++----- + pengine/test10/bundle-order-partial-start.exp | 20 ++++----- + pengine/test10/bundle-order-partial-stop.exp | 18 ++++----- + pengine/test10/bundle-order-startup-clone-2.exp | 54 ++++++++++++------------- + pengine/test10/bundle-order-startup-clone.exp | 8 ++-- + pengine/test10/bundle-order-startup.exp | 20 ++++----- + pengine/test10/bundle-order-stop-clone.exp | 2 +- + pengine/test10/bundle-order-stop.exp | 18 ++++----- + pengine/test10/guest-node-host-dies.exp | 14 +++---- + pengine/test10/whitebox-asymmetric.exp | 4 +- + pengine/test10/whitebox-fail1.exp | 12 +++--- + pengine/test10/whitebox-fail2.exp | 12 +++--- + pengine/test10/whitebox-fail3.exp | 10 ++--- + pengine/test10/whitebox-imply-stop-on-fence.exp | 6 +-- + pengine/test10/whitebox-move.exp | 12 +++--- + pengine/test10/whitebox-ms-ordering-move.exp | 8 ++-- + pengine/test10/whitebox-ms-ordering.exp | 8 ++-- + pengine/test10/whitebox-nested-group.exp | 12 +++--- + pengine/test10/whitebox-orphan-ms.exp | 8 ++-- + pengine/test10/whitebox-orphaned.exp | 8 ++-- + pengine/test10/whitebox-start.exp | 10 ++--- + pengine/test10/whitebox-stop.exp | 8 ++-- + tools/attrd_updater.c | 30 ++++++++++++++ + tools/crm_mon.c | 2 +- + 40 files changed, 339 insertions(+), 219 deletions(-) + +diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h +index 8cf22f3..5209cb1 100644 +--- a/include/crm/msg_xml.h ++++ b/include/crm/msg_xml.h +@@ -194,6 +194,7 @@ + + # define XML_CIB_TAG_RSC_TEMPLATE "template" + ++# define XML_RSC_ATTR_TARGET "container-attribute-target" + # define XML_RSC_ATTR_ISOLATION_INSTANCE "isolation-instance" + # define XML_RSC_ATTR_ISOLATION_WRAPPER "isolation-wrapper" + # define XML_RSC_ATTR_ISOLATION_HOST "isolation-host" +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index 5db90aa..d91f54a 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -298,5 +298,7 @@ bool remote_id_conflict(const char *remote_name, pe_working_set_t *data); + void common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *node, long options, void *print_data); + resource_t *find_container_child(const char *stem, resource_t * rsc, node_t *node); + bool fix_remote_addr(resource_t * rsc); ++const char *node_attribute_calculated(pe_node_t *node, const char *name, resource_t *rsc); ++const char *node_attribute_raw(pe_node_t *node, const char *name); + + #endif +diff --git a/include/crm_internal.h b/include/crm_internal.h +index acfca07..e1b25ff 100644 +--- a/include/crm_internal.h ++++ b/include/crm_internal.h +@@ -289,6 +289,9 @@ long crm_read_pidfile(const char *filename); + # define ATTRD_OP_SYNC "sync" + # define ATTRD_OP_SYNC_RESPONSE "sync-response" + ++# define PCMK_ENV_PHYSICAL_HOST "physical_host" ++ ++ + # if SUPPORT_COROSYNC + # if CS_USES_LIBQB + # include +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index 446b7f0..0e21aaa 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -431,3 +431,51 @@ add_hash_param(GHashTable * hash, const char *name, const char *value) + g_hash_table_insert(hash, strdup(name), strdup(value)); + } + } ++ ++const char * ++node_attribute_calculated(pe_node_t *node, const char *name, resource_t *rsc) ++{ ++ const char *source; ++ ++ if(node == NULL) { ++ return NULL; ++ ++ } else if(rsc == NULL) { ++ return g_hash_table_lookup(node->details->attrs, name); ++ } ++ ++ source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET); ++ if(source == NULL || safe_str_eq("host", source) == FALSE) { ++ return g_hash_table_lookup(node->details->attrs, name); ++ } ++ ++ /* Use attributes set for the containers location ++ * instead of for the container itself ++ * ++ * Useful when the container is using the host's local ++ * storage ++ */ ++ ++ CRM_ASSERT(node->details->remote_rsc); ++ CRM_ASSERT(node->details->remote_rsc->container); ++ ++ if(node->details->remote_rsc->container->running_on) { ++ pe_node_t *host = node->details->remote_rsc->container->running_on->data; ++ pe_rsc_trace(rsc, "%s: Looking for %s on the container host %s", rsc->id, name, host->details->uname); ++ return g_hash_table_lookup(host->details->attrs, name); ++ } ++ ++ pe_rsc_trace(rsc, "%s: Not looking for %s on the container host: %s is inactive", ++ rsc->id, name, node->details->remote_rsc->container->id); ++ return NULL; ++} ++ ++const char * ++node_attribute_raw(pe_node_t *node, const char *name) ++{ ++ if(node == NULL) { ++ return NULL; ++ } ++ return g_hash_table_lookup(node->details->attrs, name); ++} ++ +diff --git a/lib/pengine/container.c b/lib/pengine/container.c +index 118e716..53965cf 100644 +--- a/lib/pengine/container.c ++++ b/lib/pengine/container.c +@@ -764,8 +764,12 @@ container_unpack(resource_t * rsc, pe_working_set_t * data_set) + + offset += allocate_ip(container_data, tuple, buffer+offset, max-offset); + container_data->tuples = g_list_append(container_data->tuples, tuple); ++ container_data->attribute_target = g_hash_table_lookup(tuple->child->meta, XML_RSC_ATTR_TARGET); + } + container_data->docker_host_options = buffer; ++ if(container_data->attribute_target) { ++ g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET), strdup(container_data->attribute_target)); ++ } + + } else { + // Just a naked container, no pacemaker-remote +@@ -782,7 +786,6 @@ container_unpack(resource_t * rsc, pe_working_set_t * data_set) + container_data->docker_host_options = buffer; + } + +- + for (GListPtr gIter = container_data->tuples; gIter != NULL; gIter = gIter->next) { + container_grouping_t *tuple = (container_grouping_t *)gIter->data; + // TODO: Remove from list if create_container() returns TRUE +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 0790148..6e0651e 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -1070,7 +1070,7 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * + attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE); + add_node_attrs(attrs, this_node, TRUE, data_set); + +- shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN); ++ shutdown = node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); + if (shutdown != NULL && safe_str_neq("0", shutdown)) { + crm_info("Node %s is shutting down", this_node->details->uname); + this_node->details->shutdown = TRUE; +@@ -1079,18 +1079,18 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * + } + } + +- if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) { ++ if (crm_is_true(node_attribute_raw(this_node, "standby"))) { + crm_info("Node %s is in standby-mode", this_node->details->uname); + this_node->details->standby = TRUE; + } + +- if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "maintenance")) || ++ if (crm_is_true(node_attribute_raw(this_node, "maintenance")) || + (rsc && !is_set(rsc->flags, pe_rsc_managed))) { + crm_info("Node %s is in maintenance-mode", this_node->details->uname); + this_node->details->maintenance = TRUE; + } + +- resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY); ++ resource_discovery_enabled = node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY); + if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) { + if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) { + crm_warn("ignoring %s attribute on baremetal remote node %s, disabling resource discovery requires stonith to be enabled.", +@@ -1246,17 +1246,17 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set) + attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE); + add_node_attrs(attrs, this_node, TRUE, data_set); + +- if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "standby"))) { ++ if (crm_is_true(node_attribute_raw(this_node, "standby"))) { + crm_info("Node %s is in standby-mode", this_node->details->uname); + this_node->details->standby = TRUE; + } + +- if (crm_is_true(g_hash_table_lookup(this_node->details->attrs, "maintenance"))) { ++ if (crm_is_true(node_attribute_raw(this_node, "maintenance"))) { + crm_info("Node %s is in maintenance-mode", this_node->details->uname); + this_node->details->maintenance = TRUE; + } + +- resource_discovery_enabled = g_hash_table_lookup(this_node->details->attrs, XML_NODE_ATTR_RSC_DISCOVERY); ++ resource_discovery_enabled = node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY); + if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) { + crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes", + XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname); +@@ -1342,7 +1342,7 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat + const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER); + const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER); + const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED); +- const char *terminate = g_hash_table_lookup(this_node->details->attrs, "terminate"); ++ const char *terminate = node_attribute_raw(this_node, "terminate"); + + /* + - XML_NODE_IN_CLUSTER ::= true|false +@@ -1516,7 +1516,7 @@ determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set + + this_node->details->shutdown = FALSE; + this_node->details->expected_up = FALSE; +- shutdown = g_hash_table_lookup(this_node->details->attrs, XML_CIB_ATTR_SHUTDOWN); ++ shutdown = node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); + + if (shutdown != NULL && safe_str_neq("0", shutdown)) { + this_node->details->shutdown = TRUE; +@@ -3368,8 +3368,8 @@ add_node_attrs(xmlNode * xml_obj, node_t * node, gboolean overwrite, pe_working_ + unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL, + node->details->attrs, NULL, overwrite, data_set->now); + +- if (g_hash_table_lookup(node->details->attrs, "#site-name") == NULL) { +- const char *site_name = g_hash_table_lookup(node->details->attrs, "site-name"); ++ if (node_attribute_raw(node, "#site-name") == NULL) { ++ const char *site_name = node_attribute_raw(node, "site-name"); + + if (site_name) { + /* Prefix '#' to the key */ +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index cb1ed08..c44a1c8 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1989,10 +1987,8 @@ fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * da + char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); + op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, node, NULL, data_set); + +- const char *digest_all = g_hash_table_lookup(node->details->attrs, +- CRM_ATTR_DIGESTS_ALL); +- const char *digest_secure = g_hash_table_lookup(node->details->attrs, +- CRM_ATTR_DIGESTS_SECURE); ++ const char *digest_all = node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); ++ const char *digest_secure = node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); + + /* No 'reloads' for fencing device changes + * +diff --git a/lib/pengine/variant.h b/lib/pengine/variant.h +index c8fe159..4ceaa6b 100644 +--- a/lib/pengine/variant.h ++++ b/lib/pengine/variant.h +@@ -100,6 +100,7 @@ typedef struct container_variant_data_s { + char *docker_host_options; + char *docker_run_options; + char *docker_run_command; ++ const char *attribute_target; + + resource_t *child; + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 01b5b98..958fb23 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -861,7 +861,6 @@ apply_system_health(pe_working_set_t * data_set) + rsc2node_new(health_strategy, rsc, system_health, NULL, node, data_set); + } + } +- + } + + return TRUE; +@@ -898,7 +897,7 @@ probe_resources(pe_working_set_t * data_set) + + for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; +- const char *probed = g_hash_table_lookup(node->details->attrs, CRM_OP_PROBED); ++ const char *probed = node_attribute_raw(node, CRM_OP_PROBED); + + if (is_container_remote_node(node)) { + /* TODO enable guest node probes once ordered probing is implemented */ +diff --git a/pengine/constraints.c b/pengine/constraints.c +index 77a0152..fdc4186 100644 +--- a/pengine/constraints.c ++++ b/pengine/constraints.c +@@ -957,7 +957,7 @@ unpack_location(xmlNode * xml_obj, pe_working_set_t * data_set) + } + + static int +-get_node_score(const char *rule, const char *score, gboolean raw, node_t * node) ++get_node_score(const char *rule, const char *score, gboolean raw, node_t * node, resource_t *rsc) + { + int score_f = 0; + +@@ -968,7 +968,7 @@ get_node_score(const char *rule, const char *score, gboolean raw, node_t * node) + score_f = char2score(score); + + } else { +- const char *attr_score = g_hash_table_lookup(node->details->attrs, score); ++ const char *attr_score = node_attribute_calculated(node, score, rsc); + + if (attr_score == NULL) { + crm_debug("Rule %s: node %s did not have a value for %s", +@@ -1060,7 +1060,7 @@ generate_location_rule(resource_t * rsc, xmlNode * rule_xml, const char *discove + for (gIter = match_L; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + +- node->weight = get_node_score(rule_id, score, raw_score, node); ++ node->weight = get_node_score(rule_id, score, raw_score, node, rsc); + } + } + +@@ -1073,7 +1073,7 @@ generate_location_rule(resource_t * rsc, xmlNode * rule_xml, const char *discove + crm_trace("Rule %s %s on %s", ID(rule_xml), accept ? "passed" : "failed", + node->details->uname); + +- score_f = get_node_score(rule_id, score, raw_score, node); ++ score_f = get_node_score(rule_id, score, raw_score, node, rsc); + /* if(accept && score_f == -INFINITY) { */ + /* accept = FALSE; */ + /* } */ +diff --git a/pengine/graph.c b/pengine/graph.c +index a446fe5..3c5dc6b 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -1162,6 +1162,45 @@ action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set) + hash2smartfield((gpointer)"pcmk_external_ip", (gpointer)value, (gpointer)args_xml); + } + ++ if(is_container_remote_node(action->node)) { ++ pe_node_t *host = NULL; ++ enum action_tasks task = text2task(action->task); ++ ++ if(task == action_notify || task == action_notified) { ++ const char *n_task = g_hash_table_lookup(action->meta, "notify_operation"); ++ task = text2task(n_task); ++ } ++ ++ // Differentiate between up and down actions ++ switch (task) { ++ case stop_rsc: ++ case stopped_rsc: ++ case action_demote: ++ case action_demoted: ++ if(action->node->details->remote_rsc->container->running_on) { ++ host = action->node->details->remote_rsc->container->running_on->data; ++ } ++ break; ++ case start_rsc: ++ case started_rsc: ++ case monitor_rsc: ++ case action_promote: ++ case action_promoted: ++ if(action->node->details->remote_rsc->container->allocated_to) { ++ host = action->node->details->remote_rsc->container->allocated_to; ++ } ++ break; ++ default: ++ break; ++ } ++ ++ if(host) { ++ hash2metafield((gpointer)XML_RSC_ATTR_TARGET, ++ (gpointer)g_hash_table_lookup(action->rsc->meta, XML_RSC_ATTR_TARGET), (gpointer)args_xml); ++ hash2metafield((gpointer)PCMK_ENV_PHYSICAL_HOST, (gpointer)host->details->uname, (gpointer)args_xml); ++ } ++ } ++ + } else if (safe_str_eq(action->task, CRM_OP_FENCE) && action->node) { + g_hash_table_foreach(action->node->details->attrs, hash2metafield, args_xml); + } +diff --git a/pengine/master.c b/pengine/master.c +index 8c39f49..2631da8 100644 +--- a/pengine/master.c ++++ b/pengine/master.c +@@ -511,12 +511,7 @@ master_score(resource_t * rsc, node_t * node, int not_set_value) + attr_name = calloc(1, len); + sprintf(attr_name, "master-%s", name); + +- if (node) { +- attr_value = g_hash_table_lookup(node->details->attrs, attr_name); +- pe_rsc_trace(rsc, "%s: %s[%s] = %s", rsc->id, attr_name, node->details->uname, +- crm_str(attr_value)); +- } +- ++ attr_value = node_attribute_calculated(node, attr_name, rsc); + if (attr_value != NULL) { + score = char2score(attr_value); + } +@@ -949,11 +944,11 @@ node_hash_update_one(GHashTable * hash, node_t * other, const char *attr, int sc + } else if (attr == NULL) { + attr = "#" XML_ATTR_UNAME; + } +- +- value = g_hash_table_lookup(other->details->attrs, attr); ++ ++ value = node_attribute_raw(other, attr); + g_hash_table_iter_init(&iter, hash); + while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { +- const char *tmp = g_hash_table_lookup(node->details->attrs, attr); ++ const char *tmp = node_attribute_raw(node, attr); + + if (safe_str_eq(value, tmp)) { + crm_trace("%s: %d + %d", node->details->uname, node->weight, other->weight); +@@ -993,8 +988,7 @@ master_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocatio + } else if (is_set(rsc_lh->flags, pe_rsc_provisional)) { + GListPtr rhs = NULL; + +- gIter = rsc_rh->children; +- for (; gIter != NULL; gIter = gIter->next) { ++ for (gIter = rsc_rh->children; gIter != NULL; gIter = gIter->next) { + resource_t *child_rsc = (resource_t *) gIter->data; + node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE); + enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, FALSE); +diff --git a/pengine/native.c b/pengine/native.c +index fe7b966..37cf541 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -203,7 +203,7 @@ node_list_attr_score(GHashTable * list, const char *attr, const char *value) + weight = -INFINITY; + } + if (weight > best_score || best_node == NULL) { +- const char *tmp = g_hash_table_lookup(node->details->attrs, attr); ++ const char *tmp = node_attribute_raw(node, attr); + + if (safe_str_eq(value, tmp)) { + best_score = weight; +@@ -238,7 +238,7 @@ node_hash_update(GHashTable * list1, GHashTable * list2, const char *attr, float + CRM_LOG_ASSERT(node != NULL); + if(node == NULL) { continue; }; + +- score = node_list_attr_score(list2, attr, g_hash_table_lookup(node->details->attrs, attr)); ++ score = node_list_attr_score(list2, attr, node_attribute_raw(node, attr)); + new_score = merge_weights(factor * score, node->weight); + + if (factor < 0 && score < 0) { +@@ -1630,8 +1630,8 @@ influence_priority(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * + return; + } + +- lh_value = g_hash_table_lookup(rsc_lh->allocated_to->details->attrs, attribute); +- rh_value = g_hash_table_lookup(rsc_rh->allocated_to->details->attrs, attribute); ++ lh_value = node_attribute_raw(rsc_lh->allocated_to, attribute); ++ rh_value = node_attribute_raw(rsc_rh->allocated_to, attribute); + + if (!safe_str_eq(lh_value, rh_value)) { + if(constraint->score == INFINITY && constraint->role_lh == RSC_ROLE_MASTER) { +@@ -1669,7 +1669,7 @@ colocation_match(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * co + } + + if (rsc_rh->allocated_to) { +- value = g_hash_table_lookup(rsc_rh->allocated_to->details->attrs, attribute); ++ value = node_attribute_raw(rsc_rh->allocated_to, attribute); + do_check = TRUE; + + } else if (constraint->score < 0) { +@@ -1683,7 +1683,7 @@ colocation_match(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * co + + g_hash_table_iter_init(&iter, work); + while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { +- tmp = g_hash_table_lookup(node->details->attrs, attribute); ++ tmp = node_attribute_raw(node, attribute); + if (do_check && safe_str_eq(tmp, value)) { + if (constraint->score < INFINITY) { + pe_rsc_trace(rsc_lh, "%s: %s.%s += %d", constraint->id, rsc_lh->id, +@@ -2492,7 +2492,7 @@ StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * d + + if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { + action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, data_set); +- const char *unfenced = g_hash_table_lookup(current->details->attrs, CRM_ATTR_UNFENCED); ++ const char *unfenced = node_attribute_raw(current, CRM_ATTR_UNFENCED); + + order_actions(stop, unfence, pe_order_implies_first); + if (unfenced == NULL || safe_str_eq("0", unfenced)) { +@@ -2515,7 +2515,7 @@ StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * + + if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { + action_t *unfence = pe_fence_op(next, "on", TRUE, NULL, data_set); +- const char *unfenced = g_hash_table_lookup(next->details->attrs, CRM_ATTR_UNFENCED); ++ const char *unfenced = node_attribute_raw(next, CRM_ATTR_UNFENCED); + + order_actions(unfence, start, pe_order_implies_then); + +diff --git a/pengine/test10/bug-cl-5247.exp b/pengine/test10/bug-cl-5247.exp +index e7bac9c..2ec83c0 100644 +--- a/pengine/test10/bug-cl-5247.exp ++++ b/pengine/test10/bug-cl-5247.exp +@@ -269,7 +269,7 @@ + + + +- ++ + + + +@@ -282,7 +282,7 @@ + + + +- ++ + + + +@@ -300,7 +300,7 @@ + + + +- ++ + + + +@@ -319,7 +319,7 @@ + + + +- ++ + + + +@@ -332,7 +332,7 @@ + + + +- ++ + + + +@@ -353,7 +353,7 @@ + + + +- ++ + + + +@@ -395,7 +395,7 @@ + + + +- ++ + + + +@@ -413,7 +413,7 @@ + + + +- ++ + + + +@@ -429,7 +429,7 @@ + + + +- ++ + + + +@@ -442,7 +442,7 @@ + + + +- ++ + + + +@@ -455,7 +455,7 @@ + + + +- ++ + + + +@@ -468,7 +468,7 @@ + + + +- ++ + + + +@@ -484,7 +484,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-rh-1097457.exp b/pengine/test10/bug-rh-1097457.exp +index bf62647..eb9b225 100644 +--- a/pengine/test10/bug-rh-1097457.exp ++++ b/pengine/test10/bug-rh-1097457.exp +@@ -76,7 +76,7 @@ + + + +- ++ + + + +@@ -92,7 +92,7 @@ + + + +- ++ + + + +@@ -168,7 +168,7 @@ + + + +- ++ + + + +@@ -192,7 +192,7 @@ + + + +- ++ + + + +@@ -211,7 +211,7 @@ + + + +- ++ + + + +@@ -227,7 +227,7 @@ + + + +- ++ + + + +@@ -254,7 +254,7 @@ + + + +- ++ + + + +@@ -270,7 +270,7 @@ + + + +- ++ + + + +@@ -286,7 +286,7 @@ + + + +- ++ + + + +@@ -310,7 +310,7 @@ + + + +- ++ + + + +@@ -326,7 +326,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-nested-colocation.exp b/pengine/test10/bundle-nested-colocation.exp +index c82ee16..916c44b 100644 +--- a/pengine/test10/bundle-nested-colocation.exp ++++ b/pengine/test10/bundle-nested-colocation.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -35,7 +35,7 @@ + + + +- ++ + + + +@@ -54,7 +54,7 @@ + + + +- ++ + + + +@@ -67,7 +67,7 @@ + + + +- ++ + + + +@@ -86,7 +86,7 @@ + + + +- ++ + + + +@@ -108,7 +108,7 @@ + + + +- ++ + + + +@@ -121,7 +121,7 @@ + + + +- ++ + + + +@@ -140,7 +140,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp +index b533469..f600983 100644 +--- a/pengine/test10/bundle-order-partial-start-2.exp ++++ b/pengine/test10/bundle-order-partial-start-2.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -125,7 +125,7 @@ + + + +- ++ + + + +@@ -141,7 +141,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -286,7 +286,7 @@ + + + +- ++ + + + +@@ -299,7 +299,7 @@ + + + +- ++ + + + +@@ -312,7 +312,7 @@ + + + +- ++ + + + +@@ -328,7 +328,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp +index 1f8a3e1..d48fccf 100644 +--- a/pengine/test10/bundle-order-partial-start.exp ++++ b/pengine/test10/bundle-order-partial-start.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -125,7 +125,7 @@ + + + +- ++ + + + +@@ -141,7 +141,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -267,7 +267,7 @@ + + + +- ++ + + + +@@ -280,7 +280,7 @@ + + + +- ++ + + + +@@ -293,7 +293,7 @@ + + + +- ++ + + + +@@ -309,7 +309,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-stop.exp b/pengine/test10/bundle-order-partial-stop.exp +index 937bc59..1dff657 100644 +--- a/pengine/test10/bundle-order-partial-stop.exp ++++ b/pengine/test10/bundle-order-partial-stop.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -144,7 +144,7 @@ + + + +- ++ + + + +@@ -163,7 +163,7 @@ + + + +- ++ + + + +@@ -280,7 +280,7 @@ + + + +- ++ + + + +@@ -293,7 +293,7 @@ + + + +- ++ + + + +@@ -306,7 +306,7 @@ + + + +- ++ + + + +@@ -319,7 +319,7 @@ + + + +- ++ + + + +@@ -338,7 +338,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-startup-clone-2.exp b/pengine/test10/bundle-order-startup-clone-2.exp +index c0910a7..7d18d29 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.exp ++++ b/pengine/test10/bundle-order-startup-clone-2.exp +@@ -260,7 +260,7 @@ + + + +- ++ + + + +@@ -276,7 +276,7 @@ + + + +- ++ + + + +@@ -292,7 +292,7 @@ + + + +- ++ + + + +@@ -311,7 +311,7 @@ + + + +- ++ + + + +@@ -327,7 +327,7 @@ + + + +- ++ + + + +@@ -343,7 +343,7 @@ + + + +- ++ + + + +@@ -365,7 +365,7 @@ + + + +- ++ + + + +@@ -381,7 +381,7 @@ + + + +- ++ + + + +@@ -397,7 +397,7 @@ + + + +- ++ + + + +@@ -1010,7 +1010,7 @@ + + + +- ++ + + + +@@ -1023,7 +1023,7 @@ + + + +- ++ + + + +@@ -1036,7 +1036,7 @@ + + + +- ++ + + + +@@ -1049,7 +1049,7 @@ + + + +- ++ + + + +@@ -1074,7 +1074,7 @@ + + + +- ++ + + + +@@ -1096,7 +1096,7 @@ + + + +- ++ + + + +@@ -1115,7 +1115,7 @@ + + + +- ++ + + + +@@ -1128,7 +1128,7 @@ + + + +- ++ + + + +@@ -1141,7 +1141,7 @@ + + + +- ++ + + + +@@ -1154,7 +1154,7 @@ + + + +- ++ + + + +@@ -1179,7 +1179,7 @@ + + + +- ++ + + + +@@ -1204,7 +1204,7 @@ + + + +- ++ + + + +@@ -1226,7 +1226,7 @@ + + + +- ++ + + + +@@ -1239,7 +1239,7 @@ + + + +- ++ + + + +@@ -1252,7 +1252,7 @@ + + + +- ++ + + + +@@ -1265,7 +1265,7 @@ + + + +- ++ + + + +@@ -1290,7 +1290,7 @@ + + + +- ++ + + + +@@ -1315,7 +1315,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-startup-clone.exp b/pengine/test10/bundle-order-startup-clone.exp +index 197529f..8c8ba6aa 100644 +--- a/pengine/test10/bundle-order-startup-clone.exp ++++ b/pengine/test10/bundle-order-startup-clone.exp +@@ -119,7 +119,7 @@ + + + +- ++ + + + +@@ -132,7 +132,7 @@ + + + +- ++ + + + +@@ -151,7 +151,7 @@ + + + +- ++ + + + +@@ -170,7 +170,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp +index 161c1ae..03b064a 100644 +--- a/pengine/test10/bundle-order-startup.exp ++++ b/pengine/test10/bundle-order-startup.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -35,7 +35,7 @@ + + + +- ++ + + + +@@ -198,7 +198,7 @@ + + + +- ++ + + + +@@ -214,7 +214,7 @@ + + + +- ++ + + + +@@ -230,7 +230,7 @@ + + + +- ++ + + + +@@ -340,7 +340,7 @@ + + + +- ++ + + + +@@ -353,7 +353,7 @@ + + + +- ++ + + + +@@ -372,7 +372,7 @@ + + + +- ++ + + + +@@ -391,7 +391,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-stop-clone.exp b/pengine/test10/bundle-order-stop-clone.exp +index c359e00..e78e235 100644 +--- a/pengine/test10/bundle-order-stop-clone.exp ++++ b/pengine/test10/bundle-order-stop-clone.exp +@@ -173,7 +173,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-stop.exp b/pengine/test10/bundle-order-stop.exp +index 937bc59..1dff657 100644 +--- a/pengine/test10/bundle-order-stop.exp ++++ b/pengine/test10/bundle-order-stop.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -144,7 +144,7 @@ + + + +- ++ + + + +@@ -163,7 +163,7 @@ + + + +- ++ + + + +@@ -280,7 +280,7 @@ + + + +- ++ + + + +@@ -293,7 +293,7 @@ + + + +- ++ + + + +@@ -306,7 +306,7 @@ + + + +- ++ + + + +@@ -319,7 +319,7 @@ + + + +- ++ + + + +@@ -338,7 +338,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/guest-node-host-dies.exp b/pengine/test10/guest-node-host-dies.exp +index 235813d..cd8d81a 100644 +--- a/pengine/test10/guest-node-host-dies.exp ++++ b/pengine/test10/guest-node-host-dies.exp +@@ -144,7 +144,7 @@ + + + +- ++ + + + +@@ -172,7 +172,7 @@ + + + +- ++ + + + +@@ -196,7 +196,7 @@ + + + +- ++ + + + +@@ -214,7 +214,7 @@ + + + +- ++ + + + +@@ -230,7 +230,7 @@ + + + +- ++ + + + +@@ -246,7 +246,7 @@ + + + +- ++ + + + +@@ -270,7 +270,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-asymmetric.exp b/pengine/test10/whitebox-asymmetric.exp +index 67d6c76..1178fb6 100644 +--- a/pengine/test10/whitebox-asymmetric.exp ++++ b/pengine/test10/whitebox-asymmetric.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-fail1.exp b/pengine/test10/whitebox-fail1.exp +index 5c4b26c..4cb6136 100644 +--- a/pengine/test10/whitebox-fail1.exp ++++ b/pengine/test10/whitebox-fail1.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +@@ -142,7 +142,7 @@ + + + +- ++ + + + +@@ -158,7 +158,7 @@ + + + +- ++ + + + +@@ -179,7 +179,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-fail2.exp b/pengine/test10/whitebox-fail2.exp +index 5c4b26c..4cb6136 100644 +--- a/pengine/test10/whitebox-fail2.exp ++++ b/pengine/test10/whitebox-fail2.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +@@ -142,7 +142,7 @@ + + + +- ++ + + + +@@ -158,7 +158,7 @@ + + + +- ++ + + + +@@ -179,7 +179,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-fail3.exp b/pengine/test10/whitebox-fail3.exp +index 95fc289..ec28f12 100644 +--- a/pengine/test10/whitebox-fail3.exp ++++ b/pengine/test10/whitebox-fail3.exp +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -40,7 +40,7 @@ + + + +- ++ + + + +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -98,7 +98,7 @@ + + + +- ++ + + + +@@ -114,7 +114,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp +index 9beb51d..6639eb9 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.exp ++++ b/pengine/test10/whitebox-imply-stop-on-fence.exp +@@ -272,7 +272,7 @@ + + + +- ++ + + + +@@ -288,7 +288,7 @@ + + + +- ++ + + + +@@ -309,7 +309,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-move.exp b/pengine/test10/whitebox-move.exp +index ecea360..4461890 100644 +--- a/pengine/test10/whitebox-move.exp ++++ b/pengine/test10/whitebox-move.exp +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -51,7 +51,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +@@ -130,7 +130,7 @@ + + + +- ++ + + + +@@ -149,7 +149,7 @@ + + + +- ++ + + + +@@ -158,7 +158,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-ms-ordering-move.exp b/pengine/test10/whitebox-ms-ordering-move.exp +index 3b5598f..dc5e473 100644 +--- a/pengine/test10/whitebox-ms-ordering-move.exp ++++ b/pengine/test10/whitebox-ms-ordering-move.exp +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -54,7 +54,7 @@ + + + +- ++ + + + +@@ -70,7 +70,7 @@ + + + +- ++ + + + +@@ -83,7 +83,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-ms-ordering.exp b/pengine/test10/whitebox-ms-ordering.exp +index 43cb49a..e5eb99c 100644 +--- a/pengine/test10/whitebox-ms-ordering.exp ++++ b/pengine/test10/whitebox-ms-ordering.exp +@@ -101,7 +101,7 @@ + + + +- ++ + + + +@@ -129,7 +129,7 @@ + + + +- ++ + + + +@@ -223,7 +223,7 @@ + + + +- ++ + + + +@@ -239,7 +239,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-nested-group.exp b/pengine/test10/whitebox-nested-group.exp +index 979a0f5..388c4d4 100644 +--- a/pengine/test10/whitebox-nested-group.exp ++++ b/pengine/test10/whitebox-nested-group.exp +@@ -62,7 +62,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -248,7 +248,7 @@ + + + +- ++ + + + +@@ -264,7 +264,7 @@ + + + +- ++ + + + +@@ -386,7 +386,7 @@ + + + +- ++ + + + +@@ -402,7 +402,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-orphan-ms.exp b/pengine/test10/whitebox-orphan-ms.exp +index 6df1b8d..ef81317 100644 +--- a/pengine/test10/whitebox-orphan-ms.exp ++++ b/pengine/test10/whitebox-orphan-ms.exp +@@ -135,7 +135,7 @@ + + + +- ++ + + + +@@ -151,7 +151,7 @@ + + + +- ++ + + + +@@ -167,7 +167,7 @@ + + + +- ++ + + + +@@ -176,7 +176,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-orphaned.exp b/pengine/test10/whitebox-orphaned.exp +index 32f8141..334e594 100644 +--- a/pengine/test10/whitebox-orphaned.exp ++++ b/pengine/test10/whitebox-orphaned.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -65,7 +65,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-start.exp b/pengine/test10/whitebox-start.exp +index 11438e9..ccccf60 100644 +--- a/pengine/test10/whitebox-start.exp ++++ b/pengine/test10/whitebox-start.exp +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -70,7 +70,7 @@ + + + +- ++ + + + +@@ -86,7 +86,7 @@ + + + +- ++ + + + +@@ -140,7 +140,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-stop.exp b/pengine/test10/whitebox-stop.exp +index a1aa3ba..354b6d4 100644 +--- a/pengine/test10/whitebox-stop.exp ++++ b/pengine/test10/whitebox-stop.exp +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -65,7 +65,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c +index 91bc4b4..33d751b 100644 +--- a/tools/attrd_updater.c ++++ b/tools/attrd_updater.c +@@ -79,6 +79,7 @@ static int do_query(const char *attr_name, const char *attr_node, gboolean query + static int do_update(char command, const char *attr_node, const char *attr_name, + const char *attr_value, const char *attr_section, + const char *attr_set, const char *attr_dampen, int attr_options); ++static const char *get_hostname(const char *name); + + int + main(int argc, char **argv) +@@ -198,6 +199,8 @@ main(int argc, char **argv) + crm_help('?', EX_USAGE); + #endif + } else { ++ ++ attr_node = get_hostname(attr_node); + crm_exit(do_update(command, attr_node, attr_name, attr_value, + attr_section, attr_set, attr_dampen, attr_options)); + } +@@ -351,6 +354,8 @@ do_query(const char *attr_name, const char *attr_node, gboolean query_all) + attr_node = "localhost"; + } + ++ attr_node = get_hostname(attr_node); ++ + /* Build and send attrd request, and get XML reply */ + rc = send_attrd_query(attr_name, attr_node, &reply); + if (rc != pcmk_ok) { +@@ -394,3 +399,28 @@ do_update(char command, const char *attr_node, const char *attr_name, + } + return rc; + } ++ ++static const char * ++get_hostname(const char *name) ++{ ++ if(name != NULL ++ && safe_str_neq(name, "auto") ++ && safe_str_neq(name, "localhost")) { ++ return name; ++ ++ } else { ++ const char *target = getenv(crm_meta_name(XML_RSC_ATTR_TARGET)); ++ const char *host_pyhsical = getenv(crm_meta_name(PCMK_ENV_PHYSICAL_HOST)); ++ const char *host_pcmk = getenv("OCF_RESKEY_" CRM_META "_" XML_LRM_ATTR_TARGET); ++ ++ /* It is important we use the names by which the PE knows us */ ++ if(safe_str_eq(target, "host") && host_pyhsical != NULL) { ++ return host_pyhsical; ++ ++ } else if(host_pcmk) { ++ return host_pcmk; ++ } ++ } ++ ++ return name; ++} +diff --git a/tools/crm_mon.c b/tools/crm_mon.c +index 29fc0e3..cad468e 100644 +--- a/tools/crm_mon.c ++++ b/tools/crm_mon.c +@@ -1726,7 +1726,7 @@ print_node_attribute(gpointer name, gpointer user_data) + const char *value = NULL; + struct mon_attr_data *data = (struct mon_attr_data *) user_data; + +- value = g_hash_table_lookup(data->node->details->attrs, name); ++ value = node_attribute_raw(data->node, name); + + /* Print attribute name and value */ + switch (output_format) { +-- +1.8.3.1 + + +From 50e264254fcdf90f5ad0ce429938c15495f99acb Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Thu, 24 Aug 2017 14:34:46 +1000 +Subject: [PATCH 02/21] Fix: attrd: Allow values for all nodes to be queried + +--- + tools/attrd_updater.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c +index 33d751b..de75496 100644 +--- a/tools/attrd_updater.c ++++ b/tools/attrd_updater.c +@@ -351,11 +351,11 @@ do_query(const char *attr_name, const char *attr_node, gboolean query_all) + attr_node = NULL; + } else if (attr_node == NULL) { + crm_debug("User did not specify node for query, using localhost"); +- attr_node = "localhost"; ++ attr_node = get_hostname(attr_node); ++ } else { ++ attr_node = get_hostname(attr_node); + } + +- attr_node = get_hostname(attr_node); +- + /* Build and send attrd request, and get XML reply */ + rc = send_attrd_query(attr_name, attr_node, &reply); + if (rc != pcmk_ok) { +-- +1.8.3.1 + + +From 67ebfc43eabf77e72d784084e23d3031bfc19710 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Thu, 24 Aug 2017 14:35:29 +1000 +Subject: [PATCH 03/21] Fix: PE: Correctly interpret colocation with bundles + +--- + pengine/graph.c | 17 ++++++++++++----- + 1 file changed, 12 insertions(+), 5 deletions(-) + +diff --git a/pengine/graph.c b/pengine/graph.c +index 3c5dc6b..990d8b8 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -413,9 +413,10 @@ graph_update_action(action_t * first, action_t * then, node_t * node, + } + + static void +-mark_start_blocked(resource_t *rsc) ++mark_start_blocked(resource_t *rsc, resource_t *reason) + { + GListPtr gIter = rsc->actions; ++ char *reason_text = crm_strdup_printf("colocation with %s", reason->id); + + for (; gIter != NULL; gIter = gIter->next) { + action_t *action = (action_t *) gIter->data; +@@ -424,11 +425,12 @@ mark_start_blocked(resource_t *rsc) + continue; + } + if (is_set(action->flags, pe_action_runnable)) { +- clear_bit(action->flags, pe_action_runnable); ++ pe_action_set_flag_reason(__FUNCTION__, __LINE__, action, NULL, reason_text, pe_action_runnable, FALSE); + update_colo_start_chain(action); + update_action(action); + } + } ++ free(reason_text); + } + + void +@@ -445,6 +447,11 @@ update_colo_start_chain(action_t *action) + return; + } + ++ if(rsc->parent) { ++ /* uber_parent() stops _before_ the bundle */ ++ rsc = rsc->parent; ++ } ++ + /* if rsc has children, all the children need to have start set to + * unrunnable before we follow the colo chain for the parent. */ + for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { +@@ -458,7 +465,7 @@ update_colo_start_chain(action_t *action) + for (gIter = rsc->rsc_cons_lhs; gIter != NULL; gIter = gIter->next) { + rsc_colocation_t *colocate_with = (rsc_colocation_t *)gIter->data; + if (colocate_with->score == INFINITY) { +- mark_start_blocked(colocate_with->rsc_lh); ++ mark_start_blocked(colocate_with->rsc_lh, action->rsc); + } + } + } +@@ -490,7 +497,7 @@ update_action(action_t * then) + if (then->required_runnable_before == 0) { + then->required_runnable_before = 1; + } +- clear_bit(then->flags, pe_action_runnable); ++ pe_clear_action_bit(then, pe_action_runnable); + /* We are relying on the pe_order_one_or_more clause of + * graph_update_action(), called as part of the: + * +@@ -1585,7 +1592,7 @@ graph_has_loop(action_t * init_action, action_t * action, action_wrapper_t * wra + } + + done: +- clear_bit(wrapper->action->flags, pe_action_tracking); ++ pe_clear_action_bit(wrapper->action, pe_action_tracking); + + return has_loop; + } +-- +1.8.3.1 + + +From 3392d49b99c8d594af54549a41c962c46e26d7fd Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Fri, 25 Aug 2017 12:46:37 +1000 +Subject: [PATCH 04/21] PE: bundle: Send an additional host list when + container-attribute-target=host + +The list is modified to use the name of the host where the bundles are +running so that agents can map between the two sets when necessary. +--- + lib/pengine/container.c | 1 + + pengine/clone.c | 8 +++--- + pengine/notif.c | 68 ++++++++++++++++++++++++++++++++++++++----------- + pengine/notif.h | 2 +- + 4 files changed, 59 insertions(+), 20 deletions(-) + +diff --git a/lib/pengine/container.c b/lib/pengine/container.c +index 53965cf..5831b58 100644 +--- a/lib/pengine/container.c ++++ b/lib/pengine/container.c +@@ -769,6 +769,7 @@ container_unpack(resource_t * rsc, pe_working_set_t * data_set) + container_data->docker_host_options = buffer; + if(container_data->attribute_target) { + g_hash_table_replace(rsc->meta, strdup(XML_RSC_ATTR_TARGET), strdup(container_data->attribute_target)); ++ g_hash_table_replace(container_data->child->meta, strdup(XML_RSC_ATTR_TARGET), strdup(container_data->attribute_target)); + } + + } else { +diff --git a/pengine/clone.c b/pengine/clone.c +index 1722166..41cb5cb 100644 +--- a/pengine/clone.c ++++ b/pengine/clone.c +@@ -1227,25 +1227,25 @@ clone_expand(resource_t * rsc, pe_working_set_t * data_set) + + if (clone_data->start_notify) { + collect_notification_data(rsc, TRUE, TRUE, clone_data->start_notify); +- expand_notification_data(clone_data->start_notify, data_set); ++ expand_notification_data(rsc, clone_data->start_notify, data_set); + create_notifications(rsc, clone_data->start_notify, data_set); + } + + if (clone_data->stop_notify) { + collect_notification_data(rsc, TRUE, TRUE, clone_data->stop_notify); +- expand_notification_data(clone_data->stop_notify, data_set); ++ expand_notification_data(rsc, clone_data->stop_notify, data_set); + create_notifications(rsc, clone_data->stop_notify, data_set); + } + + if (clone_data->promote_notify) { + collect_notification_data(rsc, TRUE, TRUE, clone_data->promote_notify); +- expand_notification_data(clone_data->promote_notify, data_set); ++ expand_notification_data(rsc, clone_data->promote_notify, data_set); + create_notifications(rsc, clone_data->promote_notify, data_set); + } + + if (clone_data->demote_notify) { + collect_notification_data(rsc, TRUE, TRUE, clone_data->demote_notify); +- expand_notification_data(clone_data->demote_notify, data_set); ++ expand_notification_data(rsc, clone_data->demote_notify, data_set); + create_notifications(rsc, clone_data->demote_notify, data_set); + } + +diff --git a/pengine/notif.c b/pengine/notif.c +index b8135b5..19a9751 100644 +--- a/pengine/notif.c ++++ b/pengine/notif.c +@@ -17,6 +17,7 @@ + */ + + #include ++#include + #include + #include + #include +@@ -81,33 +82,60 @@ static notify_entry_t *dup_notify_entry(notify_entry_t *entry) + return dup; + } + +-static char * +-expand_node_list(GListPtr list) ++static void ++expand_node_list(GListPtr list, char **uname, char **metal) + { + GListPtr gIter = NULL; + char *node_list = NULL; ++ char *metal_list = NULL; + ++ CRM_ASSERT(uname != NULL); + if (list == NULL) { +- return strdup(" "); ++ *uname = strdup(" "); ++ if(metal) { ++ *metal = strdup(" "); ++ } ++ return; + } + + for (gIter = list; gIter != NULL; gIter = gIter->next) { ++ int len = 0; ++ int existing_len = 0; + node_t *node = (node_t *) gIter->data; + +- if (node->details->uname) { +- int existing_len = 0; +- int len = 2 + strlen(node->details->uname); ++ CRM_ASSERT(node->details->uname); ++ len = 2 + strlen(node->details->uname); ++ ++ if(node_list) { ++ existing_len = strlen(node_list); ++ } ++// crm_trace("Adding %s (%dc) at offset %d", node->details->uname, len - 2, existing_len); ++ node_list = realloc_safe(node_list, len + existing_len); ++ sprintf(node_list + existing_len, "%s%s", existing_len == 0 ? "":" ", node->details->uname); ++ ++ if(metal) { ++ existing_len = 0; ++ if(metal_list) { ++ existing_len = strlen(metal_list); ++ } + +- if(node_list) { +- existing_len = strlen(node_list); ++ if(node->details->remote_rsc ++ && node->details->remote_rsc->container ++ && node->details->remote_rsc->container->running_on) { ++ node = node->details->remote_rsc->container->running_on->data; + } +- crm_trace("Adding %s (%dc) at offset %d", node->details->uname, len - 2, existing_len); +- node_list = realloc_safe(node_list, len + existing_len); +- sprintf(node_list + existing_len, "%s%s", existing_len == 0 ? "":" ", node->details->uname); ++ ++ CRM_ASSERT(node->details->uname); ++ len = 2 + strlen(node->details->uname); ++ metal_list = realloc_safe(metal_list, len + existing_len); ++ sprintf(metal_list + existing_len, "%s%s", existing_len == 0 ? "":" ", node->details->uname); + } + } + +- return node_list; ++ *uname = node_list; ++ if(metal) { ++ *metal = metal_list; ++ } + } + + static void +@@ -495,7 +523,7 @@ collect_notification_data(resource_t * rsc, gboolean state, gboolean activity, + } + + gboolean +-expand_notification_data(notify_data_t * n_data, pe_working_set_t * data_set) ++expand_notification_data(resource_t *rsc, notify_data_t * n_data, pe_working_set_t * data_set) + { + /* Expand the notification entries into a key=value hashtable + * This hashtable is later used in action2xml() +@@ -503,6 +531,7 @@ expand_notification_data(notify_data_t * n_data, pe_working_set_t * data_set) + gboolean required = FALSE; + char *rsc_list = NULL; + char *node_list = NULL; ++ char *metal_list = NULL; + GListPtr nodes = NULL; + + if (n_data->stop) { +@@ -576,13 +605,22 @@ expand_notification_data(notify_data_t * n_data, pe_working_set_t * data_set) + g_hash_table_insert(n_data->keys, strdup("notify_inactive_resource"), rsc_list); + + nodes = g_hash_table_get_values(n_data->allowed_nodes); +- node_list = expand_node_list(nodes); ++ expand_node_list(nodes, &node_list, NULL); + g_hash_table_insert(n_data->keys, strdup("notify_available_uname"), node_list); + g_list_free(nodes); + +- node_list = expand_node_list(data_set->nodes); ++ expand_node_list(data_set->nodes, &node_list, &metal_list); + g_hash_table_insert(n_data->keys, strdup("notify_all_uname"), node_list); + ++ { ++ const char *source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET); ++ if(safe_str_eq("host", source)) { ++ g_hash_table_insert(n_data->keys, strdup("notify_all_hosts"), metal_list); ++ } else { ++ free(metal_list); ++ } ++ } ++ + if (required && n_data->pre) { + update_action_flags(n_data->pre, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__); + update_action_flags(n_data->pre_done, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__); +diff --git a/pengine/notif.h b/pengine/notif.h +index f299342..da68874 100644 +--- a/pengine/notif.h ++++ b/pengine/notif.h +@@ -28,7 +28,7 @@ notify_data_t * create_notification_boundaries(resource_t *rsc, + void collect_notification_data(resource_t *rsc, gboolean state, + gboolean activity, notify_data_t *n_data); + +-gboolean expand_notification_data(notify_data_t *n_data, ++gboolean expand_notification_data(resource_t *rsc, notify_data_t *n_data, + pe_working_set_t *data_set); + + void create_notifications(resource_t *rsc, notify_data_t *n_data, +-- +1.8.3.1 + + +From ccbdb2a215bad69c9e6ac2d74c0bbb1d1719db5a Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Mon, 28 Aug 2017 12:17:16 +1000 +Subject: [PATCH 05/21] Tools: Allow bundle resources to read/write master + scores based on the same logic as attrd (cf34f4c) + +--- + include/crm/attrd.h | 2 ++ + lib/common/utils.c | 28 ++++++++++++++++++++++++++++ + pacemaker.spec.in | 4 +++- + tools/Makefile.am | 2 +- + tools/attrd_updater.c | 40 +++++++++------------------------------- + tools/crm_attribute.c | 14 ++++++-------- + tools/crm_master | 6 ++---- + 7 files changed, 51 insertions(+), 45 deletions(-) + +diff --git a/include/crm/attrd.h b/include/crm/attrd.h +index e294838..240f192 100644 +--- a/include/crm/attrd.h ++++ b/include/crm/attrd.h +@@ -24,6 +24,8 @@ + #define attrd_opt_remote 0x001 + #define attrd_opt_private 0x002 + ++const char *attrd_get_target(const char *name); ++ + int attrd_update_delegate(crm_ipc_t * ipc, char command, const char *host, + const char *name, const char *value, const char *section, + const char *set, const char *dampen, const char *user_name, int options); +diff --git a/lib/common/utils.c b/lib/common/utils.c +index f594c12..290a661 100644 +--- a/lib/common/utils.c ++++ b/lib/common/utils.c +@@ -2156,3 +2156,31 @@ crm_gnutls_global_init(void) + gnutls_global_init(); + } + #endif ++ ++const char * ++attrd_get_target(const char *name) ++{ ++ if(safe_str_eq(name, "auto") || safe_str_eq(name, "localhost")) { ++ name = NULL; ++ } ++ ++ if(name != NULL) { ++ return name; ++ ++ } else { ++ const char *target = getenv(crm_meta_name(XML_RSC_ATTR_TARGET)); ++ const char *host_pyhsical = getenv(crm_meta_name(PCMK_ENV_PHYSICAL_HOST)); ++ const char *host_pcmk = getenv("OCF_RESKEY_" CRM_META "_" XML_LRM_ATTR_TARGET); ++ ++ /* It is important we use the names by which the PE knows us */ ++ if(safe_str_eq(target, "host") && host_pyhsical != NULL) { ++ return host_pyhsical; ++ ++ } else if(host_pcmk) { ++ return host_pcmk; ++ } ++ } ++ ++ // TODO? Call get_local_node_name() if name == NULL ++ return name; ++} +diff --git a/pacemaker.spec.in b/pacemaker.spec.in +index 642af74..d0bc2c3 100644 +--- a/pacemaker.spec.in ++++ b/pacemaker.spec.in +@@ -601,6 +601,7 @@ exit 0 + %exclude %{_sbindir}/pacemaker_remoted + %{_libexecdir}/pacemaker/* + ++%{_sbindir}/attrd_updater + %{_sbindir}/crm_attribute + %{_sbindir}/crm_master + %{_sbindir}/crm_node +@@ -619,6 +620,7 @@ exit 0 + %doc %{_mandir}/man7/ocf_pacemaker_o2cb.* + %doc %{_mandir}/man7/ocf_pacemaker_remote.* + %doc %{_mandir}/man8/crm_attribute.* ++%doc %{_mandir}/man8/attrd_updater.* + %doc %{_mandir}/man8/crm_node.* + %doc %{_mandir}/man8/crm_master.* + %if %{with cman} +@@ -668,7 +670,6 @@ exit 0 + %config(noreplace) %{_sysconfdir}/init/crm_mon.conf + %endif + +-%{_sbindir}/attrd_updater + %{_sbindir}/cibadmin + %{_sbindir}/crm_diff + %{_sbindir}/crm_error +@@ -704,6 +705,7 @@ exit 0 + %exclude %{_mandir}/man7/ocf_pacemaker_o2cb.* + %exclude %{_mandir}/man7/ocf_pacemaker_remote.* + %doc %{_mandir}/man8/* ++%exclude %{_mandir}/man8/attrd_updater.* + %exclude %{_mandir}/man8/crm_attribute.* + %exclude %{_mandir}/man8/crm_node.* + %exclude %{_mandir}/man8/crm_master.* +diff --git a/tools/Makefile.am b/tools/Makefile.am +index 3548035..a601b8d 100644 +--- a/tools/Makefile.am ++++ b/tools/Makefile.am +@@ -121,7 +121,7 @@ iso8601_SOURCES = test.iso8601.c + iso8601_LDADD = $(COMMONLIBS) + + attrd_updater_SOURCES = attrd_updater.c +-attrd_updater_LDADD = $(COMMONLIBS) ++attrd_updater_LDADD = $(top_builddir)/lib/cluster/libcrmcluster.la $(COMMONLIBS) + + crm_ticket_SOURCES = crm_ticket.c + crm_ticket_LDADD = $(top_builddir)/lib/pengine/libpe_rules.la \ +diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c +index de75496..5e8b837 100644 +--- a/tools/attrd_updater.c ++++ b/tools/attrd_updater.c +@@ -32,6 +32,7 @@ + #include + + #include ++#include + + /* *INDENT-OFF* */ + static struct crm_option long_options[] = { +@@ -79,7 +80,6 @@ static int do_query(const char *attr_name, const char *attr_node, gboolean query + static int do_update(char command, const char *attr_node, const char *attr_name, + const char *attr_value, const char *attr_section, + const char *attr_set, const char *attr_dampen, int attr_options); +-static const char *get_hostname(const char *name); + + int + main(int argc, char **argv) +@@ -200,7 +200,10 @@ main(int argc, char **argv) + #endif + } else { + +- attr_node = get_hostname(attr_node); ++ attr_node = attrd_get_target(attr_node); ++ if (attr_node == NULL) { ++ attr_node = get_local_node_name(); ++ } + crm_exit(do_update(command, attr_node, attr_name, attr_value, + attr_section, attr_set, attr_dampen, attr_options)); + } +@@ -349,11 +352,11 @@ do_query(const char *attr_name, const char *attr_node, gboolean query_all) + /* Decide which node(s) to query */ + if (query_all == TRUE) { + attr_node = NULL; +- } else if (attr_node == NULL) { +- crm_debug("User did not specify node for query, using localhost"); +- attr_node = get_hostname(attr_node); + } else { +- attr_node = get_hostname(attr_node); ++ attr_node = attrd_get_target(attr_node); ++ if (attr_node == NULL) { ++ attr_node = get_local_node_name(); ++ } + } + + /* Build and send attrd request, and get XML reply */ +@@ -399,28 +402,3 @@ do_update(char command, const char *attr_node, const char *attr_name, + } + return rc; + } +- +-static const char * +-get_hostname(const char *name) +-{ +- if(name != NULL +- && safe_str_neq(name, "auto") +- && safe_str_neq(name, "localhost")) { +- return name; +- +- } else { +- const char *target = getenv(crm_meta_name(XML_RSC_ATTR_TARGET)); +- const char *host_pyhsical = getenv(crm_meta_name(PCMK_ENV_PHYSICAL_HOST)); +- const char *host_pcmk = getenv("OCF_RESKEY_" CRM_META "_" XML_LRM_ATTR_TARGET); +- +- /* It is important we use the names by which the PE knows us */ +- if(safe_str_eq(target, "host") && host_pyhsical != NULL) { +- return host_pyhsical; +- +- } else if(host_pcmk) { +- return host_pcmk; +- } +- } +- +- return name; +-} +diff --git a/tools/crm_attribute.c b/tools/crm_attribute.c +index 60ded8d..b511282 100644 +--- a/tools/crm_attribute.c ++++ b/tools/crm_attribute.c +@@ -246,15 +246,13 @@ main(int argc, char **argv) + + } else if (safe_str_eq(type, XML_CIB_TAG_CRMCONFIG)) { + } else if (safe_str_neq(type, XML_CIB_TAG_TICKETS)) { ++ /* If we are being called from a resource agent via the cluster, ++ * the correct local node name will be passed as an environment ++ * variable. Otherwise, we have to ask the cluster. ++ */ ++ dest_uname = attrd_get_target(dest_uname); + if (dest_uname == NULL) { +- /* If we are being called from a resource agent via the cluster, +- * the correct local node name will be passed as an environment +- * variable. Otherwise, we have to ask the cluster. +- */ +- dest_uname = getenv("OCF_RESKEY_" CRM_META "_" XML_LRM_ATTR_TARGET); +- if (dest_uname == NULL) { +- dest_uname = get_local_node_name(); +- } ++ dest_uname = get_local_node_name(); + } + + rc = query_node_uuid(the_cib, dest_uname, &dest_node, &is_remote_node); +diff --git a/tools/crm_master b/tools/crm_master +index 7cce64d..7e31cea 100755 +--- a/tools/crm_master ++++ b/tools/crm_master +@@ -1,7 +1,5 @@ + #!/bin/bash + +-target=`crm_node -n` +- + TEMP=`getopt -o qDGQVN:U:v:i:l:r: --long version,help,resource:,node:,uname:,attr-value:,id:,update:,delete-attr,get-value,attr-id:,lifetime:,quiet \ + -n 'crm_master' -- "$@"` + +@@ -12,7 +10,7 @@ eval set -- "$TEMP" + + while true ; do + case "$1" in +- -N|--node|-U|--uname) target="$2"; shift; shift;; ++ -N|--node|-U|--uname) options="$options $1 $2"; shift; shift;; + -v|--attr-value|--update|-i|--id|--attr-id|-l|--lifetime) options="$options $1 $2"; shift; shift;; + -Q|-q|--quiet|-D|--delete-attr|-G|--get-value|-V) options="$options $1"; shift;; + -r|--resource) OCF_RESOURCE_INSTANCE=$2; shift; shift;; +@@ -53,4 +51,4 @@ if [ -z "$OCF_RESOURCE_INSTANCE" ]; then + exit 1 + fi + +-crm_attribute -N $target -n master-$OCF_RESOURCE_INSTANCE $options ++crm_attribute -n master-$OCF_RESOURCE_INSTANCE $options +-- +1.8.3.1 + + +From e3485d6fd5e9e6ec05d9cd6682d46da3e463d6c7 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 13 Sep 2017 13:31:09 +1000 +Subject: [PATCH 06/21] Fix: PE: Prevent graph loops when fencing the host out + from underneath a bundle + +--- + pengine/allocate.c | 9 ++++++++- + 1 file changed, 8 insertions(+), 1 deletion(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 958fb23..d610035 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1416,7 +1416,14 @@ fence_guest(pe_node_t *node, pe_action_t *done, pe_working_set_t *data_set) + * it is restarted, so we always order pseudo-fencing after stop, not start + * (even though start might be closer to what is done for a real reboot). + */ +- if (stop) { ++ if(stop && is_set(stop->flags, pe_action_pseudo)) { ++ pe_action_t *parent_stonith_op = pe_fence_op(stop->node, NULL, FALSE, NULL, data_set); ++ crm_info("Implying guest node %s is down (action %d) after %s fencing", ++ node->details->uname, stonith_op->id, stop->node->details->uname); ++ order_actions(parent_stonith_op, stonith_op, ++ pe_order_runnable_left|pe_order_implies_then); ++ ++ } else if (stop) { + order_actions(stop, stonith_op, + pe_order_runnable_left|pe_order_implies_then); + crm_info("Implying guest node %s is down (action %d) " +-- +1.8.3.1 + + +From 2709e5c2e231e80da557acf3b736f970516ffbb9 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 13 Sep 2017 14:14:31 +1000 +Subject: [PATCH 07/21] Fix: PE: Do not send notifications to unclean bundles + +--- + pengine/notif.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/pengine/notif.c b/pengine/notif.c +index 19a9751..76c5a4a 100644 +--- a/pengine/notif.c ++++ b/pengine/notif.c +@@ -691,7 +691,7 @@ create_notifications(resource_t * rsc, notify_data_t * n_data, pe_working_set_t + * action. There's no reason to send the fenced node a stop notification */ + if (stop && + is_set(stop->flags, pe_action_pseudo) && +- current_node->details->unclean) { ++ (current_node->details->unclean || current_node->details->remote_requires_reset) ) { + + continue; + } +-- +1.8.3.1 + + +From d29ba5dce8d2bccf4a0c906c3bd02a87f7e42974 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 13 Sep 2017 20:06:15 +1000 +Subject: [PATCH 08/21] Test: PE: Update bundle fence ordering + +--- + pengine/test10/complex_enforce_colo.summary | 4 ++-- + pengine/test10/enforce-colo1.summary | 2 +- + pengine/test10/whitebox-imply-stop-on-fence.dot | 4 ++-- + pengine/test10/whitebox-imply-stop-on-fence.exp | 4 ++-- + pengine/test10/whitebox-imply-stop-on-fence.summary | 6 +++--- + 5 files changed, 10 insertions(+), 10 deletions(-) + +diff --git a/pengine/test10/complex_enforce_colo.summary b/pengine/test10/complex_enforce_colo.summary +index 2332269..0426e98 100644 +--- a/pengine/test10/complex_enforce_colo.summary ++++ b/pengine/test10/complex_enforce_colo.summary +@@ -113,7 +113,7 @@ Transition Summary: + * Stop glance-api:2 (rhos6-node3) + * Stop cinder-api ( rhos6-node1 ) due to unrunnable keystone-clone running + * Stop cinder-scheduler ( rhos6-node1 ) due to required cinder-api start +- * Stop cinder-volume ( rhos6-node1 ) ++ * Stop cinder-volume ( rhos6-node1 ) due to colocation with cinder-scheduler + * Stop swift-account:0 (rhos6-node1) + * Stop swift-account:1 (rhos6-node2) + * Stop swift-account:2 (rhos6-node3) +@@ -194,7 +194,7 @@ Transition Summary: + * Stop heat-api-cloudwatch:0 ( rhos6-node1 ) due to required heat-api-cfn:0 start + * Stop heat-api-cloudwatch:1 ( rhos6-node2 ) due to required heat-api-cfn:1 start + * Stop heat-api-cloudwatch:2 ( rhos6-node3 ) due to required heat-api-cfn:2 start +- * Stop heat-engine ( rhos6-node2 ) due to required heat-api-cloudwatch-clone running ++ * Stop heat-engine ( rhos6-node2 ) due to colocation with heat-api-cloudwatch-clone + + Executing cluster transition: + * Pseudo action: glance-api-clone_stop_0 +diff --git a/pengine/test10/enforce-colo1.summary b/pengine/test10/enforce-colo1.summary +index 64815ae..4968326 100644 +--- a/pengine/test10/enforce-colo1.summary ++++ b/pengine/test10/enforce-colo1.summary +@@ -10,7 +10,7 @@ Online: [ rhel7-auto1 rhel7-auto2 rhel7-auto3 ] + central (ocf::heartbeat:Dummy): Started rhel7-auto3 + + Transition Summary: +- * Stop engine ( rhel7-auto3 ) ++ * Stop engine ( rhel7-auto3 ) due to colocation with central + * Stop keystone:0 (rhel7-auto2) due to node availability + * Stop keystone:1 (rhel7-auto3) due to node availability + * Stop keystone:2 (rhel7-auto1) due to node availability +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.dot b/pengine/test10/whitebox-imply-stop-on-fence.dot +index 5885d4b..8ef42fd 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.dot ++++ b/pengine/test10/whitebox-imply-stop-on-fence.dot +@@ -7,7 +7,6 @@ + "R-lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] + "R-lxc-01_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "R-lxc-01_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] +-"R-lxc-01_kiff-01_stop_0 kiff-01" -> "stonith 'reboot' lxc-01_kiff-01" [ style = bold] + "R-lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] + "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] + "R-lxc-02_kiff-01_start_0 kiff-02" -> "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style = bold] +@@ -16,7 +15,6 @@ + "R-lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] + "R-lxc-02_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "R-lxc-02_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] +-"R-lxc-02_kiff-01_stop_0 kiff-01" -> "stonith 'reboot' lxc-02_kiff-01" [ style = bold] + "R-lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] + "all_stopped" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] +@@ -77,6 +75,8 @@ + "stonith 'reboot' kiff-01" -> "lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] + "stonith 'reboot' kiff-01" -> "shared0-clone_stop_0" [ style = bold] + "stonith 'reboot' kiff-01" -> "shared0_stop_0 kiff-01" [ style = bold] ++"stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-01_kiff-01" [ style = bold] ++"stonith 'reboot' kiff-01" -> "stonith 'reboot' lxc-02_kiff-01" [ style = bold] + "stonith 'reboot' kiff-01" -> "stonith_complete" [ style = bold] + "stonith 'reboot' kiff-01" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' lxc-01_kiff-01" -> "stonith_complete" [ style = bold] +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp +index 6639eb9..d1e22ee 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.exp ++++ b/pengine/test10/whitebox-imply-stop-on-fence.exp +@@ -417,7 +417,7 @@ + + + +- ++ + + + +@@ -432,7 +432,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary +index a3e9ce8..87d7b4f 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.summary ++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary +@@ -43,15 +43,15 @@ Executing cluster transition: + * Fencing kiff-01 (reboot) + * Pseudo action: lxc-01_kiff-01_stop_0 + * Pseudo action: lxc-02_kiff-01_stop_0 +- * Pseudo action: R-lxc-01_kiff-01_stop_0 +- * Pseudo action: R-lxc-02_kiff-01_stop_0 + * Pseudo action: stonith-lxc-02_kiff-01-reboot on lxc-02_kiff-01 + * Pseudo action: stonith-lxc-01_kiff-01-reboot on lxc-01_kiff-01 + * Pseudo action: stonith_complete ++ * Pseudo action: R-lxc-01_kiff-01_stop_0 ++ * Pseudo action: R-lxc-02_kiff-01_stop_0 ++ * Pseudo action: vm-fs_stop_0 + * Pseudo action: shared0-clone_stop_0 + * Resource action: R-lxc-01_kiff-01 start on kiff-02 + * Resource action: R-lxc-02_kiff-01 start on kiff-02 +- * Pseudo action: vm-fs_stop_0 + * Resource action: lxc-01_kiff-01 start on kiff-02 + * Resource action: lxc-02_kiff-01 start on kiff-02 + * Pseudo action: shared0_stop_0 +-- +1.8.3.1 + + +From 8f19403bdaf4722b82d794c10e9911976bd68b82 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 13 Sep 2017 20:14:21 +1000 +Subject: [PATCH 09/21] Test: PE: Prevent graph loops when fencing the host out + from underneath a bundle + +--- + pengine/regression.sh | 2 + + pengine/test10/bundle-order-fencing.dot | 442 ++++++ + pengine/test10/bundle-order-fencing.exp | 1982 +++++++++++++++++++++++++++ + pengine/test10/bundle-order-fencing.scores | 1148 ++++++++++++++++ + pengine/test10/bundle-order-fencing.summary | 222 +++ + pengine/test10/bundle-order-fencing.xml | 799 +++++++++++ + 6 files changed, 4595 insertions(+) + create mode 100644 pengine/test10/bundle-order-fencing.dot + create mode 100644 pengine/test10/bundle-order-fencing.exp + create mode 100644 pengine/test10/bundle-order-fencing.scores + create mode 100644 pengine/test10/bundle-order-fencing.summary + create mode 100644 pengine/test10/bundle-order-fencing.xml + +diff --git a/pengine/regression.sh b/pengine/regression.sh +index d1a8a3f..6c3be88 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -811,6 +811,8 @@ do_test bundle-order-startup-clone-2 "Bundle startup with clones" + do_test bundle-order-stop-clone "Stop bundle because clone is stopping" + do_test bundle-nested-colocation "Colocation of nested connection resources" + ++do_test bundle-order-fencing "Order pseudo bundle fencing after parent node fencing if both are happening" ++ + echo "" + do_test whitebox-fail1 "Fail whitebox container rsc." + do_test whitebox-fail2 "Fail whitebox container rsc lrmd connection." +diff --git a/pengine/test10/bundle-order-fencing.dot b/pengine/test10/bundle-order-fencing.dot +new file mode 100644 +index 0000000..0db9605 +--- /dev/null ++++ b/pengine/test10/bundle-order-fencing.dot +@@ -0,0 +1,442 @@ ++digraph "g" { ++"Cancel redis_monitor_45000 redis-bundle-1" -> "redis_promote_0 redis-bundle-1" [ style = bold] ++"Cancel redis_monitor_45000 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"Cancel redis_monitor_60000 redis-bundle-1" -> "redis_promote_0 redis-bundle-1" [ style = bold] ++"Cancel redis_monitor_60000 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"all_stopped" -> "stonith-fence_ipmilan-5254000dcb3f_start_0 controller-2" [ style = bold] ++"all_stopped" -> "stonith-fence_ipmilan-5254003e8e97_start_0 controller-1" [ style = bold] ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-0_monitor_60000 controller-2" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-0_start_0 controller-2" -> "galera-bundle-0_monitor_60000 controller-2" [ style = dashed] ++"galera-bundle-0_start_0 controller-2" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] ++"galera-bundle-0_start_0 controller-2" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] ++"galera-bundle-0_start_0 controller-2" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera-bundle-0_start_0 controller-2" [ style=dashed color="red" fontcolor="black"] ++"galera-bundle-0_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"galera-bundle-0_stop_0 controller-0" -> "galera-bundle-0_start_0 controller-2" [ style = dashed] ++"galera-bundle-0_stop_0 controller-0" -> "galera-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"galera-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"galera-bundle-docker-0_stop_0 controller-0" -> "galera-bundle_stopped_0" [ style = bold] ++"galera-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_demote_0" -> "galera-bundle-master_demoted_0" [ style = bold] ++"galera-bundle-master_demote_0" -> "galera_demote_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_demoted_0" -> "galera-bundle-master_start_0" [ style = bold] ++"galera-bundle-master_demoted_0" -> "galera-bundle-master_stop_0" [ style = bold] ++"galera-bundle-master_demoted_0" -> "galera-bundle_demoted_0" [ style = bold] ++"galera-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_running_0" -> "galera-bundle_running_0" [ style = bold] ++"galera-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_start_0" -> "galera-bundle-master_running_0" [ style = bold] ++"galera-bundle-master_start_0" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_stop_0" -> "galera-bundle-master_stopped_0" [ style = bold] ++"galera-bundle-master_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-master_stopped_0" -> "galera-bundle-master_start_0" [ style = bold] ++"galera-bundle-master_stopped_0" -> "galera-bundle_stopped_0" [ style = bold] ++"galera-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demote_0" -> "galera-bundle-master_demote_0" [ style = bold] ++"galera-bundle_demote_0" -> "galera-bundle_demoted_0" [ style = bold] ++"galera-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_demoted_0" -> "galera-bundle_start_0" [ style = bold] ++"galera-bundle_demoted_0" -> "galera-bundle_stop_0" [ style = bold] ++"galera-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] ++"galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_stop_0" -> "galera-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"galera-bundle_stop_0" -> "galera-bundle-master_stop_0" [ style = bold] ++"galera-bundle_stop_0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle_stopped_0" -> "galera-bundle_start_0" [ style = bold] ++"galera-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"galera_demote_0 galera-bundle-0" -> "galera-bundle-master_demoted_0" [ style = bold] ++"galera_demote_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] ++"galera_demote_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] ++"galera_demote_0 galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"galera_demote_0 galera-bundle-0" [ style=bold color="green" fontcolor="orange"] ++"galera_monitor_20000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_monitor_30000 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = dashed] ++"galera_start_0 galera-bundle-0" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] ++"galera_start_0 galera-bundle-0" -> "galera_monitor_30000 galera-bundle-0" [ style = dashed] ++"galera_start_0 galera-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"galera_stop_0 galera-bundle-0" -> "all_stopped" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold] ++"galera_stop_0 galera-bundle-0" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"galera_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="orange"] ++"haproxy-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"haproxy-bundle-docker-0_stop_0 controller-0" -> "haproxy-bundle_stopped_0" [ style = bold] ++"haproxy-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"haproxy-bundle_stop_0" -> "haproxy-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"haproxy-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"haproxy-bundle_stopped_0" -> "ip-10.0.0.109_stop_0 controller-0" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-172.17.4.11_stop_0 controller-0" [ style = bold] ++"haproxy-bundle_stopped_0" -> "ip-192.168.24.7_stop_0 controller-0" [ style = bold] ++"haproxy-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"ip-10.0.0.109_monitor_10000 controller-1" [ style=bold color="green" fontcolor="black"] ++"ip-10.0.0.109_start_0 controller-1" -> "ip-10.0.0.109_monitor_10000 controller-1" [ style = bold] ++"ip-10.0.0.109_start_0 controller-1" [ style=bold color="green" fontcolor="black"] ++"ip-10.0.0.109_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"ip-10.0.0.109_stop_0 controller-0" -> "ip-10.0.0.109_start_0 controller-1" [ style = bold] ++"ip-10.0.0.109_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"ip-172.17.4.11_monitor_10000 controller-1" [ style=bold color="green" fontcolor="black"] ++"ip-172.17.4.11_start_0 controller-1" -> "ip-172.17.4.11_monitor_10000 controller-1" [ style = bold] ++"ip-172.17.4.11_start_0 controller-1" [ style=bold color="green" fontcolor="black"] ++"ip-172.17.4.11_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"ip-172.17.4.11_stop_0 controller-0" -> "ip-172.17.4.11_start_0 controller-1" [ style = bold] ++"ip-172.17.4.11_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"ip-192.168.24.7_monitor_10000 controller-2" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.24.7_start_0 controller-2" -> "ip-192.168.24.7_monitor_10000 controller-2" [ style = bold] ++"ip-192.168.24.7_start_0 controller-2" [ style=bold color="green" fontcolor="black"] ++"ip-192.168.24.7_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"ip-192.168.24.7_stop_0 controller-0" -> "ip-192.168.24.7_start_0 controller-2" [ style = bold] ++"ip-192.168.24.7_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-0_monitor_60000 controller-1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq-bundle-0_monitor_60000 controller-1" [ style = dashed] ++"rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-0_start_0 controller-1" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq-bundle-0_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-0_stop_0 controller-0" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed] ++"rabbitmq-bundle-0_stop_0 controller-0" -> "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"rabbitmq-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle-clone_pre_notify_start_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" -> "rabbitmq-bundle_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq_post_notify_running_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq_post_notify_running_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" -> "rabbitmq_post_notify_running_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_stopped_0" -> "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-clone_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq_pre_notify_start_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_start_0" -> "rabbitmq_pre_notify_start_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_stop_0" -> "rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq-bundle-clone_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_running_0" -> "rabbitmq-bundle-clone_post_notify_running_0" [ style = bold] ++"rabbitmq-bundle-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] ++"rabbitmq-bundle-clone_start_0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_post_notify_stopped_0" [ style = bold] ++"rabbitmq-bundle-clone_stopped_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 controller-0" -> "rabbitmq-bundle_stopped_0" [ style = bold] ++"rabbitmq-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"rabbitmq-bundle_stop_0" -> "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"rabbitmq-bundle_stop_0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] ++"rabbitmq-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] ++"rabbitmq_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_post_notify_running_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq_post_notify_running_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_post_notify_running_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq_post_notify_running_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_post_notify_running_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] ++"rabbitmq_post_notify_running_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] ++"rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] ++"rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-post_notify_stopped_0" [ style = bold] ++"rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] ++"rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_post_notify_stonith_0" -> "rabbitmq_confirmed-post_notify_stonith_0" [ style = bold] ++"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-1" [ style = bold] ++"rabbitmq_post_notify_stonith_0" -> "rabbitmq_post_notify_stonith_0 rabbitmq-bundle-2" [ style = bold] ++"rabbitmq_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq_pre_notify_start_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] ++"rabbitmq_pre_notify_start_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_pre_notify_start_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-pre_notify_start_0" [ style = bold] ++"rabbitmq_pre_notify_start_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-pre_notify_stop_0" [ style = bold] ++"rabbitmq_pre_notify_stop_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq_start_0 rabbitmq-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "all_stopped" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] ++"rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-0_monitor_60000 controller-1" [ style=dashed color="red" fontcolor="black"] ++"redis-bundle-0_start_0 controller-1" -> "redis-bundle-0_monitor_60000 controller-1" [ style = dashed] ++"redis-bundle-0_start_0 controller-1" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis-bundle-0_start_0 controller-1" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis-bundle-0_start_0 controller-1" -> "redis_start_0 redis-bundle-0" [ style = dashed] ++"redis-bundle-0_start_0 controller-1" [ style=dashed color="red" fontcolor="black"] ++"redis-bundle-0_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"redis-bundle-0_stop_0 controller-0" -> "redis-bundle-0_start_0 controller-1" [ style = dashed] ++"redis-bundle-0_stop_0 controller-0" -> "redis-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"redis-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"redis-bundle-docker-0_stop_0 controller-0" -> "redis-bundle_stopped_0" [ style = bold] ++"redis-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_start_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle-master_pre_notify_stop_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis-bundle_demoted_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_demoted_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis-bundle_promoted_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_promoted_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis-bundle_running_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_running_0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle-master_pre_notify_promote_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle-master_pre_notify_start_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis-bundle_stopped_0" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_stopped_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] ++"redis-bundle-master_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_demote_0" -> "redis-bundle-master_post_notify_demoted_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_start_0" -> "redis-bundle-master_start_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_confirmed-pre_notify_stop_0" -> "redis-bundle-master_post_notify_stopped_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] ++"redis-bundle-master_confirmed-pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_demote_0" -> "redis-bundle-master_demoted_0" [ style = bold] ++"redis-bundle-master_demote_0" -> "redis_demote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_post_notify_demoted_0" [ style = bold] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_start_0" [ style = bold] ++"redis-bundle-master_demoted_0" -> "redis-bundle-master_stop_0" [ style = bold] ++"redis-bundle-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_demoted_0" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] ++"redis-bundle-master_post_notify_demoted_0" -> "redis_post_notify_demoted_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_post_notify_demoted_0" -> "redis_post_notify_demoted_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_promoted_0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" -> "redis_post_notify_promoted_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_post_notify_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_running_0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis-bundle-master_post_notify_running_0" -> "redis_post_notify_running_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_post_notify_running_0" -> "redis_post_notify_running_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_post_notify_running_0" -> "redis_post_notify_running_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_post_notify_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_post_notify_stopped_0" -> "redis-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] ++"redis-bundle-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_post_notify_stopped_0" -> "redis_post_notify_stonith_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_demote_0" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] ++"redis-bundle-master_pre_notify_demote_0" -> "redis_pre_notify_demote_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_pre_notify_demote_0" -> "redis_pre_notify_demote_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_pre_notify_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_promote_0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" -> "redis_pre_notify_promote_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_pre_notify_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_start_0" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold] ++"redis-bundle-master_pre_notify_start_0" -> "redis_pre_notify_start_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_pre_notify_start_0" -> "redis_pre_notify_start_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_pre_notify_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_pre_notify_stop_0" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] ++"redis-bundle-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_pre_notify_stop_0" -> "redis_pre_notify_stop_0 redis-bundle-2" [ style = bold] ++"redis-bundle-master_pre_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_promote_0" -> "redis_promote_0 redis-bundle-1" [ style = bold] ++"redis-bundle-master_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_promoted_0" -> "redis-bundle-master_post_notify_promoted_0" [ style = bold] ++"redis-bundle-master_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_running_0" -> "redis-bundle-master_post_notify_running_0" [ style = bold] ++"redis-bundle-master_running_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle-master_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_start_0" -> "redis-bundle-master_running_0" [ style = bold] ++"redis-bundle-master_start_0" -> "redis_start_0 redis-bundle-0" [ style = dashed] ++"redis-bundle-master_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_stop_0" -> "redis-bundle-master_stopped_0" [ style = bold] ++"redis-bundle-master_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"redis-bundle-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-master_stopped_0" -> "redis-bundle-master_post_notify_stopped_0" [ style = bold] ++"redis-bundle-master_stopped_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle-master_stopped_0" -> "redis-bundle-master_start_0" [ style = bold] ++"redis-bundle-master_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_demote_0" -> "redis-bundle-master_demote_0" [ style = bold] ++"redis-bundle_demote_0" -> "redis-bundle_demoted_0" [ style = bold] ++"redis-bundle_demote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_demoted_0" -> "redis-bundle_promote_0" [ style = bold] ++"redis-bundle_demoted_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_demoted_0" -> "redis-bundle_stop_0" [ style = bold] ++"redis-bundle_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_promote_0" -> "redis-bundle-master_promote_0" [ style = bold] ++"redis-bundle_promote_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_running_0" -> "redis-bundle_promote_0" [ style = bold] ++"redis-bundle_running_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_start_0" -> "redis-bundle-master_start_0" [ style = bold] ++"redis-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_stop_0" -> "redis-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"redis-bundle_stop_0" -> "redis-bundle-master_stop_0" [ style = bold] ++"redis-bundle_stop_0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"redis-bundle_stop_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle_stopped_0" -> "redis-bundle_promote_0" [ style = bold] ++"redis-bundle_stopped_0" -> "redis-bundle_start_0" [ style = bold] ++"redis-bundle_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"redis_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] ++"redis_confirmed-post_notify_stonith_0" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] ++"redis_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] ++"redis_demote_0 redis-bundle-0" -> "redis-bundle-master_demoted_0" [ style = bold] ++"redis_demote_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis_demote_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis_demote_0 redis-bundle-0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"redis_demote_0 redis-bundle-0" [ style=bold color="green" fontcolor="orange"] ++"redis_monitor_20000 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_monitor_45000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_monitor_60000 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_post_notify_demoted_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] ++"redis_post_notify_demoted_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_demoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_demoted_0" [ style = bold] ++"redis_post_notify_demoted_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_promoted_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis_post_notify_promoted_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_promoted_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis_post_notify_promoted_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_promoted_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_promoted_0" [ style = bold] ++"redis_post_notify_promoted_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_running_0 redis-bundle-0" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis_post_notify_running_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_running_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis_post_notify_running_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_running_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_running_0" [ style = bold] ++"redis_post_notify_running_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_stonith_0 redis-bundle-1" -> "redis-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] ++"redis_post_notify_stonith_0 redis-bundle-1" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] ++"redis_post_notify_stonith_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_stonith_0 redis-bundle-2" -> "redis-bundle-master_confirmed-post_notify_stopped_0" [ style = bold] ++"redis_post_notify_stonith_0 redis-bundle-2" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] ++"redis_post_notify_stonith_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis_post_notify_stonith_0" -> "redis_confirmed-post_notify_stonith_0" [ style = bold] ++"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 redis-bundle-1" [ style = bold] ++"redis_post_notify_stonith_0" -> "redis_post_notify_stonith_0 redis-bundle-2" [ style = bold] ++"redis_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] ++"redis_pre_notify_demote_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] ++"redis_pre_notify_demote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_demote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_demote_0" [ style = bold] ++"redis_pre_notify_demote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_promote_0 redis-bundle-0" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis_pre_notify_promote_0 redis-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_promote_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis_pre_notify_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_promote_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_promote_0" [ style = bold] ++"redis_pre_notify_promote_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_start_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold] ++"redis_pre_notify_start_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_start_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_start_0" [ style = bold] ++"redis_pre_notify_start_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_stop_0 redis-bundle-1" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] ++"redis_pre_notify_stop_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_pre_notify_stop_0 redis-bundle-2" -> "redis-bundle-master_confirmed-pre_notify_stop_0" [ style = bold] ++"redis_pre_notify_stop_0 redis-bundle-2" [ style=bold color="green" fontcolor="black"] ++"redis_promote_0 redis-bundle-1" -> "redis-bundle-master_promoted_0" [ style = bold] ++"redis_promote_0 redis-bundle-1" -> "redis_monitor_20000 redis-bundle-1" [ style = bold] ++"redis_promote_0 redis-bundle-1" [ style=bold color="green" fontcolor="black"] ++"redis_start_0 redis-bundle-0" -> "redis-bundle-master_running_0" [ style = dashed] ++"redis_start_0 redis-bundle-0" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] ++"redis_start_0 redis-bundle-0" -> "redis_monitor_60000 redis-bundle-0" [ style = dashed] ++"redis_start_0 redis-bundle-0" [ style=dashed color="red" fontcolor="black"] ++"redis_stop_0 redis-bundle-0" -> "all_stopped" [ style = bold] ++"redis_stop_0 redis-bundle-0" -> "redis-bundle-master_stopped_0" [ style = bold] ++"redis_stop_0 redis-bundle-0" -> "redis_start_0 redis-bundle-0" [ style = dashed] ++"redis_stop_0 redis-bundle-0" [ style=bold color="green" fontcolor="orange"] ++"stonith 'off' galera-bundle-0" -> "galera-bundle-master_stop_0" [ style = bold] ++"stonith 'off' galera-bundle-0" -> "galera_demote_0 galera-bundle-0" [ style = bold] ++"stonith 'off' galera-bundle-0" -> "galera_stop_0 galera-bundle-0" [ style = bold] ++"stonith 'off' galera-bundle-0" -> "stonith_complete" [ style = bold] ++"stonith 'off' galera-bundle-0" [ style=bold color="green" fontcolor="orange"] ++"stonith 'off' rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stop_0" [ style = bold] ++"stonith 'off' rabbitmq-bundle-0" -> "rabbitmq_post_notify_stonith_0" [ style = bold] ++"stonith 'off' rabbitmq-bundle-0" -> "rabbitmq_stop_0 rabbitmq-bundle-0" [ style = bold] ++"stonith 'off' rabbitmq-bundle-0" -> "stonith_complete" [ style = bold] ++"stonith 'off' rabbitmq-bundle-0" [ style=bold color="green" fontcolor="orange"] ++"stonith 'off' redis-bundle-0" -> "redis-bundle-master_stop_0" [ style = bold] ++"stonith 'off' redis-bundle-0" -> "redis_demote_0 redis-bundle-0" [ style = bold] ++"stonith 'off' redis-bundle-0" -> "redis_post_notify_stonith_0" [ style = bold] ++"stonith 'off' redis-bundle-0" -> "redis_stop_0 redis-bundle-0" [ style = bold] ++"stonith 'off' redis-bundle-0" -> "stonith_complete" [ style = bold] ++"stonith 'off' redis-bundle-0" [ style=bold color="green" fontcolor="orange"] ++"stonith 'reboot' controller-0" -> "galera-bundle-0_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "galera-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "haproxy-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "ip-10.0.0.109_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "ip-172.17.4.11_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "ip-192.168.24.7_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "rabbitmq-bundle-0_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "redis-bundle-0_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "redis-bundle-docker-0_stop_0 controller-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "stonith 'off' galera-bundle-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "stonith 'off' rabbitmq-bundle-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "stonith 'off' redis-bundle-0" [ style = bold] ++"stonith 'reboot' controller-0" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' controller-0" [ style=bold color="green" fontcolor="black"] ++"stonith-fence_ipmilan-5254000dcb3f_monitor_60000 controller-2" [ style=bold color="green" fontcolor="black"] ++"stonith-fence_ipmilan-5254000dcb3f_start_0 controller-2" -> "stonith-fence_ipmilan-5254000dcb3f_monitor_60000 controller-2" [ style = bold] ++"stonith-fence_ipmilan-5254000dcb3f_start_0 controller-2" [ style=bold color="green" fontcolor="black"] ++"stonith-fence_ipmilan-5254000dcb3f_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"stonith-fence_ipmilan-5254000dcb3f_stop_0 controller-0" -> "stonith-fence_ipmilan-5254000dcb3f_start_0 controller-2" [ style = bold] ++"stonith-fence_ipmilan-5254000dcb3f_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"stonith-fence_ipmilan-5254003e8e97_monitor_60000 controller-1" [ style=bold color="green" fontcolor="black"] ++"stonith-fence_ipmilan-5254003e8e97_start_0 controller-1" -> "stonith-fence_ipmilan-5254003e8e97_monitor_60000 controller-1" [ style = bold] ++"stonith-fence_ipmilan-5254003e8e97_start_0 controller-1" [ style=bold color="green" fontcolor="black"] ++"stonith-fence_ipmilan-5254003e8e97_stop_0 controller-0" -> "all_stopped" [ style = bold] ++"stonith-fence_ipmilan-5254003e8e97_stop_0 controller-0" -> "stonith-fence_ipmilan-5254003e8e97_start_0 controller-1" [ style = bold] ++"stonith-fence_ipmilan-5254003e8e97_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "galera-bundle-0_start_0 controller-2" [ style = dashed] ++"stonith_complete" -> "galera_start_0 galera-bundle-0" [ style = dashed] ++"stonith_complete" -> "ip-10.0.0.109_start_0 controller-1" [ style = bold] ++"stonith_complete" -> "ip-172.17.4.11_start_0 controller-1" [ style = bold] ++"stonith_complete" -> "ip-192.168.24.7_start_0 controller-2" [ style = bold] ++"stonith_complete" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed] ++"stonith_complete" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] ++"stonith_complete" -> "redis-bundle-0_start_0 controller-1" [ style = dashed] ++"stonith_complete" -> "redis_promote_0 redis-bundle-1" [ style = bold] ++"stonith_complete" -> "redis_start_0 redis-bundle-0" [ style = dashed] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] ++} +diff --git a/pengine/test10/bundle-order-fencing.exp b/pengine/test10/bundle-order-fencing.exp +new file mode 100644 +index 0000000..d072d85 +--- /dev/null ++++ b/pengine/test10/bundle-order-fencing.exp +@@ -0,0 +1,1982 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bundle-order-fencing.scores b/pengine/test10/bundle-order-fencing.scores +new file mode 100644 +index 0000000..9037624 +--- /dev/null ++++ b/pengine/test10/bundle-order-fencing.scores +@@ -0,0 +1,1148 @@ ++Allocation scores: ++Using the original execution date of: 2017-09-12 10:51:59Z ++clone_color: galera-bundle-master allocation score on controller-0: -INFINITY ++clone_color: galera-bundle-master allocation score on controller-1: -INFINITY ++clone_color: galera-bundle-master allocation score on controller-2: -INFINITY ++clone_color: galera-bundle-master allocation score on galera-bundle-0: 0 ++clone_color: galera-bundle-master allocation score on galera-bundle-1: 0 ++clone_color: galera-bundle-master allocation score on galera-bundle-2: 0 ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: galera-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: galera:0 allocation score on controller-0: -INFINITY ++clone_color: galera:0 allocation score on controller-1: -INFINITY ++clone_color: galera:0 allocation score on controller-2: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-0: INFINITY ++clone_color: galera:0 allocation score on galera-bundle-1: -INFINITY ++clone_color: galera:0 allocation score on galera-bundle-2: -INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: galera:1 allocation score on controller-0: -INFINITY ++clone_color: galera:1 allocation score on controller-1: -INFINITY ++clone_color: galera:1 allocation score on controller-2: -INFINITY ++clone_color: galera:1 allocation score on galera-bundle-0: -INFINITY ++clone_color: galera:1 allocation score on galera-bundle-1: INFINITY ++clone_color: galera:1 allocation score on galera-bundle-2: -INFINITY ++clone_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: galera:2 allocation score on controller-0: -INFINITY ++clone_color: galera:2 allocation score on controller-1: -INFINITY ++clone_color: galera:2 allocation score on controller-2: -INFINITY ++clone_color: galera:2 allocation score on galera-bundle-0: -INFINITY ++clone_color: galera:2 allocation score on galera-bundle-1: -INFINITY ++clone_color: galera:2 allocation score on galera-bundle-2: INFINITY ++clone_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on controller-0: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on controller-1: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on controller-2: -INFINITY ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0 ++clone_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0 ++clone_color: rabbitmq:0 allocation score on controller-0: -INFINITY ++clone_color: rabbitmq:0 allocation score on controller-1: -INFINITY ++clone_color: rabbitmq:0 allocation score on controller-2: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: rabbitmq:1 allocation score on controller-0: -INFINITY ++clone_color: rabbitmq:1 allocation score on controller-1: -INFINITY ++clone_color: rabbitmq:1 allocation score on controller-2: -INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++clone_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: rabbitmq:2 allocation score on controller-0: -INFINITY ++clone_color: rabbitmq:2 allocation score on controller-1: -INFINITY ++clone_color: rabbitmq:2 allocation score on controller-2: -INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++clone_color: redis-bundle-master allocation score on controller-0: -INFINITY ++clone_color: redis-bundle-master allocation score on controller-1: -INFINITY ++clone_color: redis-bundle-master allocation score on controller-2: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY ++clone_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: redis-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: redis-bundle-master allocation score on redis-bundle-0: 0 ++clone_color: redis-bundle-master allocation score on redis-bundle-1: 0 ++clone_color: redis-bundle-master allocation score on redis-bundle-2: 0 ++clone_color: redis:0 allocation score on controller-0: -INFINITY ++clone_color: redis:0 allocation score on controller-1: -INFINITY ++clone_color: redis:0 allocation score on controller-2: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-1: -INFINITY ++clone_color: redis:0 allocation score on galera-bundle-2: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-0: INFINITY ++clone_color: redis:0 allocation score on redis-bundle-1: -INFINITY ++clone_color: redis:0 allocation score on redis-bundle-2: -INFINITY ++clone_color: redis:1 allocation score on controller-0: -INFINITY ++clone_color: redis:1 allocation score on controller-1: -INFINITY ++clone_color: redis:1 allocation score on controller-2: -INFINITY ++clone_color: redis:1 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:1 allocation score on galera-bundle-1: -INFINITY ++clone_color: redis:1 allocation score on galera-bundle-2: -INFINITY ++clone_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: redis:1 allocation score on redis-bundle-0: -INFINITY ++clone_color: redis:1 allocation score on redis-bundle-1: INFINITY ++clone_color: redis:1 allocation score on redis-bundle-2: -INFINITY ++clone_color: redis:2 allocation score on controller-0: -INFINITY ++clone_color: redis:2 allocation score on controller-1: -INFINITY ++clone_color: redis:2 allocation score on controller-2: -INFINITY ++clone_color: redis:2 allocation score on galera-bundle-0: -INFINITY ++clone_color: redis:2 allocation score on galera-bundle-1: -INFINITY ++clone_color: redis:2 allocation score on galera-bundle-2: -INFINITY ++clone_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY ++clone_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY ++clone_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY ++clone_color: redis:2 allocation score on redis-bundle-0: -INFINITY ++clone_color: redis:2 allocation score on redis-bundle-1: -INFINITY ++clone_color: redis:2 allocation score on redis-bundle-2: INFINITY ++container_color: galera-bundle allocation score on controller-0: 0 ++container_color: galera-bundle allocation score on controller-1: 0 ++container_color: galera-bundle allocation score on controller-2: 0 ++container_color: galera-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-0 allocation score on controller-0: INFINITY ++container_color: galera-bundle-0 allocation score on controller-1: 0 ++container_color: galera-bundle-0 allocation score on controller-2: 0 ++container_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-1 allocation score on controller-0: 0 ++container_color: galera-bundle-1 allocation score on controller-1: INFINITY ++container_color: galera-bundle-1 allocation score on controller-2: 0 ++container_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-2 allocation score on controller-0: 0 ++container_color: galera-bundle-2 allocation score on controller-1: 0 ++container_color: galera-bundle-2 allocation score on controller-2: INFINITY ++container_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on controller-0: INFINITY ++container_color: galera-bundle-docker-0 allocation score on controller-1: 0 ++container_color: galera-bundle-docker-0 allocation score on controller-2: 0 ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on controller-0: 0 ++container_color: galera-bundle-docker-1 allocation score on controller-1: INFINITY ++container_color: galera-bundle-docker-1 allocation score on controller-2: 0 ++container_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on controller-0: 0 ++container_color: galera-bundle-docker-2 allocation score on controller-1: 0 ++container_color: galera-bundle-docker-2 allocation score on controller-2: INFINITY ++container_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera-bundle-master allocation score on controller-0: 0 ++container_color: galera-bundle-master allocation score on controller-1: 0 ++container_color: galera-bundle-master allocation score on controller-2: 0 ++container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-1: -INFINITY ++container_color: galera-bundle-master allocation score on galera-bundle-2: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera:0 allocation score on controller-0: 0 ++container_color: galera:0 allocation score on controller-1: 0 ++container_color: galera:0 allocation score on controller-2: 0 ++container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera:1 allocation score on controller-0: 0 ++container_color: galera:1 allocation score on controller-1: 0 ++container_color: galera:1 allocation score on controller-2: 0 ++container_color: galera:1 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:1 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: galera:2 allocation score on controller-0: 0 ++container_color: galera:2 allocation score on controller-1: 0 ++container_color: galera:2 allocation score on controller-2: 0 ++container_color: galera:2 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:2 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on controller-0: 0 ++container_color: haproxy-bundle allocation score on controller-0: 0 ++container_color: haproxy-bundle allocation score on controller-0: 0 ++container_color: haproxy-bundle allocation score on controller-0: 0 ++container_color: haproxy-bundle allocation score on controller-0: 0 ++container_color: haproxy-bundle allocation score on controller-0: 0 ++container_color: haproxy-bundle allocation score on controller-0: 0 ++container_color: haproxy-bundle allocation score on controller-1: 0 ++container_color: haproxy-bundle allocation score on controller-1: 0 ++container_color: haproxy-bundle allocation score on controller-1: 0 ++container_color: haproxy-bundle allocation score on controller-1: 0 ++container_color: haproxy-bundle allocation score on controller-1: 0 ++container_color: haproxy-bundle allocation score on controller-1: 0 ++container_color: haproxy-bundle allocation score on controller-1: 0 ++container_color: haproxy-bundle allocation score on controller-2: 0 ++container_color: haproxy-bundle allocation score on controller-2: 0 ++container_color: haproxy-bundle allocation score on controller-2: 0 ++container_color: haproxy-bundle allocation score on controller-2: 0 ++container_color: haproxy-bundle allocation score on controller-2: 0 ++container_color: haproxy-bundle allocation score on controller-2: 0 ++container_color: haproxy-bundle allocation score on controller-2: 0 ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-0: INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-1: 0 ++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on controller-2: 0 ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-0: 0 ++container_color: haproxy-bundle-docker-1 allocation score on controller-1: 0 ++container_color: haproxy-bundle-docker-1 allocation score on controller-1: 0 ++container_color: haproxy-bundle-docker-1 allocation score on controller-1: 0 ++container_color: haproxy-bundle-docker-1 allocation score on controller-1: 0 ++container_color: haproxy-bundle-docker-1 allocation score on controller-1: 0 ++container_color: haproxy-bundle-docker-1 allocation score on controller-1: 0 ++container_color: haproxy-bundle-docker-1 allocation score on controller-1: 0 ++container_color: haproxy-bundle-docker-1 allocation score on controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on controller-2: INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-0: 0 ++container_color: haproxy-bundle-docker-2 allocation score on controller-1: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-1: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-1: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-1: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-1: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-1: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-1: INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on controller-2: 0 ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: rabbitmq-bundle allocation score on controller-0: 0 ++container_color: rabbitmq-bundle allocation score on controller-1: 0 ++container_color: rabbitmq-bundle allocation score on controller-2: 0 ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on controller-0: INFINITY ++container_color: rabbitmq-bundle-0 allocation score on controller-1: 0 ++container_color: rabbitmq-bundle-0 allocation score on controller-2: 0 ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on controller-0: 0 ++container_color: rabbitmq-bundle-1 allocation score on controller-1: INFINITY ++container_color: rabbitmq-bundle-1 allocation score on controller-2: 0 ++container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on controller-0: 0 ++container_color: rabbitmq-bundle-2 allocation score on controller-1: 0 ++container_color: rabbitmq-bundle-2 allocation score on controller-2: INFINITY ++container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-clone allocation score on controller-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on controller-1: 0 ++container_color: rabbitmq-bundle-clone allocation score on controller-2: 0 ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-1: 0 ++container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-2: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on controller-0: INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on controller-1: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on controller-2: 0 ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on controller-0: 0 ++container_color: rabbitmq-bundle-docker-1 allocation score on controller-1: INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on controller-2: 0 ++container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on controller-0: 0 ++container_color: rabbitmq-bundle-docker-2 allocation score on controller-1: 0 ++container_color: rabbitmq-bundle-docker-2 allocation score on controller-2: INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq:0 allocation score on controller-0: 0 ++container_color: rabbitmq:0 allocation score on controller-1: 0 ++container_color: rabbitmq:0 allocation score on controller-2: 0 ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: 0 ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: 0 ++container_color: rabbitmq:1 allocation score on controller-0: 0 ++container_color: rabbitmq:1 allocation score on controller-1: 0 ++container_color: rabbitmq:1 allocation score on controller-2: 0 ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: 0 ++container_color: rabbitmq:2 allocation score on controller-0: 0 ++container_color: rabbitmq:2 allocation score on controller-1: 0 ++container_color: rabbitmq:2 allocation score on controller-2: 0 ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: 0 ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++container_color: redis-bundle allocation score on controller-0: 0 ++container_color: redis-bundle allocation score on controller-1: 0 ++container_color: redis-bundle allocation score on controller-2: 0 ++container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-0 allocation score on controller-0: INFINITY ++container_color: redis-bundle-0 allocation score on controller-1: 0 ++container_color: redis-bundle-0 allocation score on controller-2: 0 ++container_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-1 allocation score on controller-0: 0 ++container_color: redis-bundle-1 allocation score on controller-1: INFINITY ++container_color: redis-bundle-1 allocation score on controller-2: 0 ++container_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-2 allocation score on controller-0: 0 ++container_color: redis-bundle-2 allocation score on controller-1: 0 ++container_color: redis-bundle-2 allocation score on controller-2: INFINITY ++container_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on controller-0: INFINITY ++container_color: redis-bundle-docker-0 allocation score on controller-1: 0 ++container_color: redis-bundle-docker-0 allocation score on controller-2: 0 ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on controller-0: 0 ++container_color: redis-bundle-docker-1 allocation score on controller-1: INFINITY ++container_color: redis-bundle-docker-1 allocation score on controller-2: 0 ++container_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on controller-0: 0 ++container_color: redis-bundle-docker-2 allocation score on controller-1: 0 ++container_color: redis-bundle-docker-2 allocation score on controller-2: INFINITY ++container_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++container_color: redis-bundle-master allocation score on controller-0: 0 ++container_color: redis-bundle-master allocation score on controller-1: 0 ++container_color: redis-bundle-master allocation score on controller-2: 0 ++container_color: redis-bundle-master allocation score on galera-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-1: -INFINITY ++container_color: redis-bundle-master allocation score on galera-bundle-2: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-1: -INFINITY ++container_color: redis-bundle-master allocation score on redis-bundle-2: -INFINITY ++container_color: redis:0 allocation score on controller-0: 0 ++container_color: redis:0 allocation score on controller-1: 0 ++container_color: redis:0 allocation score on controller-2: 0 ++container_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:0 allocation score on galera-bundle-1: -INFINITY ++container_color: redis:0 allocation score on galera-bundle-2: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-2: -INFINITY ++container_color: redis:1 allocation score on controller-0: 0 ++container_color: redis:1 allocation score on controller-1: 0 ++container_color: redis:1 allocation score on controller-2: 0 ++container_color: redis:1 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:1 allocation score on galera-bundle-1: -INFINITY ++container_color: redis:1 allocation score on galera-bundle-2: -INFINITY ++container_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-2: -INFINITY ++container_color: redis:2 allocation score on controller-0: 0 ++container_color: redis:2 allocation score on controller-1: 0 ++container_color: redis:2 allocation score on controller-2: 0 ++container_color: redis:2 allocation score on galera-bundle-0: -INFINITY ++container_color: redis:2 allocation score on galera-bundle-1: -INFINITY ++container_color: redis:2 allocation score on galera-bundle-2: -INFINITY ++container_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-2: -INFINITY ++galera:0 promotion score on galera-bundle-0: -1 ++galera:1 promotion score on galera-bundle-1: 100 ++galera:2 promotion score on galera-bundle-2: 100 ++native_color: galera-bundle-0 allocation score on controller-0: INFINITY ++native_color: galera-bundle-0 allocation score on controller-1: 0 ++native_color: galera-bundle-0 allocation score on controller-2: 0 ++native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-0 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-1 allocation score on controller-0: 0 ++native_color: galera-bundle-1 allocation score on controller-1: INFINITY ++native_color: galera-bundle-1 allocation score on controller-2: 0 ++native_color: galera-bundle-1 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-1 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-1 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-2 allocation score on controller-0: 0 ++native_color: galera-bundle-2 allocation score on controller-1: 0 ++native_color: galera-bundle-2 allocation score on controller-2: INFINITY ++native_color: galera-bundle-2 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-2 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-2 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on controller-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on controller-1: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on controller-2: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on controller-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on controller-1: INFINITY ++native_color: galera-bundle-docker-1 allocation score on controller-2: 0 ++native_color: galera-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on controller-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on controller-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on controller-2: INFINITY ++native_color: galera-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera:0 allocation score on controller-0: -INFINITY ++native_color: galera:0 allocation score on controller-1: -INFINITY ++native_color: galera:0 allocation score on controller-2: -INFINITY ++native_color: galera:0 allocation score on galera-bundle-0: INFINITY ++native_color: galera:0 allocation score on galera-bundle-1: -INFINITY ++native_color: galera:0 allocation score on galera-bundle-2: -INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera:1 allocation score on controller-0: -INFINITY ++native_color: galera:1 allocation score on controller-1: -INFINITY ++native_color: galera:1 allocation score on controller-2: -INFINITY ++native_color: galera:1 allocation score on galera-bundle-0: -INFINITY ++native_color: galera:1 allocation score on galera-bundle-1: INFINITY ++native_color: galera:1 allocation score on galera-bundle-2: -INFINITY ++native_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: galera:2 allocation score on controller-0: -INFINITY ++native_color: galera:2 allocation score on controller-1: -INFINITY ++native_color: galera:2 allocation score on controller-2: -INFINITY ++native_color: galera:2 allocation score on galera-bundle-0: -INFINITY ++native_color: galera:2 allocation score on galera-bundle-1: -INFINITY ++native_color: galera:2 allocation score on galera-bundle-2: INFINITY ++native_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on controller-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on controller-1: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on controller-2: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on controller-0: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on controller-1: 0 ++native_color: haproxy-bundle-docker-1 allocation score on controller-2: INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on controller-0: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on controller-1: INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on controller-2: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++native_color: haproxy-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-10.0.0.109 allocation score on controller-0: -INFINITY ++native_color: ip-10.0.0.109 allocation score on controller-1: 0 ++native_color: ip-10.0.0.109 allocation score on controller-2: 0 ++native_color: ip-10.0.0.109 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-10.0.0.109 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-10.0.0.109 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-10.0.0.109 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-10.0.0.109 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-10.0.0.109 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-10.0.0.109 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-10.0.0.109 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-10.0.0.109 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-172.17.1.14 allocation score on controller-0: -INFINITY ++native_color: ip-172.17.1.14 allocation score on controller-1: 0 ++native_color: ip-172.17.1.14 allocation score on controller-2: INFINITY ++native_color: ip-172.17.1.14 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-172.17.1.14 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-172.17.1.14 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-172.17.1.14 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-172.17.1.14 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-172.17.1.14 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-172.17.1.14 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-172.17.1.14 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-172.17.1.14 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-172.17.1.19 allocation score on controller-0: -INFINITY ++native_color: ip-172.17.1.19 allocation score on controller-1: 0 ++native_color: ip-172.17.1.19 allocation score on controller-2: INFINITY ++native_color: ip-172.17.1.19 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-172.17.1.19 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-172.17.1.19 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-172.17.1.19 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-172.17.1.19 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-172.17.1.19 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-172.17.1.19 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-172.17.1.19 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-172.17.1.19 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-172.17.3.19 allocation score on controller-0: -INFINITY ++native_color: ip-172.17.3.19 allocation score on controller-1: 0 ++native_color: ip-172.17.3.19 allocation score on controller-2: INFINITY ++native_color: ip-172.17.3.19 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-172.17.3.19 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-172.17.3.19 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-172.17.3.19 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-172.17.3.19 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-172.17.3.19 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-172.17.3.19 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-172.17.3.19 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-172.17.3.19 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-172.17.4.11 allocation score on controller-0: -INFINITY ++native_color: ip-172.17.4.11 allocation score on controller-1: 0 ++native_color: ip-172.17.4.11 allocation score on controller-2: 0 ++native_color: ip-172.17.4.11 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-172.17.4.11 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-172.17.4.11 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-172.17.4.11 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-172.17.4.11 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-172.17.4.11 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-172.17.4.11 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-172.17.4.11 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-172.17.4.11 allocation score on redis-bundle-2: -INFINITY ++native_color: ip-192.168.24.7 allocation score on controller-0: -INFINITY ++native_color: ip-192.168.24.7 allocation score on controller-1: 0 ++native_color: ip-192.168.24.7 allocation score on controller-2: 0 ++native_color: ip-192.168.24.7 allocation score on galera-bundle-0: -INFINITY ++native_color: ip-192.168.24.7 allocation score on galera-bundle-1: -INFINITY ++native_color: ip-192.168.24.7 allocation score on galera-bundle-2: -INFINITY ++native_color: ip-192.168.24.7 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: ip-192.168.24.7 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: ip-192.168.24.7 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: ip-192.168.24.7 allocation score on redis-bundle-0: -INFINITY ++native_color: ip-192.168.24.7 allocation score on redis-bundle-1: -INFINITY ++native_color: ip-192.168.24.7 allocation score on redis-bundle-2: -INFINITY ++native_color: openstack-cinder-volume allocation score on controller-0: 0 ++native_color: openstack-cinder-volume allocation score on controller-1: 0 ++native_color: openstack-cinder-volume allocation score on controller-2: INFINITY ++native_color: openstack-cinder-volume allocation score on galera-bundle-0: -INFINITY ++native_color: openstack-cinder-volume allocation score on galera-bundle-1: -INFINITY ++native_color: openstack-cinder-volume allocation score on galera-bundle-2: -INFINITY ++native_color: openstack-cinder-volume allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: openstack-cinder-volume allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: openstack-cinder-volume allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: openstack-cinder-volume allocation score on redis-bundle-0: -INFINITY ++native_color: openstack-cinder-volume allocation score on redis-bundle-1: -INFINITY ++native_color: openstack-cinder-volume allocation score on redis-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on controller-0: INFINITY ++native_color: rabbitmq-bundle-0 allocation score on controller-1: 0 ++native_color: rabbitmq-bundle-0 allocation score on controller-2: 0 ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on controller-0: 0 ++native_color: rabbitmq-bundle-1 allocation score on controller-1: INFINITY ++native_color: rabbitmq-bundle-1 allocation score on controller-2: 0 ++native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on controller-0: 0 ++native_color: rabbitmq-bundle-2 allocation score on controller-1: 0 ++native_color: rabbitmq-bundle-2 allocation score on controller-2: INFINITY ++native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on controller-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on controller-1: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on controller-2: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on controller-0: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on controller-1: INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on controller-2: 0 ++native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on controller-0: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on controller-1: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on controller-2: INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq:0 allocation score on controller-0: -INFINITY ++native_color: rabbitmq:0 allocation score on controller-1: -INFINITY ++native_color: rabbitmq:0 allocation score on controller-2: -INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq:1 allocation score on controller-0: -INFINITY ++native_color: rabbitmq:1 allocation score on controller-1: -INFINITY ++native_color: rabbitmq:1 allocation score on controller-2: -INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: INFINITY ++native_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: rabbitmq:2 allocation score on controller-0: -INFINITY ++native_color: rabbitmq:2 allocation score on controller-1: -INFINITY ++native_color: rabbitmq:2 allocation score on controller-2: -INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: INFINITY ++native_color: redis-bundle-0 allocation score on controller-0: INFINITY ++native_color: redis-bundle-0 allocation score on controller-1: 0 ++native_color: redis-bundle-0 allocation score on controller-2: 0 ++native_color: redis-bundle-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-0 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-0 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-1 allocation score on controller-0: 0 ++native_color: redis-bundle-1 allocation score on controller-1: INFINITY ++native_color: redis-bundle-1 allocation score on controller-2: 0 ++native_color: redis-bundle-1 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-1 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-1 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-1 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-1 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-1 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-2 allocation score on controller-0: 0 ++native_color: redis-bundle-2 allocation score on controller-1: 0 ++native_color: redis-bundle-2 allocation score on controller-2: INFINITY ++native_color: redis-bundle-2 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-2 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-2 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-2 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-2 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-2 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on controller-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on controller-1: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on controller-2: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-docker-0 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on controller-0: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on controller-1: INFINITY ++native_color: redis-bundle-docker-1 allocation score on controller-2: 0 ++native_color: redis-bundle-docker-1 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-docker-1 allocation score on redis-bundle-2: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on controller-0: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on controller-1: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on controller-2: INFINITY ++native_color: redis-bundle-docker-2 allocation score on galera-bundle-0: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on galera-bundle-1: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on galera-bundle-2: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on redis-bundle-0: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on redis-bundle-1: -INFINITY ++native_color: redis-bundle-docker-2 allocation score on redis-bundle-2: -INFINITY ++native_color: redis:0 allocation score on controller-0: -INFINITY ++native_color: redis:0 allocation score on controller-1: -INFINITY ++native_color: redis:0 allocation score on controller-2: -INFINITY ++native_color: redis:0 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:0 allocation score on galera-bundle-1: -INFINITY ++native_color: redis:0 allocation score on galera-bundle-2: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-0: INFINITY ++native_color: redis:0 allocation score on redis-bundle-1: -INFINITY ++native_color: redis:0 allocation score on redis-bundle-2: -INFINITY ++native_color: redis:1 allocation score on controller-0: -INFINITY ++native_color: redis:1 allocation score on controller-1: -INFINITY ++native_color: redis:1 allocation score on controller-2: -INFINITY ++native_color: redis:1 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:1 allocation score on galera-bundle-1: -INFINITY ++native_color: redis:1 allocation score on galera-bundle-2: -INFINITY ++native_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis:1 allocation score on redis-bundle-0: -INFINITY ++native_color: redis:1 allocation score on redis-bundle-1: INFINITY ++native_color: redis:1 allocation score on redis-bundle-2: -INFINITY ++native_color: redis:2 allocation score on controller-0: -INFINITY ++native_color: redis:2 allocation score on controller-1: -INFINITY ++native_color: redis:2 allocation score on controller-2: -INFINITY ++native_color: redis:2 allocation score on galera-bundle-0: -INFINITY ++native_color: redis:2 allocation score on galera-bundle-1: -INFINITY ++native_color: redis:2 allocation score on galera-bundle-2: -INFINITY ++native_color: redis:2 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: redis:2 allocation score on redis-bundle-0: -INFINITY ++native_color: redis:2 allocation score on redis-bundle-1: -INFINITY ++native_color: redis:2 allocation score on redis-bundle-2: INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on controller-0: INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on controller-1: -INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on controller-2: 0 ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on galera-bundle-0: -INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on galera-bundle-1: -INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on galera-bundle-2: -INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on redis-bundle-0: -INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on redis-bundle-1: -INFINITY ++native_color: stonith-fence_ipmilan-5254000dcb3f allocation score on redis-bundle-2: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on controller-0: INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on controller-1: 0 ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on controller-2: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on galera-bundle-0: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on galera-bundle-1: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on galera-bundle-2: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on redis-bundle-0: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on redis-bundle-1: -INFINITY ++native_color: stonith-fence_ipmilan-5254003e8e97 allocation score on redis-bundle-2: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on controller-0: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on controller-1: 0 ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on controller-2: INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on galera-bundle-0: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on galera-bundle-1: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on galera-bundle-2: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on rabbitmq-bundle-0: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on rabbitmq-bundle-1: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on rabbitmq-bundle-2: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on redis-bundle-0: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on redis-bundle-1: -INFINITY ++native_color: stonith-fence_ipmilan-525400efba5c allocation score on redis-bundle-2: -INFINITY ++redis:0 promotion score on redis-bundle-0: -1 ++redis:1 promotion score on redis-bundle-1: 1 ++redis:2 promotion score on redis-bundle-2: 1 +diff --git a/pengine/test10/bundle-order-fencing.summary b/pengine/test10/bundle-order-fencing.summary +new file mode 100644 +index 0000000..e2128cb +--- /dev/null ++++ b/pengine/test10/bundle-order-fencing.summary +@@ -0,0 +1,222 @@ ++Using the original execution date of: 2017-09-12 10:51:59Z ++ ++Current cluster status: ++Node controller-0 (1): UNCLEAN (offline) ++Online: [ controller-1 controller-2 ] ++Containers: [ galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] ++ ++ Docker container set: rabbitmq-bundle [192.168.24.1:8787/rhosp12/openstack-rabbitmq-docker:pcmklatest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): FAILED controller-0 (UNCLEAN) ++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-1 ++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-2 ++ Docker container set: galera-bundle [192.168.24.1:8787/rhosp12/openstack-mariadb-docker:pcmklatest] ++ galera-bundle-0 (ocf::heartbeat:galera): FAILED Master controller-0 (UNCLEAN) ++ galera-bundle-1 (ocf::heartbeat:galera): Master controller-1 ++ galera-bundle-2 (ocf::heartbeat:galera): Master controller-2 ++ Docker container set: redis-bundle [192.168.24.1:8787/rhosp12/openstack-redis-docker:pcmklatest] ++ redis-bundle-0 (ocf::heartbeat:redis): FAILED Master controller-0 (UNCLEAN) ++ redis-bundle-1 (ocf::heartbeat:redis): Slave controller-1 ++ redis-bundle-2 (ocf::heartbeat:redis): Slave controller-2 ++ ip-192.168.24.7 (ocf::heartbeat:IPaddr2): Started controller-0 (UNCLEAN) ++ ip-10.0.0.109 (ocf::heartbeat:IPaddr2): Started controller-0 (UNCLEAN) ++ ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ++ ip-172.17.1.19 (ocf::heartbeat:IPaddr2): Started controller-2 ++ ip-172.17.3.19 (ocf::heartbeat:IPaddr2): Started controller-2 ++ ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-0 (UNCLEAN) ++ Docker container set: haproxy-bundle [192.168.24.1:8787/rhosp12/openstack-haproxy-docker:pcmklatest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Started controller-0 (UNCLEAN) ++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-2 ++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-1 ++ openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-2 ++ stonith-fence_ipmilan-525400efba5c (stonith:fence_ipmilan): Started controller-2 ++ stonith-fence_ipmilan-5254003e8e97 (stonith:fence_ipmilan): Started controller-0 (UNCLEAN) ++ stonith-fence_ipmilan-5254000dcb3f (stonith:fence_ipmilan): Started controller-0 (UNCLEAN) ++ ++Transition Summary: ++ * Fence (off) redis-bundle-0 (resource: redis-bundle-docker-0) 'guest is unclean' ++ * Fence (off) rabbitmq-bundle-0 (resource: rabbitmq-bundle-docker-0) 'guest is unclean' ++ * Fence (off) galera-bundle-0 (resource: galera-bundle-docker-0) 'guest is unclean' ++ * Fence (reboot) controller-0 'peer is no longer part of the cluster' ++ * Stop rabbitmq-bundle-docker-0 ( controller-0 ) due to node availability ++ * Stop rabbitmq-bundle-0 ( controller-0 ) due to unrunnable rabbitmq-bundle-docker-0 start ++ * Stop rabbitmq:0 ( rabbitmq-bundle-0 ) due to unrunnable rabbitmq-bundle-docker-0 start ++ * Stop galera-bundle-docker-0 ( controller-0 ) due to node availability ++ * Stop galera-bundle-0 ( controller-0 ) due to unrunnable galera-bundle-docker-0 start ++ * Stop galera:0 ( Master galera-bundle-0 ) due to unrunnable galera-bundle-docker-0 start ++ * Stop redis-bundle-docker-0 ( controller-0 ) due to node availability ++ * Stop redis-bundle-0 ( controller-0 ) due to unrunnable redis-bundle-docker-0 start ++ * Stop redis:0 ( Master redis-bundle-0 ) due to unrunnable redis-bundle-docker-0 start ++ * Promote redis:1 ( Slave -> Master redis-bundle-1 ) ++ * Move ip-192.168.24.7 ( controller-0 -> controller-2 ) ++ * Move ip-10.0.0.109 ( controller-0 -> controller-1 ) ++ * Move ip-172.17.4.11 ( controller-0 -> controller-1 ) ++ * Stop haproxy-bundle-docker-0 ( controller-0 ) due to node availability ++ * Move stonith-fence_ipmilan-5254003e8e97 ( controller-0 -> controller-1 ) ++ * Move stonith-fence_ipmilan-5254000dcb3f ( controller-0 -> controller-2 ) ++ ++Executing cluster transition: ++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_stop_0 ++ * Resource action: redis cancel=45000 on redis-bundle-1 ++ * Resource action: redis cancel=60000 on redis-bundle-1 ++ * Pseudo action: redis-bundle-master_pre_notify_demote_0 ++ * Pseudo action: stonith-fence_ipmilan-5254003e8e97_stop_0 ++ * Pseudo action: stonith-fence_ipmilan-5254000dcb3f_stop_0 ++ * Pseudo action: haproxy-bundle_stop_0 ++ * Pseudo action: redis-bundle_demote_0 ++ * Pseudo action: galera-bundle_demote_0 ++ * Pseudo action: rabbitmq-bundle_stop_0 ++ * Pseudo action: rabbitmq-bundle_start_0 ++ * Fencing controller-0 (reboot) ++ * Resource action: rabbitmq notify on rabbitmq-bundle-1 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-2 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_stop_0 ++ * Pseudo action: rabbitmq-bundle-0_stop_0 ++ * Pseudo action: galera-bundle-master_demote_0 ++ * Pseudo action: galera-bundle-0_stop_0 ++ * Resource action: redis notify on redis-bundle-1 ++ * Resource action: redis notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_demote_0 ++ * Pseudo action: redis-bundle-master_demote_0 ++ * Pseudo action: redis-bundle-0_stop_0 ++ * Pseudo action: haproxy-bundle-docker-0_stop_0 ++ * Pseudo action: stonith-redis-bundle-0-off on redis-bundle-0 ++ * Pseudo action: stonith-rabbitmq-bundle-0-off on rabbitmq-bundle-0 ++ * Pseudo action: stonith-galera-bundle-0-off on galera-bundle-0 ++ * Pseudo action: stonith_complete ++ * Pseudo action: haproxy-bundle_stopped_0 ++ * Pseudo action: rabbitmq_post_notify_stop_0 ++ * Pseudo action: rabbitmq-bundle-clone_stop_0 ++ * Pseudo action: rabbitmq-bundle-docker-0_stop_0 ++ * Pseudo action: galera_demote_0 ++ * Pseudo action: galera-bundle-master_demoted_0 ++ * Pseudo action: redis_post_notify_stop_0 ++ * Pseudo action: redis_demote_0 ++ * Pseudo action: redis-bundle-master_demoted_0 ++ * Pseudo action: ip-192.168.24.7_stop_0 ++ * Pseudo action: ip-10.0.0.109_stop_0 ++ * Pseudo action: ip-172.17.4.11_stop_0 ++ * Pseudo action: galera-bundle_demoted_0 ++ * Pseudo action: galera-bundle_stop_0 ++ * Pseudo action: rabbitmq_stop_0 ++ * Pseudo action: rabbitmq-bundle-clone_stopped_0 ++ * Pseudo action: galera-bundle-master_stop_0 ++ * Pseudo action: galera-bundle-docker-0_stop_0 ++ * Pseudo action: redis-bundle-master_post_notify_demoted_0 ++ * Resource action: ip-192.168.24.7 start on controller-2 ++ * Resource action: ip-10.0.0.109 start on controller-1 ++ * Resource action: ip-172.17.4.11 start on controller-1 ++ * Pseudo action: rabbitmq-bundle-clone_post_notify_stopped_0 ++ * Pseudo action: galera_stop_0 ++ * Pseudo action: galera-bundle-master_stopped_0 ++ * Resource action: redis notify on redis-bundle-1 ++ * Resource action: redis notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_demoted_0 ++ * Pseudo action: redis-bundle-master_pre_notify_stop_0 ++ * Resource action: ip-192.168.24.7 monitor=10000 on controller-2 ++ * Resource action: ip-10.0.0.109 monitor=10000 on controller-1 ++ * Resource action: ip-172.17.4.11 monitor=10000 on controller-1 ++ * Pseudo action: redis-bundle_demoted_0 ++ * Pseudo action: redis-bundle_stop_0 ++ * Pseudo action: galera-bundle_stopped_0 ++ * Pseudo action: galera-bundle_start_0 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-1 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-2 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_stopped_0 ++ * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 ++ * Pseudo action: galera-bundle-master_start_0 ++ * Resource action: redis notify on redis-bundle-1 ++ * Resource action: redis notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_stop_0 ++ * Pseudo action: redis-bundle-master_stop_0 ++ * Pseudo action: redis-bundle-docker-0_stop_0 ++ * Pseudo action: rabbitmq-bundle_stopped_0 ++ * Pseudo action: rabbitmq_notified_0 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-1 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-2 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 ++ * Pseudo action: galera-bundle-master_running_0 ++ * Pseudo action: redis_stop_0 ++ * Pseudo action: redis-bundle-master_stopped_0 ++ * Pseudo action: galera-bundle_running_0 ++ * Pseudo action: rabbitmq-bundle-clone_running_0 ++ * Pseudo action: redis-bundle-master_post_notify_stopped_0 ++ * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 ++ * Resource action: redis notify on redis-bundle-1 ++ * Resource action: redis notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_stopped_0 ++ * Pseudo action: redis-bundle-master_pre_notify_start_0 ++ * Pseudo action: redis-bundle_stopped_0 ++ * Pseudo action: redis-bundle_start_0 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-0 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-1 ++ * Resource action: rabbitmq notify on rabbitmq-bundle-2 ++ * Pseudo action: rabbitmq-bundle-clone_confirmed-post_notify_running_0 ++ * Pseudo action: redis_notified_0 ++ * Resource action: redis notify on redis-bundle-1 ++ * Resource action: redis notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_start_0 ++ * Pseudo action: redis-bundle-master_start_0 ++ * Pseudo action: rabbitmq-bundle_running_0 ++ * Pseudo action: all_stopped ++ * Pseudo action: redis-bundle-master_running_0 ++ * Resource action: stonith-fence_ipmilan-5254003e8e97 start on controller-1 ++ * Resource action: stonith-fence_ipmilan-5254000dcb3f start on controller-2 ++ * Pseudo action: redis-bundle-master_post_notify_running_0 ++ * Resource action: stonith-fence_ipmilan-5254003e8e97 monitor=60000 on controller-1 ++ * Resource action: stonith-fence_ipmilan-5254000dcb3f monitor=60000 on controller-2 ++ * Resource action: redis notify on redis-bundle-0 ++ * Resource action: redis notify on redis-bundle-1 ++ * Resource action: redis notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_running_0 ++ * Pseudo action: redis-bundle_running_0 ++ * Pseudo action: redis-bundle-master_pre_notify_promote_0 ++ * Pseudo action: redis-bundle_promote_0 ++ * Resource action: redis notify on redis-bundle-0 ++ * Resource action: redis notify on redis-bundle-1 ++ * Resource action: redis notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-pre_notify_promote_0 ++ * Pseudo action: redis-bundle-master_promote_0 ++ * Resource action: redis promote on redis-bundle-1 ++ * Pseudo action: redis-bundle-master_promoted_0 ++ * Pseudo action: redis-bundle-master_post_notify_promoted_0 ++ * Resource action: redis notify on redis-bundle-0 ++ * Resource action: redis notify on redis-bundle-1 ++ * Resource action: redis notify on redis-bundle-2 ++ * Pseudo action: redis-bundle-master_confirmed-post_notify_promoted_0 ++ * Pseudo action: redis-bundle_promoted_0 ++ * Resource action: redis monitor=20000 on redis-bundle-1 ++Using the original execution date of: 2017-09-12 10:51:59Z ++ ++Revised cluster status: ++Online: [ controller-1 controller-2 ] ++OFFLINE: [ controller-0 ] ++Containers: [ galera-bundle-1:galera-bundle-docker-1 galera-bundle-2:galera-bundle-docker-2 rabbitmq-bundle-1:rabbitmq-bundle-docker-1 rabbitmq-bundle-2:rabbitmq-bundle-docker-2 redis-bundle-1:redis-bundle-docker-1 redis-bundle-2:redis-bundle-docker-2 ] ++ ++ Docker container set: rabbitmq-bundle [192.168.24.1:8787/rhosp12/openstack-rabbitmq-docker:pcmklatest] ++ rabbitmq-bundle-0 (ocf::heartbeat:rabbitmq-cluster): FAILED ++ rabbitmq-bundle-1 (ocf::heartbeat:rabbitmq-cluster): Started controller-1 ++ rabbitmq-bundle-2 (ocf::heartbeat:rabbitmq-cluster): Started controller-2 ++ Docker container set: galera-bundle [192.168.24.1:8787/rhosp12/openstack-mariadb-docker:pcmklatest] ++ galera-bundle-0 (ocf::heartbeat:galera): FAILED Master ++ galera-bundle-1 (ocf::heartbeat:galera): Master controller-1 ++ galera-bundle-2 (ocf::heartbeat:galera): Master controller-2 ++ Docker container set: redis-bundle [192.168.24.1:8787/rhosp12/openstack-redis-docker:pcmklatest] ++ redis-bundle-0 (ocf::heartbeat:redis): FAILED Master ++ redis-bundle-1 (ocf::heartbeat:redis): Master controller-1 ++ redis-bundle-2 (ocf::heartbeat:redis): Slave controller-2 ++ ip-192.168.24.7 (ocf::heartbeat:IPaddr2): Started controller-2 ++ ip-10.0.0.109 (ocf::heartbeat:IPaddr2): Started controller-1 ++ ip-172.17.1.14 (ocf::heartbeat:IPaddr2): Started controller-2 ++ ip-172.17.1.19 (ocf::heartbeat:IPaddr2): Started controller-2 ++ ip-172.17.3.19 (ocf::heartbeat:IPaddr2): Started controller-2 ++ ip-172.17.4.11 (ocf::heartbeat:IPaddr2): Started controller-1 ++ Docker container set: haproxy-bundle [192.168.24.1:8787/rhosp12/openstack-haproxy-docker:pcmklatest] ++ haproxy-bundle-docker-0 (ocf::heartbeat:docker): Stopped ++ haproxy-bundle-docker-1 (ocf::heartbeat:docker): Started controller-2 ++ haproxy-bundle-docker-2 (ocf::heartbeat:docker): Started controller-1 ++ openstack-cinder-volume (systemd:openstack-cinder-volume): Started controller-2 ++ stonith-fence_ipmilan-525400efba5c (stonith:fence_ipmilan): Started controller-2 ++ stonith-fence_ipmilan-5254003e8e97 (stonith:fence_ipmilan): Started controller-1 ++ stonith-fence_ipmilan-5254000dcb3f (stonith:fence_ipmilan): Started controller-2 ++ +diff --git a/pengine/test10/bundle-order-fencing.xml b/pengine/test10/bundle-order-fencing.xml +new file mode 100644 +index 0000000..b303116 +--- /dev/null ++++ b/pengine/test10/bundle-order-fencing.xml +@@ -0,0 +1,799 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +-- +1.8.3.1 + + +From 14583c664638d17a14bccfd130739c7242f5cec4 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Wed, 20 Sep 2017 21:53:00 +1000 +Subject: [PATCH 10/21] Fix: PE: Resources in bundles should also respect + failcounts + +--- + pengine/container.c | 35 ++++++++++++++++++++++++++++++++++- + 1 file changed, 34 insertions(+), 1 deletion(-) + +diff --git a/pengine/container.c b/pengine/container.c +index 1e9b971..0e556bb 100644 +--- a/pengine/container.c ++++ b/pengine/container.c +@@ -74,6 +74,38 @@ static GListPtr get_containers_or_children(resource_t *rsc) + } + } + ++static bool ++migration_threshold_reached(resource_t *rsc, node_t *node, ++ pe_working_set_t *data_set) ++{ ++ int fail_count, countdown; ++ ++ /* Migration threshold of 0 means never force away */ ++ if (rsc->migration_threshold == 0) { ++ return FALSE; ++ } ++ ++ /* If there are no failures, there's no need to force away */ ++ fail_count = get_failcount_all(node, rsc, NULL, data_set); ++ if (fail_count <= 0) { ++ return FALSE; ++ } ++ ++ /* How many more times recovery will be tried on this node */ ++ countdown = QB_MAX(rsc->migration_threshold - fail_count, 0); ++ ++ if (countdown == 0) { ++ crm_warn("Forcing %s away from %s after %d failures (max=%d)", ++ rsc->id, node->details->uname, fail_count, ++ rsc->migration_threshold); ++ return TRUE; ++ } ++ ++ crm_info("%s can fail %d more times on %s before being forced off", ++ rsc->id, countdown, node->details->uname); ++ return FALSE; ++} ++ + node_t * + container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + { +@@ -128,7 +160,7 @@ container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + while (g_hash_table_iter_next(&iter, NULL, (gpointer *) & node)) { + if(node->details != tuple->node->details) { + node->weight = -INFINITY; +- } else { ++ } else if(migration_threshold_reached(tuple->child, node, data_set) == FALSE) { + node->weight = INFINITY; + } + } +@@ -158,6 +190,7 @@ container_color(resource_t * rsc, node_t * prefer, pe_working_set_t * data_set) + return NULL; + } + ++ + void + container_create_actions(resource_t * rsc, pe_working_set_t * data_set) + { +-- +1.8.3.1 + + +From 1eacc741d04098d24b97631a273501ef0aff2fc1 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Thu, 21 Sep 2017 20:59:16 +1000 +Subject: [PATCH 11/21] crm_resource: Do not send history erasure commands for + nodes where this is none + +If a resource is not known on a given node, thats because we have no +operation history for it there. So there is no benefit to requesting a +cleanup for that resource/node combination. +--- + tools/crm_resource_runtime.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c +index 8f9ae0d..210ea7d 100644 +--- a/tools/crm_resource_runtime.c ++++ b/tools/crm_resource_runtime.c +@@ -573,7 +573,7 @@ send_lrm_rsc_op(crm_ipc_t * crmd_channel, const char *op, + rc = 0; + + } else { +- CMD_ERR("Could not send %s op to the crmd", op); ++ crm_debug("Could not send %s op to the crmd", op); + rc = -ENOTCONN; + } + +@@ -627,8 +627,9 @@ cli_resource_delete(crm_ipc_t *crmd_channel, const char *host_uname, + + } else if (host_uname == NULL) { + GListPtr lpc = NULL; ++ GListPtr nodes = g_hash_table_get_values(rsc->known_on); + +- for (lpc = data_set->nodes; lpc != NULL; lpc = lpc->next) { ++ for (lpc = nodes; lpc != NULL; lpc = lpc->next) { + node = (node_t *) lpc->data; + + if (node->details->online) { +@@ -637,6 +638,7 @@ cli_resource_delete(crm_ipc_t *crmd_channel, const char *host_uname, + } + } + ++ g_list_free(nodes); + return pcmk_ok; + } + +-- +1.8.3.1 + + +From e10299600e87356114dbb96df981c2ff196b8f9e Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Fri, 22 Sep 2017 19:51:35 +1000 +Subject: [PATCH 12/21] Log: attrd: Tweak log messages for clarity + +--- + attrd/commands.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/attrd/commands.c b/attrd/commands.c +index 6f40d60..96a1167 100644 +--- a/attrd/commands.c ++++ b/attrd/commands.c +@@ -670,12 +670,12 @@ attrd_peer_remove(const char *host, gboolean uncache, const char *source) + GHashTableIter aIter; + + CRM_CHECK(host != NULL, return); +- crm_notice("Removing all %s attributes for %s", host, source); ++ crm_notice("Removing all %s attributes for peer %s", host, source); + + g_hash_table_iter_init(&aIter, attributes); + while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) { + if(g_hash_table_remove(a->values, host)) { +- crm_debug("Removed %s[%s] for %s", a->id, host, source); ++ crm_debug("Removed %s[%s] for peer %s", a->id, host, source); + } + } + +-- +1.8.3.1 + + +From e4bd6faed4481c10befd65e5c8315d81d5e8e43d Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Fri, 22 Sep 2017 20:03:02 +1000 +Subject: [PATCH 13/21] PE: Revert e21a4d00 since we will support probing + remote connections + +Also it causes containers to be unnecessarily restarted when the admin +triggers a cleanup. +--- + lib/pengine/unpack.c | 24 +----------------------- + 1 file changed, 1 insertion(+), 23 deletions(-) + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 6e0651e..859d638 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -2342,9 +2342,6 @@ unpack_lrm_resources(node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * d + { + xmlNode *rsc_entry = NULL; + gboolean found_orphaned_container_filler = FALSE; +- GListPtr unexpected_containers = NULL; +- GListPtr gIter = NULL; +- resource_t *remote = NULL; + + CRM_CHECK(node != NULL, return FALSE); + +@@ -2354,31 +2351,13 @@ unpack_lrm_resources(node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * d + rsc_entry = __xml_next_element(rsc_entry)) { + + if (crm_str_eq((const char *)rsc_entry->name, XML_LRM_TAG_RESOURCE, TRUE)) { +- resource_t *rsc; +- rsc = unpack_lrm_rsc_state(node, rsc_entry, data_set); ++ resource_t *rsc = unpack_lrm_rsc_state(node, rsc_entry, data_set); + if (!rsc) { + continue; + } + if (is_set(rsc->flags, pe_rsc_orphan_container_filler)) { + found_orphaned_container_filler = TRUE; + } +- if (is_set(rsc->flags, pe_rsc_unexpectedly_running)) { +- remote = rsc_contains_remote_node(data_set, rsc); +- if (remote) { +- unexpected_containers = g_list_append(unexpected_containers, remote); +- } +- } +- } +- } +- +- /* If a container resource is unexpectedly up... and the remote-node +- * connection resource for that container is not up, the entire container +- * must be recovered. */ +- for (gIter = unexpected_containers; gIter != NULL; gIter = gIter->next) { +- remote = (resource_t *) gIter->data; +- if (remote->role != RSC_ROLE_STARTED) { +- crm_warn("Recovering container resource %s. Resource is unexpectedly running and involves a remote-node.", remote->container->id); +- set_bit(remote->container->flags, pe_rsc_failed); + } + } + +@@ -2388,7 +2367,6 @@ unpack_lrm_resources(node_t * node, xmlNode * lrm_rsc_list, pe_working_set_t * d + if (found_orphaned_container_filler) { + handle_orphaned_container_fillers(lrm_rsc_list, data_set); + } +- g_list_free(unexpected_containers); + return TRUE; + } + +-- +1.8.3.1 + + +From 12d453ccfa3a3a36456e6c2985f65137fde2a01d Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Mon, 25 Sep 2017 17:48:40 +1000 +Subject: [PATCH 14/21] PE: Implement probing of container remote nodes + +--- + include/crm/pengine/status.h | 7 + + lib/pengine/container.c | 3 + + pengine/allocate.c | 3 +- + pengine/native.c | 71 +++- + pengine/pengine.h | 6 - + pengine/test10/bug-cl-5247.dot | 6 + + pengine/test10/bug-cl-5247.exp | 346 +++++++++------- + pengine/test10/bug-cl-5247.summary | 2 + + pengine/test10/bug-rh-1097457.dot | 32 ++ + pengine/test10/bug-rh-1097457.exp | 456 +++++++++++++++------ + pengine/test10/bug-rh-1097457.summary | 20 + + pengine/test10/bundle-order-partial-start-2.dot | 2 + + pengine/test10/bundle-order-partial-start-2.exp | 282 +++++++------ + .../test10/bundle-order-partial-start-2.summary | 1 + + pengine/test10/bundle-order-partial-start.dot | 2 + + pengine/test10/bundle-order-partial-start.exp | 272 ++++++------ + pengine/test10/bundle-order-partial-start.summary | 1 + + pengine/test10/bundle-order-startup.exp | 8 +- + pengine/test10/container-is-remote-node.dot | 4 + + pengine/test10/container-is-remote-node.exp | 39 +- + pengine/test10/container-is-remote-node.summary | 4 + + pengine/test10/guest-node-host-dies.dot | 2 + + pengine/test10/guest-node-host-dies.exp | 6 + + pengine/test10/whitebox-asymmetric.dot | 4 + + pengine/test10/whitebox-asymmetric.exp | 48 ++- + pengine/test10/whitebox-asymmetric.summary | 2 + + pengine/test10/whitebox-fail1.dot | 5 + + pengine/test10/whitebox-fail1.exp | 145 ++++--- + pengine/test10/whitebox-fail1.summary | 3 + + pengine/test10/whitebox-fail2.dot | 5 + + pengine/test10/whitebox-fail2.exp | 145 ++++--- + pengine/test10/whitebox-fail2.summary | 3 + + pengine/test10/whitebox-fail3.dot | 3 + + pengine/test10/whitebox-fail3.exp | 15 +- + pengine/test10/whitebox-fail3.summary | 4 +- + pengine/test10/whitebox-imply-stop-on-fence.dot | 19 + + pengine/test10/whitebox-imply-stop-on-fence.exp | 325 ++++++++++----- + .../test10/whitebox-imply-stop-on-fence.summary | 8 + + pengine/test10/whitebox-migrate1.dot | 1 + + pengine/test10/whitebox-move.dot | 3 + + pengine/test10/whitebox-move.exp | 108 ++--- + pengine/test10/whitebox-move.summary | 3 +- + pengine/test10/whitebox-ms-ordering-move.dot | 12 + + pengine/test10/whitebox-ms-ordering-move.exp | 294 ++++++++----- + pengine/test10/whitebox-ms-ordering-move.summary | 12 + + pengine/test10/whitebox-nested-group.dot | 21 + + pengine/test10/whitebox-nested-group.exp | 66 ++- + pengine/test10/whitebox-nested-group.summary | 16 +- + pengine/test10/whitebox-orphan-ms.dot | 18 + + pengine/test10/whitebox-orphan-ms.exp | 92 ++--- + pengine/test10/whitebox-orphaned.dot | 8 + + pengine/test10/whitebox-orphaned.exp | 120 ++++-- + pengine/test10/whitebox-orphaned.summary | 9 +- + pengine/test10/whitebox-start.dot | 4 + + pengine/test10/whitebox-start.exp | 99 +++-- + pengine/test10/whitebox-start.summary | 10 +- + pengine/test10/whitebox-stop.dot | 5 + + pengine/test10/whitebox-stop.exp | 82 ++-- + pengine/test10/whitebox-stop.summary | 9 +- + pengine/test10/whitebox-unexpectedly-running.dot | 24 +- + pengine/test10/whitebox-unexpectedly-running.exp | 112 +++-- + .../test10/whitebox-unexpectedly-running.scores | 8 + + .../test10/whitebox-unexpectedly-running.summary | 23 +- + pengine/test10/whitebox-unexpectedly-running.xml | 11 + + 64 files changed, 2322 insertions(+), 1157 deletions(-) + +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index fcd5226..dc5f6eb 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -395,6 +395,13 @@ enum pe_link_state { + pe_link_dup, + }; + ++ ++enum rsc_discover_e { ++ discover_always = 0, ++ discover_never, ++ discover_exclusive, ++}; ++ + /* *INDENT-OFF* */ + enum pe_ordering { + pe_order_none = 0x0, /* deleted */ +diff --git a/lib/pengine/container.c b/lib/pengine/container.c +index 5831b58..a550ff3 100644 +--- a/lib/pengine/container.c ++++ b/lib/pengine/container.c +@@ -348,6 +348,7 @@ disallow_node(resource_t *rsc, const char *uname) + + if (match) { + ((pe_node_t *) match)->weight = -INFINITY; ++ ((pe_node_t *) match)->rsc_discover_mode = discover_never; + } + if (rsc->children) { + GListPtr child; +@@ -434,6 +435,7 @@ create_remote_resource( + } else { + node->weight = -INFINITY; + } ++ node->rsc_discover_mode = discover_never; + + /* unpack_remote_nodes() ensures that each remote node and guest node + * has a pe_node_t entry. Ideally, it would do the same for bundle nodes. +@@ -458,6 +460,7 @@ create_remote_resource( + + tuple->node = node_copy(node); + tuple->node->weight = 500; ++ tuple->node->rsc_discover_mode = discover_exclusive; + + if (common_unpack(xml_remote, &tuple->remote, parent, data_set) == FALSE) { + return FALSE; +diff --git a/pengine/allocate.c b/pengine/allocate.c +index d610035..401c18e 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -900,8 +900,7 @@ probe_resources(pe_working_set_t * data_set) + const char *probed = node_attribute_raw(node, CRM_OP_PROBED); + + if (is_container_remote_node(node)) { +- /* TODO enable guest node probes once ordered probing is implemented */ +- continue; ++ /* Guest node probes and their ordering requirements are now functional */ + + } else if (node->details->online == FALSE && node->details->remote_rsc) { + enum remote_connection_state state = get_remote_node_state(node); +diff --git a/pengine/native.c b/pengine/native.c +index 37cf541..2e40a4c 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -2784,10 +2784,6 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + if (force == FALSE && is_not_set(data_set->flags, pe_flag_startup_probes)) { + pe_rsc_trace(rsc, "Skipping active resource detection for %s", rsc->id); + return FALSE; +- } else if (force == FALSE && is_container_remote_node(node)) { +- pe_rsc_trace(rsc, "Skipping active resource detection for %s on container %s", +- rsc->id, node->details->id); +- return FALSE; + } + + if (is_remote_node(node)) { +@@ -2847,6 +2843,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + } + + allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id); ++ + if (rsc->exclusive_discover || top->exclusive_discover) { + if (allowed == NULL) { + /* exclusive discover is enabled and this node is not in the allowed list. */ +@@ -2857,11 +2854,77 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + return FALSE; + } + } ++ ++ if(allowed == NULL && node->rsc_discover_mode == discover_never) { ++ /* If this node was allowed to host this resource it would ++ * have been explicitly added to the 'allowed_nodes' list. ++ * However it wasn't and the node has discovery disabled, so ++ * no need to probe for this resource. ++ */ ++ return FALSE; ++ } ++ + if (allowed && allowed->rsc_discover_mode == discover_never) { + /* this resource is marked as not needing to be discovered on this node */ + return FALSE; + } + ++ if(allowed != NULL && is_container_remote_node(allowed)) { ++ resource_t *remote = allowed->details->remote_rsc; ++ ++ if(remote->role == RSC_ROLE_STOPPED) { ++ /* If the container is stopped, then we know anything that ++ * might have been inside it is also stopped and there is ++ * no need to probe. ++ * ++ * If we don't know the container's state on the target ++ * either: ++ * ++ * - the container is running, the transition will abort ++ * and we'll end up in a different case next time, or ++ * ++ * - the container is stopped ++ * ++ * Either way there is no need to probe. ++ * ++ */ ++ if(remote->allocated_to ++ && g_hash_table_lookup(remote->known_on, remote->allocated_to->details->id) == NULL) { ++ /* For safety, we order the 'rsc' start after 'remote' ++ * has been probed. ++ * ++ * Using 'top' helps for groups, but in we may need to ++ * follow the start's ordering chain backwards. ++ */ ++ custom_action_order(remote, generate_op_key(remote->id, RSC_STATUS, 0), NULL, ++ top, generate_op_key(top->id, RSC_START, 0), NULL, ++ pe_order_optional, data_set); ++ } ++ return FALSE; ++ ++ /* Here we really we want to check if remote->stop is required, ++ * but that information doesn't exist yet ++ */ ++ } else if(allowed->details->remote_requires_reset ++ || allowed->details->unclean ++ || is_set(remote->flags, pe_rsc_failed) ++ || remote->next_role == RSC_ROLE_STOPPED ++ || (remote->allocated_to ++ && pe_find_node(remote->running_on, remote->allocated_to->details->uname) == NULL) ++ ) { ++ /* The container is stopping or restarting, don't start ++ * 'rsc' until 'remote' stops as this also implies that ++ * 'rsc' is stopped - avoiding the need to probe ++ */ ++ custom_action_order(remote, generate_op_key(remote->id, RSC_STOP, 0), NULL, ++ top, generate_op_key(top->id, RSC_START, 0), NULL, ++ pe_order_optional, data_set); ++ return FALSE; ++/* } else { ++ * The container is running so there is no problem probing it ++ */ ++ } ++ } + + key = generate_op_key(rsc->id, RSC_STATUS, 0); + probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set); +diff --git a/pengine/pengine.h b/pengine/pengine.h +index e3f4874..2c13258 100644 +--- a/pengine/pengine.h ++++ b/pengine/pengine.h +@@ -75,12 +75,6 @@ struct rsc_ticket_s { + int role_lh; + }; + +-enum rsc_discover_e { +- discover_always = 0, +- discover_never, +- discover_exclusive, +-}; +- + struct rsc_to_node_s { + char *id; + resource_t *rsc_lh; +diff --git a/pengine/test10/bug-cl-5247.dot b/pengine/test10/bug-cl-5247.dot +index 44f788d..decf5c3 100644 +--- a/pengine/test10/bug-cl-5247.dot ++++ b/pengine/test10/bug-cl-5247.dot +@@ -127,6 +127,9 @@ digraph "g" { + "stonith_complete" -> "vip-master_start_0 pgsr01" [ style = bold] + "stonith_complete" -> "vip-rep_start_0 pgsr01" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] ++"vip-master_monitor_0 pgsr01" -> "vip-master_start_0 pgsr01" [ style = bold] ++"vip-master_monitor_0 pgsr01" -> "vip-master_stop_0 pgsr02" [ style = bold] ++"vip-master_monitor_0 pgsr01" [ style=bold color="green" fontcolor="black"] + "vip-master_monitor_10000 pgsr01" [ style=bold color="green" fontcolor="black"] + "vip-master_start_0 pgsr01" -> "master-group_running_0" [ style = bold] + "vip-master_start_0 pgsr01" -> "vip-master_monitor_10000 pgsr01" [ style = bold] +@@ -136,6 +139,9 @@ digraph "g" { + "vip-master_stop_0 pgsr02" -> "master-group_stopped_0" [ style = bold] + "vip-master_stop_0 pgsr02" -> "vip-master_start_0 pgsr01" [ style = bold] + "vip-master_stop_0 pgsr02" [ style=bold color="green" fontcolor="orange"] ++"vip-rep_monitor_0 pgsr01" -> "vip-rep_start_0 pgsr01" [ style = bold] ++"vip-rep_monitor_0 pgsr01" -> "vip-rep_stop_0 pgsr02" [ style = bold] ++"vip-rep_monitor_0 pgsr01" [ style=bold color="green" fontcolor="black"] + "vip-rep_monitor_10000 pgsr01" [ style=bold color="green" fontcolor="black"] + "vip-rep_start_0 pgsr01" -> "master-group_running_0" [ style = bold] + "vip-rep_start_0 pgsr01" -> "vip-rep_monitor_10000 pgsr01" [ style = bold] +diff --git a/pengine/test10/bug-cl-5247.exp b/pengine/test10/bug-cl-5247.exp +index 2ec83c0..47c369a 100644 +--- a/pengine/test10/bug-cl-5247.exp ++++ b/pengine/test10/bug-cl-5247.exp +@@ -8,28 +8,28 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -46,32 +46,32 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -87,10 +87,10 @@ + + + +- ++ + + +- ++ + + + +@@ -109,22 +109,22 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -132,7 +132,7 @@ + + + +- ++ + + + +@@ -141,32 +141,32 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -182,10 +182,10 @@ + + + +- ++ + + +- ++ + + + +@@ -204,472 +204,502 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ ++ ++ ++ + + + + + +- ++ + + + + + +- ++ ++ ++ ++ + + +- ++ + + +- ++ + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ + + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -678,9 +708,9 @@ + + + +- ++ + +- ++ + + + +@@ -694,7 +724,7 @@ + + + +- ++ + + + +@@ -703,9 +733,9 @@ + + + +- ++ + +- ++ + + + +@@ -718,19 +748,19 @@ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + + + +@@ -741,31 +771,31 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary +index 3e5511d..f15ef2e 100644 +--- a/pengine/test10/bug-cl-5247.summary ++++ b/pengine/test10/bug-cl-5247.summary +@@ -32,6 +32,8 @@ Executing cluster transition: + * Resource action: prmStonith1-2 stop on bl460g8n4 + * Pseudo action: grpStonith2_stop_0 + * Resource action: prmStonith2-2 stop on bl460g8n3 ++ * Resource action: vip-master monitor on pgsr01 ++ * Resource action: vip-rep monitor on pgsr01 + * Pseudo action: msPostgresql_pre_notify_demote_0 + * Resource action: pgsr01 monitor on bl460g8n4 + * Resource action: pgsr02 monitor on bl460g8n3 +diff --git a/pengine/test10/bug-rh-1097457.dot b/pengine/test10/bug-rh-1097457.dot +index e74c8fb..7791e06 100644 +--- a/pengine/test10/bug-rh-1097457.dot ++++ b/pengine/test10/bug-rh-1097457.dot +@@ -1,4 +1,16 @@ + digraph "g" { ++"FAKE1-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] ++"FAKE1_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] ++"FAKE2-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] ++"FAKE2_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] ++"FAKE3-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] ++"FAKE3_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] ++"FAKE4-IP_monitor_0 lamaVM1" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] ++"FAKE4-IP_monitor_0 lamaVM1" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] ++"FAKE4-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] ++"FAKE4-IP_monitor_0 lamaVM3" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] ++"FAKE4-IP_monitor_0 lamaVM3" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] ++"FAKE4-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] + "FAKE4-IP_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] + "FAKE4-IP_start_0 lamaVM2" -> "FAKE4-IP_monitor_30000 lamaVM2" [ style = bold] + "FAKE4-IP_start_0 lamaVM2" -> "lamaVM2-G4_running_0" [ style = bold] +@@ -8,6 +20,12 @@ digraph "g" { + "FAKE4-IP_stop_0 lamaVM2" -> "all_stopped" [ style = bold] + "FAKE4-IP_stop_0 lamaVM2" -> "lamaVM2-G4_stopped_0" [ style = bold] + "FAKE4-IP_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] ++"FAKE4_monitor_0 lamaVM1" -> "FAKE4_start_0 lamaVM2" [ style = bold] ++"FAKE4_monitor_0 lamaVM1" -> "FAKE4_stop_0 lamaVM2" [ style = bold] ++"FAKE4_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] ++"FAKE4_monitor_0 lamaVM3" -> "FAKE4_start_0 lamaVM2" [ style = bold] ++"FAKE4_monitor_0 lamaVM3" -> "FAKE4_stop_0 lamaVM2" [ style = bold] ++"FAKE4_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] + "FAKE4_monitor_30000 lamaVM2" [ style=bold color="green" fontcolor="black"] + "FAKE4_start_0 lamaVM2" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] + "FAKE4_start_0 lamaVM2" -> "FAKE4_monitor_30000 lamaVM2" [ style = bold] +@@ -17,6 +35,11 @@ digraph "g" { + "FAKE4_stop_0 lamaVM2" -> "all_stopped" [ style = bold] + "FAKE4_stop_0 lamaVM2" -> "lamaVM2-G4_stopped_0" [ style = bold] + "FAKE4_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] ++"FAKE5-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] ++"FAKE5-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] ++"FAKE5_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] ++"FAKE6-IP_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] ++"FAKE6-IP_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] + "FAKE6-clone_running_0" [ style=bold color="green" fontcolor="orange"] + "FAKE6-clone_start_0" -> "FAKE6-clone_running_0" [ style = bold] + "FAKE6-clone_start_0" -> "FAKE6_start_0 lamaVM2" [ style = bold] +@@ -34,6 +57,14 @@ digraph "g" { + "FAKE6_stop_0 lamaVM2" -> "FAKE6_start_0 lamaVM2" [ style = bold] + "FAKE6_stop_0 lamaVM2" -> "all_stopped" [ style = bold] + "FAKE6_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] ++"FSlun1_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] ++"FSlun2_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] ++"FSlun3_monitor_0 lamaVM1" -> "FSlun3_start_0 lama2" [ style = bold] ++"FSlun3_monitor_0 lamaVM1" -> "FSlun3_stop_0 lamaVM2" [ style = bold] ++"FSlun3_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] ++"FSlun3_monitor_0 lamaVM3" -> "FSlun3_start_0 lama2" [ style = bold] ++"FSlun3_monitor_0 lamaVM3" -> "FSlun3_stop_0 lamaVM2" [ style = bold] ++"FSlun3_monitor_0 lamaVM3" [ style=bold color="green" fontcolor="black"] + "FSlun3_monitor_10000 lama2" [ style=bold color="green" fontcolor="black"] + "FSlun3_monitor_10000 lamaVM2" [ style=bold color="green" fontcolor="black"] + "FSlun3_start_0 lama2" -> "FSlun3_monitor_10000 lama2" [ style = bold] +@@ -42,6 +73,7 @@ digraph "g" { + "FSlun3_stop_0 lamaVM2" -> "FSlun3_start_0 lama2" [ style = bold] + "FSlun3_stop_0 lamaVM2" -> "all_stopped" [ style = bold] + "FSlun3_stop_0 lamaVM2" [ style=bold color="green" fontcolor="orange"] ++"FSlun4_monitor_0 lamaVM1" [ style=bold color="green" fontcolor="black"] + "VM2_monitor_10000 lama3" [ style=bold color="green" fontcolor="black"] + "VM2_start_0 lama3" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] + "VM2_start_0 lama3" -> "FAKE4_start_0 lamaVM2" [ style = bold] +diff --git a/pengine/test10/bug-rh-1097457.exp b/pengine/test10/bug-rh-1097457.exp +index eb9b225..33c13bd 100644 +--- a/pengine/test10/bug-rh-1097457.exp ++++ b/pengine/test10/bug-rh-1097457.exp +@@ -1,34 +1,52 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + + + +@@ -37,58 +55,88 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ ++ ++ ++ + + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -97,117 +145,255 @@ + + + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -216,57 +402,87 @@ + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -275,54 +491,54 @@ + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -331,89 +547,89 @@ + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -423,7 +639,7 @@ + + + +- ++ + + + +@@ -432,13 +648,13 @@ + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -447,23 +663,23 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + + + +@@ -471,25 +687,25 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary +index c50df5f..e23c6ad 100644 +--- a/pengine/test10/bug-rh-1097457.summary ++++ b/pengine/test10/bug-rh-1097457.summary +@@ -41,6 +41,26 @@ Transition Summary: + * Restart lamaVM2 ( lama3 ) due to required VM2 start + + Executing cluster transition: ++ * Resource action: FSlun1 monitor on lamaVM3 ++ * Resource action: FSlun2 monitor on lamaVM3 ++ * Resource action: FSlun3 monitor on lamaVM3 ++ * Resource action: FSlun3 monitor on lamaVM1 ++ * Resource action: FSlun4 monitor on lamaVM1 ++ * Resource action: FAKE5-IP monitor on lamaVM3 ++ * Resource action: FAKE5-IP monitor on lamaVM1 ++ * Resource action: FAKE6-IP monitor on lamaVM3 ++ * Resource action: FAKE6-IP monitor on lamaVM1 ++ * Resource action: FAKE5 monitor on lamaVM1 ++ * Resource action: FAKE1 monitor on lamaVM3 ++ * Resource action: FAKE1-IP monitor on lamaVM3 ++ * Resource action: FAKE2 monitor on lamaVM3 ++ * Resource action: FAKE2-IP monitor on lamaVM3 ++ * Resource action: FAKE3 monitor on lamaVM3 ++ * Resource action: FAKE3-IP monitor on lamaVM3 ++ * Resource action: FAKE4 monitor on lamaVM3 ++ * Resource action: FAKE4 monitor on lamaVM1 ++ * Resource action: FAKE4-IP monitor on lamaVM3 ++ * Resource action: FAKE4-IP monitor on lamaVM1 + * Resource action: lamaVM2 stop on lama3 + * Resource action: VM2 stop on lama3 + * Pseudo action: stonith-lamaVM2-reboot on lamaVM2 +diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot +index d58e809..163be1f 100644 +--- a/pengine/test10/bundle-order-partial-start-2.dot ++++ b/pengine/test10/bundle-order-partial-start-2.dot +@@ -64,6 +64,8 @@ digraph "g" { + "rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] + "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq:0_monitor_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq:0_monitor_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] + "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp +index f600983..087ed6c 100644 +--- a/pengine/test10/bundle-order-partial-start-2.exp ++++ b/pengine/test10/bundle-order-partial-start-2.exp +@@ -1,234 +1,246 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + + + +@@ -237,29 +249,29 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -269,7 +281,7 @@ + + + +- ++ + + + +@@ -278,277 +290,277 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +@@ -556,10 +568,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary +index bf14db0..50933a2 100644 +--- a/pengine/test10/bundle-order-partial-start-2.summary ++++ b/pengine/test10/bundle-order-partial-start-2.summary +@@ -29,6 +29,7 @@ Transition Summary: + * Start haproxy-bundle-docker-0 (undercloud) + + Executing cluster transition: ++ * Resource action: rabbitmq:0 monitor on rabbitmq-bundle-0 + * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 + * Resource action: galera-bundle-0 stop on undercloud + * Pseudo action: redis-bundle-master_pre_notify_promote_0 +diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot +index 796749d..958e7f4 100644 +--- a/pengine/test10/bundle-order-partial-start.dot ++++ b/pengine/test10/bundle-order-partial-start.dot +@@ -58,6 +58,8 @@ digraph "g" { + "rabbitmq-bundle_running_0" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] + "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq:0_monitor_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq:0_monitor_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] + "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp +index d48fccf..8aad157 100644 +--- a/pengine/test10/bundle-order-partial-start.exp ++++ b/pengine/test10/bundle-order-partial-start.exp +@@ -1,529 +1,541 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start.summary b/pengine/test10/bundle-order-partial-start.summary +index 4e02e88..9045545 100644 +--- a/pengine/test10/bundle-order-partial-start.summary ++++ b/pengine/test10/bundle-order-partial-start.summary +@@ -29,6 +29,7 @@ Transition Summary: + * Start haproxy-bundle-docker-0 (undercloud) + + Executing cluster transition: ++ * Resource action: rabbitmq:0 monitor on rabbitmq-bundle-0 + * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 + * Resource action: galera-bundle-docker-0 monitor on undercloud + * Pseudo action: redis-bundle-master_pre_notify_promote_0 +diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp +index 03b064a..ec1053c 100644 +--- a/pengine/test10/bundle-order-startup.exp ++++ b/pengine/test10/bundle-order-startup.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +@@ -338,7 +338,7 @@ + + + +- ++ + + + +@@ -417,7 +417,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/container-is-remote-node.dot b/pengine/test10/container-is-remote-node.dot +index d8f1c9f..ebb0793 100644 +--- a/pengine/test10/container-is-remote-node.dot ++++ b/pengine/test10/container-is-remote-node.dot +@@ -1,2 +1,6 @@ + digraph "g" { ++"clvmd_monitor_0 RNVM1" [ style=bold color="green" fontcolor="black"] ++"dlm_monitor_0 RNVM1" [ style=bold color="green" fontcolor="black"] ++"gfs2-lv_1_1_monitor_0 RNVM1" [ style=bold color="green" fontcolor="black"] ++"gfs2-lv_1_2_monitor_0 RNVM1" [ style=bold color="green" fontcolor="black"] + } +diff --git a/pengine/test10/container-is-remote-node.exp b/pengine/test10/container-is-remote-node.exp +index 56e315f..d656dd2 100644 +--- a/pengine/test10/container-is-remote-node.exp ++++ b/pengine/test10/container-is-remote-node.exp +@@ -1 +1,38 @@ +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/container-is-remote-node.summary b/pengine/test10/container-is-remote-node.summary +index 6ed0526..f5c78ce 100644 +--- a/pengine/test10/container-is-remote-node.summary ++++ b/pengine/test10/container-is-remote-node.summary +@@ -26,6 +26,10 @@ Containers: [ RNVM1:VM1 ] + Transition Summary: + + Executing cluster transition: ++ * Resource action: dlm monitor on RNVM1 ++ * Resource action: clvmd monitor on RNVM1 ++ * Resource action: gfs2-lv_1_1 monitor on RNVM1 ++ * Resource action: gfs2-lv_1_2 monitor on RNVM1 + + Revised cluster status: + Online: [ lama2 lama3 ] +diff --git a/pengine/test10/guest-node-host-dies.dot b/pengine/test10/guest-node-host-dies.dot +index 04152d1..83f38c4 100644 +--- a/pengine/test10/guest-node-host-dies.dot ++++ b/pengine/test10/guest-node-host-dies.dot +@@ -83,6 +83,7 @@ digraph "g" { + "lxc1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "lxc1_stop_0 rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] + "lxc1_stop_0 rhel7-1" -> "lxc1_start_0 rhel7-2" [ style = bold] ++"lxc1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] + "lxc1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] + "lxc2_monitor_0 rhel7-2" -> "lxc2_start_0 rhel7-3" [ style = bold] + "lxc2_monitor_0 rhel7-2" -> "lxc2_stop_0 rhel7-1" [ style = bold] +@@ -101,6 +102,7 @@ digraph "g" { + "lxc2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "lxc2_stop_0 rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold] + "lxc2_stop_0 rhel7-1" -> "lxc2_start_0 rhel7-3" [ style = bold] ++"lxc2_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] + "lxc2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] + "rsc_rhel7-1_monitor_5000 rhel7-5" [ style=bold color="green" fontcolor="black"] + "rsc_rhel7-1_start_0 rhel7-5" -> "rsc_rhel7-1_monitor_5000 rhel7-5" [ style = bold] +diff --git a/pengine/test10/guest-node-host-dies.exp b/pengine/test10/guest-node-host-dies.exp +index cd8d81a..01fe678 100644 +--- a/pengine/test10/guest-node-host-dies.exp ++++ b/pengine/test10/guest-node-host-dies.exp +@@ -62,6 +62,12 @@ + + + ++ ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/whitebox-asymmetric.dot b/pengine/test10/whitebox-asymmetric.dot +index 896f66a..da2f3e3 100644 +--- a/pengine/test10/whitebox-asymmetric.dot ++++ b/pengine/test10/whitebox-asymmetric.dot +@@ -3,8 +3,12 @@ + "18node2_start_0 18builder" -> "18node2_monitor_30000 18builder" [ style = bold] + "18node2_start_0 18builder" -> "nfs_mount_monitor_10000 18node2" [ style = bold] + "18node2_start_0 18builder" -> "nfs_mount_start_0 18node2" [ style = bold] ++"18node2_start_0 18builder" -> "vg_tags_dup_monitor_0 18node2" [ style = bold] ++"18node2_start_0 18builder" -> "webserver_monitor_0 18node2" [ style = bold] + "18node2_start_0 18builder" [ style=bold color="green" fontcolor="black"] + "nfs_mount_monitor_10000 18node2" [ style=bold color="green" fontcolor="black"] + "nfs_mount_start_0 18node2" -> "nfs_mount_monitor_10000 18node2" [ style = bold] + "nfs_mount_start_0 18node2" [ style=bold color="green" fontcolor="black"] ++"vg_tags_dup_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] ++"webserver_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] + } +diff --git a/pengine/test10/whitebox-asymmetric.exp b/pengine/test10/whitebox-asymmetric.exp +index 1178fb6..db71443 100644 +--- a/pengine/test10/whitebox-asymmetric.exp ++++ b/pengine/test10/whitebox-asymmetric.exp +@@ -1,49 +1,75 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +diff --git a/pengine/test10/whitebox-asymmetric.summary b/pengine/test10/whitebox-asymmetric.summary +index 1c5011d..8da9996 100644 +--- a/pengine/test10/whitebox-asymmetric.summary ++++ b/pengine/test10/whitebox-asymmetric.summary +@@ -17,7 +17,9 @@ Transition Summary: + + Executing cluster transition: + * Resource action: 18node2 start on 18builder ++ * Resource action: webserver monitor on 18node2 + * Resource action: nfs_mount start on 18node2 ++ * Resource action: vg_tags_dup monitor on 18node2 + * Resource action: 18node2 monitor=30000 on 18builder + * Resource action: nfs_mount monitor=10000 on 18node2 + +diff --git a/pengine/test10/whitebox-fail1.dot b/pengine/test10/whitebox-fail1.dot +index 0123a58..3cf4f76 100644 +--- a/pengine/test10/whitebox-fail1.dot ++++ b/pengine/test10/whitebox-fail1.dot +@@ -1,10 +1,15 @@ + digraph "g" { ++"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"B_monitor_0 lxc2" -> "B_start_0 lxc1" [ style = bold] ++"B_monitor_0 lxc2" -> "B_stop_0 lxc1" [ style = bold] ++"B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "B_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] + "B_start_0 lxc1" -> "B_monitor_10000 lxc1" [ style = bold] + "B_start_0 lxc1" [ style=bold color="green" fontcolor="black"] + "B_stop_0 lxc1" -> "B_start_0 lxc1" [ style = bold] + "B_stop_0 lxc1" -> "all_stopped" [ style = bold] + "B_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] ++"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "M-clone_running_0" [ style=bold color="green" fontcolor="orange"] + "M-clone_start_0" -> "M-clone_running_0" [ style = bold] + "M-clone_start_0" -> "M_start_0 lxc1" [ style = bold] +diff --git a/pengine/test10/whitebox-fail1.exp b/pengine/test10/whitebox-fail1.exp +index 4cb6136..ab739b5 100644 +--- a/pengine/test10/whitebox-fail1.exp ++++ b/pengine/test10/whitebox-fail1.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -11,7 +11,7 @@ + + + +- ++ + + + +@@ -24,192 +24,225 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -219,7 +252,7 @@ + + + +- ++ + + + +@@ -228,13 +261,13 @@ + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -247,19 +280,19 @@ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + + + +@@ -270,16 +303,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary +index 1271d4f..d1f3480 100644 +--- a/pengine/test10/whitebox-fail1.summary ++++ b/pengine/test10/whitebox-fail1.summary +@@ -21,6 +21,9 @@ Transition Summary: + * Restart lxc1 ( 18node2 ) due to required container1 start + + Executing cluster transition: ++ * Resource action: A monitor on lxc2 ++ * Resource action: B monitor on lxc2 ++ * Resource action: D monitor on lxc2 + * Resource action: lxc1 stop on 18node2 + * Resource action: container1 stop on 18node2 + * Pseudo action: stonith-lxc1-reboot on lxc1 +diff --git a/pengine/test10/whitebox-fail2.dot b/pengine/test10/whitebox-fail2.dot +index 0123a58..3cf4f76 100644 +--- a/pengine/test10/whitebox-fail2.dot ++++ b/pengine/test10/whitebox-fail2.dot +@@ -1,10 +1,15 @@ + digraph "g" { ++"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"B_monitor_0 lxc2" -> "B_start_0 lxc1" [ style = bold] ++"B_monitor_0 lxc2" -> "B_stop_0 lxc1" [ style = bold] ++"B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "B_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] + "B_start_0 lxc1" -> "B_monitor_10000 lxc1" [ style = bold] + "B_start_0 lxc1" [ style=bold color="green" fontcolor="black"] + "B_stop_0 lxc1" -> "B_start_0 lxc1" [ style = bold] + "B_stop_0 lxc1" -> "all_stopped" [ style = bold] + "B_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] ++"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "M-clone_running_0" [ style=bold color="green" fontcolor="orange"] + "M-clone_start_0" -> "M-clone_running_0" [ style = bold] + "M-clone_start_0" -> "M_start_0 lxc1" [ style = bold] +diff --git a/pengine/test10/whitebox-fail2.exp b/pengine/test10/whitebox-fail2.exp +index 4cb6136..ab739b5 100644 +--- a/pengine/test10/whitebox-fail2.exp ++++ b/pengine/test10/whitebox-fail2.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -11,7 +11,7 @@ + + + +- ++ + + + +@@ -24,192 +24,225 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -219,7 +252,7 @@ + + + +- ++ + + + +@@ -228,13 +261,13 @@ + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -247,19 +280,19 @@ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + + + +@@ -270,16 +303,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary +index 5fd1ebd..ebf6c51 100644 +--- a/pengine/test10/whitebox-fail2.summary ++++ b/pengine/test10/whitebox-fail2.summary +@@ -21,6 +21,9 @@ Transition Summary: + * Recover lxc1 ( 18node2 ) + + Executing cluster transition: ++ * Resource action: A monitor on lxc2 ++ * Resource action: B monitor on lxc2 ++ * Resource action: D monitor on lxc2 + * Resource action: lxc1 stop on 18node2 + * Resource action: container1 stop on 18node2 + * Pseudo action: stonith-lxc1-reboot on lxc1 +diff --git a/pengine/test10/whitebox-fail3.dot b/pengine/test10/whitebox-fail3.dot +index 9814f66..6f608d1 100644 +--- a/pengine/test10/whitebox-fail3.dot ++++ b/pengine/test10/whitebox-fail3.dot +@@ -1,5 +1,8 @@ + digraph "g" { + "18builder_monitor_0 dvossel-laptop2" -> "18builder_start_0 dvossel-laptop2" [ style = bold] ++"18builder_monitor_0 dvossel-laptop2" -> "FAKE_start_0 18builder" [ style = bold] ++"18builder_monitor_0 dvossel-laptop2" -> "W-master_start_0" [ style = bold] ++"18builder_monitor_0 dvossel-laptop2" -> "X-master_start_0" [ style = bold] + "18builder_monitor_0 dvossel-laptop2" [ style=bold color="green" fontcolor="black"] + "18builder_monitor_30000 dvossel-laptop2" [ style=bold color="green" fontcolor="black"] + "18builder_start_0 dvossel-laptop2" -> "18builder_monitor_30000 dvossel-laptop2" [ style = bold] +diff --git a/pengine/test10/whitebox-fail3.exp b/pengine/test10/whitebox-fail3.exp +index ec28f12..cddd7dd 100644 +--- a/pengine/test10/whitebox-fail3.exp ++++ b/pengine/test10/whitebox-fail3.exp +@@ -17,6 +17,9 @@ + + + ++ ++ ++ + + + +@@ -92,7 +95,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -150,7 +157,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/whitebox-fail3.summary b/pengine/test10/whitebox-fail3.summary +index eded099..374fb69 100644 +--- a/pengine/test10/whitebox-fail3.summary ++++ b/pengine/test10/whitebox-fail3.summary +@@ -22,10 +22,10 @@ Transition Summary: + Executing cluster transition: + * Resource action: vm start on dvossel-laptop2 + * Resource action: FAKE stop on dvossel-laptop2 +- * Pseudo action: W-master_start_0 +- * Pseudo action: X-master_start_0 + * Resource action: 18builder monitor on dvossel-laptop2 + * Pseudo action: all_stopped ++ * Pseudo action: W-master_start_0 ++ * Pseudo action: X-master_start_0 + * Resource action: 18builder start on dvossel-laptop2 + * Resource action: FAKE start on 18builder + * Resource action: W start on 18builder +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.dot b/pengine/test10/whitebox-imply-stop-on-fence.dot +index 8ef42fd..029e9f1 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.dot ++++ b/pengine/test10/whitebox-imply-stop-on-fence.dot +@@ -23,6 +23,10 @@ + "clvmd-clone_stop_0" [ style=bold color="green" fontcolor="orange"] + "clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] + "clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd_monitor_0 lxc-01_kiff-02" -> "clvmd_stop_0 kiff-01" [ style = bold] ++"clvmd_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] ++"clvmd_monitor_0 lxc-02_kiff-02" -> "clvmd_stop_0 kiff-01" [ style = bold] ++"clvmd_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] + "clvmd_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "clvmd_stop_0 kiff-01" -> "clvmd-clone_stopped_0" [ style = bold] + "clvmd_stop_0 kiff-01" -> "dlm_stop_0 kiff-01" [ style = bold] +@@ -31,6 +35,10 @@ + "dlm-clone_stop_0" -> "dlm_stop_0 kiff-01" [ style = bold] + "dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"] + "dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"dlm_monitor_0 lxc-01_kiff-02" -> "dlm_stop_0 kiff-01" [ style = bold] ++"dlm_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] ++"dlm_monitor_0 lxc-02_kiff-02" -> "dlm_stop_0 kiff-01" [ style = bold] ++"dlm_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] + "dlm_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "dlm_stop_0 kiff-01" -> "dlm-clone_stopped_0" [ style = bold] + "dlm_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] +@@ -55,12 +63,17 @@ + "lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] + "lxc-02_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "lxc-02_kiff-01_stop_0 kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] ++"lxc-02_kiff-01_stop_0 kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] + "lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] + "shared0-clone_stop_0" -> "shared0-clone_stopped_0" [ style = bold] + "shared0-clone_stop_0" -> "shared0_stop_0 kiff-01" [ style = bold] + "shared0-clone_stop_0" [ style=bold color="green" fontcolor="orange"] + "shared0-clone_stopped_0" -> "clvmd-clone_stop_0" [ style = bold] + "shared0-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"shared0_monitor_0 lxc-01_kiff-02" -> "shared0_stop_0 kiff-01" [ style = bold] ++"shared0_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] ++"shared0_monitor_0 lxc-02_kiff-02" -> "shared0_stop_0 kiff-01" [ style = bold] ++"shared0_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] + "shared0_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "shared0_stop_0 kiff-01" -> "clvmd_stop_0 kiff-01" [ style = bold] + "shared0_stop_0 kiff-01" -> "shared0-clone_stopped_0" [ style = bold] +@@ -91,6 +104,12 @@ + "stonith_complete" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] + "stonith_complete" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] ++"vm-fs_monitor_0 lxc-01_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] ++"vm-fs_monitor_0 lxc-01_kiff-02" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] ++"vm-fs_monitor_0 lxc-01_kiff-02" [ style=bold color="green" fontcolor="black"] ++"vm-fs_monitor_0 lxc-02_kiff-02" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] ++"vm-fs_monitor_0 lxc-02_kiff-02" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] ++"vm-fs_monitor_0 lxc-02_kiff-02" [ style=bold color="green" fontcolor="black"] + "vm-fs_monitor_20000 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] + "vm-fs_start_0 lxc-01_kiff-01" -> "vm-fs_monitor_20000 lxc-01_kiff-01" [ style = bold] + "vm-fs_start_0 lxc-01_kiff-01" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp +index d1e22ee..b5700c0 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.exp ++++ b/pengine/test10/whitebox-imply-stop-on-fence.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -24,13 +24,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -38,7 +38,7 @@ + + + +- ++ + + + +@@ -47,31 +47,55 @@ + + + +- ++ + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -80,13 +104,13 @@ + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -95,31 +119,55 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ + + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -128,13 +176,13 @@ + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -143,28 +191,52 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -173,45 +245,45 @@ + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -220,42 +292,42 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -264,95 +336,128 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -362,41 +467,41 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -406,9 +511,9 @@ + + + +- ++ + +- ++ + + + +@@ -421,9 +526,9 @@ + + + +- ++ + +- ++ + + + +@@ -436,9 +541,9 @@ + + + +- ++ + +- ++ + + + +@@ -447,14 +552,14 @@ + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -462,38 +567,38 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary +index 87d7b4f..d272b25 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.summary ++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary +@@ -40,6 +40,14 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: fence-kiff-02_stop_0 ++ * Resource action: dlm monitor on lxc-02_kiff-02 ++ * Resource action: dlm monitor on lxc-01_kiff-02 ++ * Resource action: clvmd monitor on lxc-02_kiff-02 ++ * Resource action: clvmd monitor on lxc-01_kiff-02 ++ * Resource action: shared0 monitor on lxc-02_kiff-02 ++ * Resource action: shared0 monitor on lxc-01_kiff-02 ++ * Resource action: vm-fs monitor on lxc-02_kiff-02 ++ * Resource action: vm-fs monitor on lxc-01_kiff-02 + * Fencing kiff-01 (reboot) + * Pseudo action: lxc-01_kiff-01_stop_0 + * Pseudo action: lxc-02_kiff-01_stop_0 +diff --git a/pengine/test10/whitebox-migrate1.dot b/pengine/test10/whitebox-migrate1.dot +index e54df2c..53d37e0 100644 +--- a/pengine/test10/whitebox-migrate1.dot ++++ b/pengine/test10/whitebox-migrate1.dot +@@ -32,6 +32,7 @@ + "rhel7-node1_monitor_30000 rhel7-node3" [ style=bold color="green" fontcolor="black"] + "rhel7-node1_start_0 rhel7-node3" -> "rhel7-node1_monitor_30000 rhel7-node3" [ style = bold] + "rhel7-node1_start_0 rhel7-node3" [ style=bold color="green" fontcolor="orange"] ++"rhel7-node1_stop_0 rhel7-node2" -> "FAKE3_start_0 rhel7-node2" [ style = dashed] + "rhel7-node1_stop_0 rhel7-node2" -> "all_stopped" [ style = bold] + "rhel7-node1_stop_0 rhel7-node2" -> "remote-rsc_migrate_to_0 rhel7-node2" [ style = dashed] + "rhel7-node1_stop_0 rhel7-node2" -> "remote-rsc_stop_0 rhel7-node2" [ style = bold] +diff --git a/pengine/test10/whitebox-move.dot b/pengine/test10/whitebox-move.dot +index f47c95b..ed06b7c 100644 +--- a/pengine/test10/whitebox-move.dot ++++ b/pengine/test10/whitebox-move.dot +@@ -1,4 +1,7 @@ + digraph "g" { ++"A_monitor_0 lxc2" -> "A_start_0 lxc1" [ style = bold] ++"A_monitor_0 lxc2" -> "A_stop_0 lxc1" [ style = bold] ++"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "A_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] + "A_start_0 lxc1" -> "A_monitor_10000 lxc1" [ style = bold] + "A_start_0 lxc1" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/whitebox-move.exp b/pengine/test10/whitebox-move.exp +index 4461890..6f00275 100644 +--- a/pengine/test10/whitebox-move.exp ++++ b/pengine/test10/whitebox-move.exp +@@ -1,62 +1,62 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -69,31 +69,31 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -101,61 +101,77 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ ++ ++ ++ + + + + + +- ++ + + + + +- ++ ++ ++ ++ ++ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -163,45 +179,45 @@ + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -211,14 +227,14 @@ + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -226,16 +242,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-move.summary b/pengine/test10/whitebox-move.summary +index 5e27a67..23bc5de 100644 +--- a/pengine/test10/whitebox-move.summary ++++ b/pengine/test10/whitebox-move.summary +@@ -18,10 +18,11 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: M-clone_stop_0 +- * Resource action: A stop on lxc1 ++ * Resource action: A monitor on lxc2 + * Resource action: M stop on lxc1 + * Pseudo action: M-clone_stopped_0 + * Pseudo action: M-clone_start_0 ++ * Resource action: A stop on lxc1 + * Resource action: lxc1 stop on 18node1 + * Resource action: container1 stop on 18node1 + * Pseudo action: all_stopped +diff --git a/pengine/test10/whitebox-ms-ordering-move.dot b/pengine/test10/whitebox-ms-ordering-move.dot +index 43c1d4f..e6b5406 100644 +--- a/pengine/test10/whitebox-ms-ordering-move.dot ++++ b/pengine/test10/whitebox-ms-ordering-move.dot +@@ -7,6 +7,7 @@ digraph "g" { + "container1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "container1_stop_0 rhel7-1" -> "container1_start_0 rhel7-2" [ style = bold] + "container1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="black"] ++"lsb-dummy_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] + "lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc1" [ style = bold] + "lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"] +@@ -68,4 +69,15 @@ digraph "g" { + "lxc2_monitor_0 rhel7-3" [ style=bold color="green" fontcolor="black"] + "lxc2_monitor_0 rhel7-4" [ style=bold color="green" fontcolor="black"] + "lxc2_monitor_0 rhel7-5" [ style=bold color="green" fontcolor="black"] ++"migrator_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"petulant_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"ping-1_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"r192.168.122.207_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"r192.168.122.208_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"rsc_rhel7-1_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"rsc_rhel7-2_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"rsc_rhel7-3_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"rsc_rhel7-4_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"rsc_rhel7-5_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"stateful-1_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + } +diff --git a/pengine/test10/whitebox-ms-ordering-move.exp b/pengine/test10/whitebox-ms-ordering-move.exp +index dc5e473..a8ffa64 100644 +--- a/pengine/test10/whitebox-ms-ordering-move.exp ++++ b/pengine/test10/whitebox-ms-ordering-move.exp +@@ -1,260 +1,368 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -264,98 +372,98 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +@@ -363,13 +471,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-ms-ordering-move.summary b/pengine/test10/whitebox-ms-ordering-move.summary +index 80156b0..af86d74 100644 +--- a/pengine/test10/whitebox-ms-ordering-move.summary ++++ b/pengine/test10/whitebox-ms-ordering-move.summary +@@ -35,6 +35,18 @@ Transition Summary: + * Move lxc1 ( rhel7-1 -> rhel7-2 ) + + Executing cluster transition: ++ * Resource action: rsc_rhel7-1 monitor on lxc2 ++ * Resource action: rsc_rhel7-2 monitor on lxc2 ++ * Resource action: rsc_rhel7-3 monitor on lxc2 ++ * Resource action: rsc_rhel7-4 monitor on lxc2 ++ * Resource action: rsc_rhel7-5 monitor on lxc2 ++ * Resource action: migrator monitor on lxc2 ++ * Resource action: ping-1 monitor on lxc2 ++ * Resource action: stateful-1 monitor on lxc2 ++ * Resource action: r192.168.122.207 monitor on lxc2 ++ * Resource action: petulant monitor on lxc2 ++ * Resource action: r192.168.122.208 monitor on lxc2 ++ * Resource action: lsb-dummy monitor on lxc2 + * Pseudo action: lxc-ms-master_demote_0 + * Resource action: lxc1 monitor on rhel7-5 + * Resource action: lxc1 monitor on rhel7-4 +diff --git a/pengine/test10/whitebox-nested-group.dot b/pengine/test10/whitebox-nested-group.dot +index 9e1abce..e5749ec 100644 +--- a/pengine/test10/whitebox-nested-group.dot ++++ b/pengine/test10/whitebox-nested-group.dot +@@ -1,9 +1,30 @@ + digraph "g" { + "c7auto4_monitor_0 c7auto1" -> "c7auto4_start_0 c7auto1" [ style = bold] ++"c7auto4_monitor_0 c7auto1" -> "fake1_start_0 c7auto3" [ style = bold] ++"c7auto4_monitor_0 c7auto1" -> "fake2_start_0 c7auto4" [ style = bold] ++"c7auto4_monitor_0 c7auto1" -> "fake3_start_0 c7auto2" [ style = bold] ++"c7auto4_monitor_0 c7auto1" -> "fake4_start_0 c7auto3" [ style = bold] ++"c7auto4_monitor_0 c7auto1" -> "fake5_start_0 c7auto4" [ style = bold] ++"c7auto4_monitor_0 c7auto1" -> "fake_clone_start_0" [ style = bold] ++"c7auto4_monitor_0 c7auto1" -> "fake_group_start_0" [ style = bold] + "c7auto4_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] + "c7auto4_monitor_0 c7auto2" -> "c7auto4_start_0 c7auto1" [ style = bold] ++"c7auto4_monitor_0 c7auto2" -> "fake1_start_0 c7auto3" [ style = bold] ++"c7auto4_monitor_0 c7auto2" -> "fake2_start_0 c7auto4" [ style = bold] ++"c7auto4_monitor_0 c7auto2" -> "fake3_start_0 c7auto2" [ style = bold] ++"c7auto4_monitor_0 c7auto2" -> "fake4_start_0 c7auto3" [ style = bold] ++"c7auto4_monitor_0 c7auto2" -> "fake5_start_0 c7auto4" [ style = bold] ++"c7auto4_monitor_0 c7auto2" -> "fake_clone_start_0" [ style = bold] ++"c7auto4_monitor_0 c7auto2" -> "fake_group_start_0" [ style = bold] + "c7auto4_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] + "c7auto4_monitor_0 c7auto3" -> "c7auto4_start_0 c7auto1" [ style = bold] ++"c7auto4_monitor_0 c7auto3" -> "fake1_start_0 c7auto3" [ style = bold] ++"c7auto4_monitor_0 c7auto3" -> "fake2_start_0 c7auto4" [ style = bold] ++"c7auto4_monitor_0 c7auto3" -> "fake3_start_0 c7auto2" [ style = bold] ++"c7auto4_monitor_0 c7auto3" -> "fake4_start_0 c7auto3" [ style = bold] ++"c7auto4_monitor_0 c7auto3" -> "fake5_start_0 c7auto4" [ style = bold] ++"c7auto4_monitor_0 c7auto3" -> "fake_clone_start_0" [ style = bold] ++"c7auto4_monitor_0 c7auto3" -> "fake_group_start_0" [ style = bold] + "c7auto4_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] + "c7auto4_monitor_30000 c7auto1" [ style=bold color="green" fontcolor="black"] + "c7auto4_start_0 c7auto1" -> "c7auto4_monitor_30000 c7auto1" [ style = bold] +diff --git a/pengine/test10/whitebox-nested-group.exp b/pengine/test10/whitebox-nested-group.exp +index 388c4d4..5625c92 100644 +--- a/pengine/test10/whitebox-nested-group.exp ++++ b/pengine/test10/whitebox-nested-group.exp +@@ -24,11 +24,20 @@ + + + ++ ++ ++ + + + ++ ++ ++ + + ++ ++ ++ + + + +@@ -86,12 +95,21 @@ + + + ++ ++ ++ + + + ++ ++ ++ + + + ++ ++ ++ + + + +@@ -151,11 +169,20 @@ + + + ++ ++ ++ + + + ++ ++ ++ + + ++ ++ ++ + + + +@@ -210,11 +237,20 @@ + + + ++ ++ ++ + + + ++ ++ ++ + + ++ ++ ++ + + + +@@ -272,12 +308,21 @@ + + + ++ ++ ++ + + + ++ ++ ++ + + + ++ ++ ++ + + + +@@ -487,11 +532,20 @@ + + + ++ ++ ++ + + + ++ ++ ++ + + ++ ++ ++ + + + +@@ -518,7 +572,17 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/whitebox-nested-group.summary b/pengine/test10/whitebox-nested-group.summary +index ca9c47f..b03b357 100644 +--- a/pengine/test10/whitebox-nested-group.summary ++++ b/pengine/test10/whitebox-nested-group.summary +@@ -47,8 +47,6 @@ Executing cluster transition: + * Resource action: fake:0 monitor on c7auto2 + * Resource action: fake:1 monitor on c7auto3 + * Resource action: fake:3 monitor on c7auto1 +- * Pseudo action: fake_clone_start_0 +- * Pseudo action: fake_group_start_0 + * Resource action: fake_fs monitor on c7auto3 + * Resource action: fake_fs monitor on c7auto2 + * Resource action: fake_fs monitor on c7auto1 +@@ -58,9 +56,8 @@ Executing cluster transition: + * Resource action: fake1 start on c7auto3 + * Resource action: fake3 start on c7auto2 + * Resource action: fake4 start on c7auto3 +- * Resource action: fake:0 start on c7auto2 +- * Resource action: fake:1 start on c7auto3 +- * Resource action: fake:3 start on c7auto1 ++ * Pseudo action: fake_clone_start_0 ++ * Pseudo action: fake_group_start_0 + * Resource action: fake_fs start on c7auto1 + * Resource action: container start on c7auto1 + * Resource action: c7auto4 start on c7auto1 +@@ -69,10 +66,10 @@ Executing cluster transition: + * Resource action: fake3 monitor=10000 on c7auto2 + * Resource action: fake4 monitor=10000 on c7auto3 + * Resource action: fake5 start on c7auto4 +- * Resource action: fake:0 monitor=10000 on c7auto2 +- * Resource action: fake:1 monitor=10000 on c7auto3 ++ * Resource action: fake:0 start on c7auto2 ++ * Resource action: fake:1 start on c7auto3 + * Resource action: fake:2 start on c7auto4 +- * Resource action: fake:3 monitor=10000 on c7auto1 ++ * Resource action: fake:3 start on c7auto1 + * Pseudo action: fake_clone_running_0 + * Pseudo action: fake_group_running_0 + * Resource action: fake_fs monitor=10000 on c7auto1 +@@ -80,7 +77,10 @@ Executing cluster transition: + * Resource action: c7auto4 monitor=30000 on c7auto1 + * Resource action: fake2 monitor=10000 on c7auto4 + * Resource action: fake5 monitor=10000 on c7auto4 ++ * Resource action: fake:0 monitor=10000 on c7auto2 ++ * Resource action: fake:1 monitor=10000 on c7auto3 + * Resource action: fake:2 monitor=10000 on c7auto4 ++ * Resource action: fake:3 monitor=10000 on c7auto1 + + Revised cluster status: + Online: [ c7auto1 c7auto2 c7auto3 ] +diff --git a/pengine/test10/whitebox-orphan-ms.dot b/pengine/test10/whitebox-orphan-ms.dot +index 46b6cda..4e2e211 100644 +--- a/pengine/test10/whitebox-orphan-ms.dot ++++ b/pengine/test10/whitebox-orphan-ms.dot +@@ -25,6 +25,8 @@ + "container2_stop_0 18node1" -> "container2_delete_0 18node2" [ style = bold] + "container2_stop_0 18node1" -> "container2_delete_0 18node3" [ style = bold] + "container2_stop_0 18node1" [ style=bold color="green" fontcolor="black"] ++"lsb-dummy_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"lsb-dummy_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] + "lxc-ms_clear_failcount_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = dashed] + "lxc-ms_clear_failcount_0 lxc1" -> "lxc-ms_stop_0 lxc2" [ style = dashed] + "lxc-ms_clear_failcount_0 lxc1" [ style=dashed color="red" fontcolor="black"] +@@ -69,4 +71,20 @@ + "lxc2_stop_0 18node1" -> "lxc2_delete_0 18node2" [ style = bold] + "lxc2_stop_0 18node1" -> "lxc2_delete_0 18node3" [ style = bold] + "lxc2_stop_0 18node1" [ style=bold color="green" fontcolor="black"] ++"migrator_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"migrator_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] ++"ping-1_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"ping-1_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] ++"r192.168.122.87_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"r192.168.122.87_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] ++"r192.168.122.88_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"r192.168.122.88_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] ++"r192.168.122.89_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"r192.168.122.89_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] ++"rsc_18node1_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"rsc_18node1_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] ++"rsc_18node2_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"rsc_18node2_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] ++"rsc_18node3_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"rsc_18node3_monitor_0 lxc2" [ style=dashed color="red" fontcolor="black"] + } +diff --git a/pengine/test10/whitebox-orphan-ms.exp b/pengine/test10/whitebox-orphan-ms.exp +index ef81317..564e92d 100644 +--- a/pengine/test10/whitebox-orphan-ms.exp ++++ b/pengine/test10/whitebox-orphan-ms.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -23,14 +23,14 @@ + + + +- ++ + + + + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -69,13 +69,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -85,10 +85,10 @@ + + + +- ++ + + +- ++ + + + +@@ -101,7 +101,7 @@ + + + +- ++ + + + +@@ -114,7 +114,7 @@ + + + +- ++ + + + +@@ -127,45 +127,45 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -174,7 +174,7 @@ + + + +- ++ + + + +@@ -190,10 +190,10 @@ + + + +- ++ + + +- ++ + + + +@@ -206,10 +206,10 @@ + + + +- ++ + + +- ++ + + + +@@ -222,16 +222,16 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -241,10 +241,10 @@ + + + +- ++ + + +- ++ + + + +@@ -257,7 +257,7 @@ + + + +- ++ + + + +@@ -270,7 +270,7 @@ + + + +- ++ + + + +@@ -283,20 +283,20 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -309,7 +309,7 @@ + + + +- ++ + + + +@@ -322,7 +322,7 @@ + + + +- ++ + + + +@@ -335,7 +335,7 @@ + + + +- ++ + + + +@@ -347,25 +347,25 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-orphaned.dot b/pengine/test10/whitebox-orphaned.dot +index e50252d..4ef1f7e 100644 +--- a/pengine/test10/whitebox-orphaned.dot ++++ b/pengine/test10/whitebox-orphaned.dot +@@ -1,4 +1,9 @@ + digraph "g" { ++"A_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"B_monitor_0 lxc2" -> "B_start_0 lxc2" [ style = bold] ++"B_monitor_0 lxc2" -> "B_stop_0 lxc1" [ style = bold] ++"B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "B_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] + "B_start_0 lxc2" -> "B_monitor_10000 lxc2" [ style = bold] + "B_start_0 lxc2" [ style=bold color="green" fontcolor="black"] +@@ -6,6 +11,9 @@ + "B_stop_0 lxc1" -> "all_stopped" [ style = bold] + "B_stop_0 lxc1" -> "lxc1_stop_0 18node2" [ style = bold] + "B_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] ++"C_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"D_monitor_0 lxc1" [ style=dashed color="red" fontcolor="black"] ++"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "M-clone_stop_0" -> "M-clone_stopped_0" [ style = bold] + "M-clone_stop_0" -> "M_stop_0 lxc1" [ style = bold] + "M-clone_stop_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/whitebox-orphaned.exp b/pengine/test10/whitebox-orphaned.exp +index 334e594..843f186 100644 +--- a/pengine/test10/whitebox-orphaned.exp ++++ b/pengine/test10/whitebox-orphaned.exp +@@ -1,35 +1,35 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -37,65 +37,99 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +@@ -104,11 +138,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -117,11 +151,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -130,13 +164,13 @@ + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -146,26 +180,26 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +@@ -174,11 +208,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -187,11 +221,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -200,11 +234,11 @@ + + + +- ++ + + + +- ++ + + + +@@ -212,16 +246,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-orphaned.summary b/pengine/test10/whitebox-orphaned.summary +index 7d173b2..e0928b1 100644 +--- a/pengine/test10/whitebox-orphaned.summary ++++ b/pengine/test10/whitebox-orphaned.summary +@@ -23,22 +23,25 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: M-clone_stop_0 +- * Resource action: B stop on lxc1 ++ * Resource action: A monitor on lxc2 ++ * Resource action: B monitor on lxc2 ++ * Resource action: D monitor on lxc2 + * Cluster action: clear_failcount for container1 on 18node2 + * Cluster action: clear_failcount for lxc1 on 18node2 + * Resource action: M stop on lxc1 + * Pseudo action: M-clone_stopped_0 +- * Resource action: B start on lxc2 ++ * Resource action: B stop on lxc1 + * Resource action: lxc1 stop on 18node2 + * Resource action: lxc1 delete on 18node3 + * Resource action: lxc1 delete on 18node2 + * Resource action: lxc1 delete on 18node1 +- * Resource action: B monitor=10000 on lxc2 ++ * Resource action: B start on lxc2 + * Resource action: container1 stop on 18node2 + * Resource action: container1 delete on 18node3 + * Resource action: container1 delete on 18node2 + * Resource action: container1 delete on 18node1 + * Pseudo action: all_stopped ++ * Resource action: B monitor=10000 on lxc2 + + Revised cluster status: + Online: [ 18node1 18node2 18node3 ] +diff --git a/pengine/test10/whitebox-start.dot b/pengine/test10/whitebox-start.dot +index 8b4dbcd..28f747a 100644 +--- a/pengine/test10/whitebox-start.dot ++++ b/pengine/test10/whitebox-start.dot +@@ -1,4 +1,7 @@ + digraph "g" { ++"A_monitor_0 lxc2" -> "A_start_0 lxc1" [ style = bold] ++"A_monitor_0 lxc2" -> "A_stop_0 18node1" [ style = bold] ++"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "A_monitor_10000 lxc1" [ style=bold color="green" fontcolor="black"] + "A_start_0 lxc1" -> "A_monitor_10000 lxc1" [ style = bold] + "A_start_0 lxc1" [ style=bold color="green" fontcolor="black"] +@@ -11,6 +14,7 @@ digraph "g" { + "B_stop_0 lxc2" -> "B_start_0 18node3" [ style = bold] + "B_stop_0 lxc2" -> "all_stopped" [ style = bold] + "B_stop_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "M-clone_running_0" [ style=bold color="green" fontcolor="orange"] + "M-clone_start_0" -> "M-clone_running_0" [ style = bold] + "M-clone_start_0" -> "M_start_0 lxc1" [ style = bold] +diff --git a/pengine/test10/whitebox-start.exp b/pengine/test10/whitebox-start.exp +index ccccf60..360b8a7 100644 +--- a/pengine/test10/whitebox-start.exp ++++ b/pengine/test10/whitebox-start.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -10,57 +10,57 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -68,110 +68,135 @@ + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ ++ ++ ++ + + + + + +- ++ + + + + +- ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + + + +@@ -179,10 +204,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-start.summary b/pengine/test10/whitebox-start.summary +index e76e28a..01a1b74 100644 +--- a/pengine/test10/whitebox-start.summary ++++ b/pengine/test10/whitebox-start.summary +@@ -24,18 +24,20 @@ Transition Summary: + Executing cluster transition: + * Resource action: container1 start on 18node1 + * Pseudo action: M-clone_start_0 +- * Resource action: A stop on 18node1 ++ * Resource action: A monitor on lxc2 + * Resource action: B stop on lxc2 ++ * Resource action: D monitor on lxc2 + * Resource action: lxc1 start on 18node1 +- * Pseudo action: all_stopped + * Resource action: M start on lxc1 + * Pseudo action: M-clone_running_0 +- * Resource action: A start on lxc1 ++ * Resource action: A stop on 18node1 + * Resource action: B start on 18node3 + * Resource action: lxc1 monitor=30000 on 18node1 ++ * Pseudo action: all_stopped + * Resource action: M monitor=10000 on lxc1 +- * Resource action: A monitor=10000 on lxc1 ++ * Resource action: A start on lxc1 + * Resource action: B monitor=10000 on 18node3 ++ * Resource action: A monitor=10000 on lxc1 + + Revised cluster status: + Online: [ 18node1 18node2 18node3 ] +diff --git a/pengine/test10/whitebox-stop.dot b/pengine/test10/whitebox-stop.dot +index 9900483..0ecdcba 100644 +--- a/pengine/test10/whitebox-stop.dot ++++ b/pengine/test10/whitebox-stop.dot +@@ -1,4 +1,8 @@ + digraph "g" { ++"A_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"B_monitor_0 lxc2" -> "B_start_0 lxc2" [ style = bold] ++"B_monitor_0 lxc2" -> "B_stop_0 lxc1" [ style = bold] ++"B_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "B_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] + "B_start_0 lxc2" -> "B_monitor_10000 lxc2" [ style = bold] + "B_start_0 lxc2" [ style=bold color="green" fontcolor="black"] +@@ -6,6 +10,7 @@ digraph "g" { + "B_stop_0 lxc1" -> "all_stopped" [ style = bold] + "B_stop_0 lxc1" -> "lxc1_stop_0 18node2" [ style = bold] + "B_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] ++"D_monitor_0 lxc2" [ style=bold color="green" fontcolor="black"] + "M-clone_stop_0" -> "M-clone_stopped_0" [ style = bold] + "M-clone_stop_0" -> "M_stop_0 lxc1" [ style = bold] + "M-clone_stop_0" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/whitebox-stop.exp b/pengine/test10/whitebox-stop.exp +index 354b6d4..84a5288 100644 +--- a/pengine/test10/whitebox-stop.exp ++++ b/pengine/test10/whitebox-stop.exp +@@ -1,48 +1,48 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -50,42 +50,76 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -95,14 +129,14 @@ + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -110,16 +144,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-stop.summary b/pengine/test10/whitebox-stop.summary +index 9b15ea0..44e570f 100644 +--- a/pengine/test10/whitebox-stop.summary ++++ b/pengine/test10/whitebox-stop.summary +@@ -22,14 +22,17 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: M-clone_stop_0 +- * Resource action: B stop on lxc1 ++ * Resource action: A monitor on lxc2 ++ * Resource action: B monitor on lxc2 ++ * Resource action: D monitor on lxc2 + * Resource action: M stop on lxc1 + * Pseudo action: M-clone_stopped_0 +- * Resource action: B start on lxc2 ++ * Resource action: B stop on lxc1 + * Resource action: lxc1 stop on 18node2 + * Resource action: container1 stop on 18node2 +- * Resource action: B monitor=10000 on lxc2 ++ * Resource action: B start on lxc2 + * Pseudo action: all_stopped ++ * Resource action: B monitor=10000 on lxc2 + + Revised cluster status: + Online: [ 18node1 18node2 18node3 ] +diff --git a/pengine/test10/whitebox-unexpectedly-running.dot b/pengine/test10/whitebox-unexpectedly-running.dot +index 0e2e5bb..fa1171e 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.dot ++++ b/pengine/test10/whitebox-unexpectedly-running.dot +@@ -1,20 +1,26 @@ + digraph "g" { ++"FAKE-crashed_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"] ++"FAKE-crashed_start_0 18builder" -> "FAKE-crashed_monitor_60000 18builder" [ style = bold] ++"FAKE-crashed_start_0 18builder" -> "remote2_start_0 18builder" [ style = bold] ++"FAKE-crashed_start_0 18builder" [ style=bold color="green" fontcolor="black"] ++"FAKE-crashed_stop_0 18builder" -> "FAKE-crashed_start_0 18builder" [ style = bold] ++"FAKE-crashed_stop_0 18builder" -> "all_stopped" [ style = bold] ++"FAKE-crashed_stop_0 18builder" -> "stonith 'reboot' remote2" [ style = bold] ++"FAKE-crashed_stop_0 18builder" [ style=bold color="green" fontcolor="black"] + "FAKE_monitor_60000 18builder" [ style=bold color="green" fontcolor="black"] +-"FAKE_start_0 18builder" -> "FAKE_monitor_60000 18builder" [ style = bold] +-"FAKE_start_0 18builder" -> "remote1_start_0 18builder" [ style = bold] +-"FAKE_start_0 18builder" [ style=bold color="green" fontcolor="black"] +-"FAKE_stop_0 18builder" -> "FAKE_start_0 18builder" [ style = bold] +-"FAKE_stop_0 18builder" -> "all_stopped" [ style = bold] +-"FAKE_stop_0 18builder" -> "stonith 'reboot' remote1" [ style = bold] +-"FAKE_stop_0 18builder" [ style=bold color="green" fontcolor="black"] + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "remote1_monitor_0 18builder" -> "remote1_start_0 18builder" [ style = bold] + "remote1_monitor_0 18builder" [ style=bold color="green" fontcolor="black"] + "remote1_monitor_30000 18builder" [ style=bold color="green" fontcolor="black"] + "remote1_start_0 18builder" -> "remote1_monitor_30000 18builder" [ style = bold] + "remote1_start_0 18builder" [ style=bold color="green" fontcolor="black"] +-"stonith 'reboot' remote1" -> "stonith_complete" [ style = bold] +-"stonith 'reboot' remote1" [ style=bold color="green" fontcolor="orange"] ++"remote2_monitor_0 18builder" -> "remote2_start_0 18builder" [ style = bold] ++"remote2_monitor_0 18builder" [ style=bold color="green" fontcolor="black"] ++"remote2_monitor_30000 18builder" [ style=bold color="green" fontcolor="black"] ++"remote2_start_0 18builder" -> "remote2_monitor_30000 18builder" [ style = bold] ++"remote2_start_0 18builder" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' remote2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' remote2" [ style=bold color="green" fontcolor="orange"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/whitebox-unexpectedly-running.exp b/pengine/test10/whitebox-unexpectedly-running.exp +index a0e5cae..c4e13b9 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.exp ++++ b/pengine/test10/whitebox-unexpectedly-running.exp +@@ -1,116 +1,160 @@ + + + +- ++ + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + + + +- ++ + +- +- +- ++ ++ ++ + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- +- +- +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- +- ++ ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-unexpectedly-running.scores b/pengine/test10/whitebox-unexpectedly-running.scores +index 7f8a1d9..45e1d39 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.scores ++++ b/pengine/test10/whitebox-unexpectedly-running.scores +@@ -1,5 +1,13 @@ + Allocation scores: + native_color: FAKE allocation score on 18builder: 0 + native_color: FAKE allocation score on remote1: -INFINITY ++native_color: FAKE allocation score on remote2: -INFINITY ++native_color: FAKE-crashed allocation score on 18builder: 0 ++native_color: FAKE-crashed allocation score on remote1: -INFINITY ++native_color: FAKE-crashed allocation score on remote2: -INFINITY + native_color: remote1 allocation score on 18builder: 0 + native_color: remote1 allocation score on remote1: -INFINITY ++native_color: remote1 allocation score on remote2: -INFINITY ++native_color: remote2 allocation score on 18builder: 0 ++native_color: remote2 allocation score on remote1: -INFINITY ++native_color: remote2 allocation score on remote2: -INFINITY +diff --git a/pengine/test10/whitebox-unexpectedly-running.summary b/pengine/test10/whitebox-unexpectedly-running.summary +index eef4f63..3e0c898 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.summary ++++ b/pengine/test10/whitebox-unexpectedly-running.summary +@@ -2,27 +2,34 @@ + Current cluster status: + Online: [ 18builder ] + +- FAKE (ocf::pacemaker:Dummy): FAILED 18builder ++ FAKE (ocf::pacemaker:Dummy): Started 18builder ++ FAKE-crashed (ocf::pacemaker:Dummy): FAILED 18builder + + Transition Summary: +- * Fence (reboot) remote1 (resource: FAKE) 'guest is unclean' +- * Recover FAKE ( 18builder ) ++ * Fence (reboot) remote2 (resource: FAKE-crashed) 'guest is unclean' ++ * Recover FAKE-crashed ( 18builder ) + * Start remote1 (18builder) ++ * Start remote2 ( 18builder ) + + Executing cluster transition: +- * Resource action: FAKE stop on 18builder ++ * Resource action: FAKE monitor=60000 on 18builder ++ * Resource action: FAKE-crashed stop on 18builder + * Resource action: remote1 monitor on 18builder +- * Pseudo action: stonith-remote1-reboot on remote1 ++ * Resource action: remote2 monitor on 18builder ++ * Pseudo action: stonith-remote2-reboot on remote2 + * Pseudo action: stonith_complete + * Pseudo action: all_stopped +- * Resource action: FAKE start on 18builder ++ * Resource action: FAKE-crashed start on 18builder + * Resource action: remote1 start on 18builder +- * Resource action: FAKE monitor=60000 on 18builder ++ * Resource action: remote2 start on 18builder ++ * Resource action: FAKE-crashed monitor=60000 on 18builder + * Resource action: remote1 monitor=30000 on 18builder ++ * Resource action: remote2 monitor=30000 on 18builder + + Revised cluster status: + Online: [ 18builder ] +-Containers: [ remote1:FAKE ] ++Containers: [ remote1:FAKE remote2:FAKE-crashed ] + + FAKE (ocf::pacemaker:Dummy): Started 18builder ++ FAKE-crashed (ocf::pacemaker:Dummy): Started 18builder + +diff --git a/pengine/test10/whitebox-unexpectedly-running.xml b/pengine/test10/whitebox-unexpectedly-running.xml +index 4ec20c4..638ed1a 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.xml ++++ b/pengine/test10/whitebox-unexpectedly-running.xml +@@ -21,6 +21,14 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -36,6 +44,9 @@ + + + ++ ++ ++ + + + +-- +1.8.3.1 + + +From 62b9b988123589d590b30c17b0f4cee269cdad70 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 26 Sep 2017 20:44:34 +1000 +Subject: [PATCH 15/21] Fix: PE: Ensure the bundle nodes get set with the + correct discovery mode + +--- + lib/pengine/container.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/lib/pengine/container.c b/lib/pengine/container.c +index a550ff3..1d658e8 100644 +--- a/lib/pengine/container.c ++++ b/lib/pengine/container.c +@@ -462,6 +462,9 @@ create_remote_resource( + tuple->node->weight = 500; + tuple->node->rsc_discover_mode = discover_exclusive; + ++ /* Ensure the node shows up as allowed and with the correct discovery set */ ++ g_hash_table_insert(tuple->child->allowed_nodes, (gpointer) tuple->node->details->id, node_copy(tuple->node)); ++ + if (common_unpack(xml_remote, &tuple->remote, parent, data_set) == FALSE) { + return FALSE; + } +-- +1.8.3.1 + + +From 37b36a1c02b73711c82e90231b00d56e51179a90 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 26 Sep 2017 20:46:00 +1000 +Subject: [PATCH 16/21] Fix: PE: Make assumptions about the state of + conatinerized resources based ont he container state, not the connection + resource + +--- + pengine/native.c | 10 +++++++++- + 1 file changed, 9 insertions(+), 1 deletion(-) + +diff --git a/pengine/native.c b/pengine/native.c +index 2e40a4c..06a5c19 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -2847,10 +2847,12 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + if (rsc->exclusive_discover || top->exclusive_discover) { + if (allowed == NULL) { + /* exclusive discover is enabled and this node is not in the allowed list. */ ++ pe_rsc_trace(rsc, "Skipping probe for %s on node %s, A", rsc->id, node->details->id); + return FALSE; + } else if (allowed->rsc_discover_mode != discover_exclusive) { + /* exclusive discover is enabled and this node is not marked + * as a node this resource should be discovered on */ ++ pe_rsc_trace(rsc, "Skipping probe for %s on node %s, B", rsc->id, node->details->id); + return FALSE; + } + } +@@ -2861,16 +2863,18 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + * However it wasn't and the node has discovery disabled, so + * no need to probe for this resource. + */ ++ pe_rsc_trace(rsc, "Skipping probe for %s on node %s, C", rsc->id, node->details->id); + return FALSE; + } + + if (allowed && allowed->rsc_discover_mode == discover_never) { + /* this resource is marked as not needing to be discovered on this node */ ++ pe_rsc_trace(rsc, "Skipping probe for %s on node %s, discovery mode", rsc->id, node->details->id); + return FALSE; + } + + if(allowed != NULL && is_container_remote_node(allowed)) { +- resource_t *remote = allowed->details->remote_rsc; ++ resource_t *remote = allowed->details->remote_rsc->container; + + if(remote->role == RSC_ROLE_STOPPED) { + /* If the container is stopped, then we know anything that +@@ -2900,6 +2904,8 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + top, generate_op_key(top->id, RSC_START, 0), NULL, + pe_order_optional, data_set); + } ++ pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopped", ++ rsc->id, node->details->id, remote->id); + return FALSE; + + /* Here we really we want to check if remote->stop is required, +@@ -2919,6 +2925,8 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + custom_action_order(remote, generate_op_key(remote->id, RSC_STOP, 0), NULL, + top, generate_op_key(top->id, RSC_START, 0), NULL, + pe_order_optional, data_set); ++ pe_rsc_trace(rsc, "Skipping probe for %s on node %s, %s is stopping, restarting or moving", ++ rsc->id, node->details->id, remote->id); + return FALSE; + /* } else { + * The container is running so there is no problem probing it +-- +1.8.3.1 + + +From f3dcf9364b2bc7753c7683fb95959454da824afc Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 26 Sep 2017 21:19:17 +1000 +Subject: [PATCH 17/21] Test: PE: Make assumptions about the state of + conatinerized resources based ont he container state, not the connection + resource + +--- + pengine/test10/bundle-nested-colocation.dot | 9 + + pengine/test10/bundle-nested-colocation.exp | 252 +++++++++++-------- + pengine/test10/bundle-nested-colocation.scores | 6 +- + pengine/test10/bundle-nested-colocation.summary | 11 +- + pengine/test10/bundle-order-fencing.scores | 12 +- + pengine/test10/bundle-order-partial-start-2.dot | 3 + + pengine/test10/bundle-order-partial-start-2.exp | 272 +++++++++++---------- + pengine/test10/bundle-order-partial-start-2.scores | 6 +- + .../test10/bundle-order-partial-start-2.summary | 3 +- + pengine/test10/bundle-order-partial-start.dot | 1 + + pengine/test10/bundle-order-partial-start.exp | 3 + + pengine/test10/bundle-order-partial-start.scores | 6 +- + pengine/test10/bundle-order-partial-stop.scores | 4 +- + pengine/test10/bundle-order-startup-clone-2.dot | 18 ++ + pengine/test10/bundle-order-startup-clone-2.exp | 54 ++++ + pengine/test10/bundle-order-startup-clone-2.scores | 12 +- + pengine/test10/bundle-order-startup-clone.dot | 6 + + pengine/test10/bundle-order-startup-clone.exp | 9 + + pengine/test10/bundle-order-startup-clone.scores | 4 +- + pengine/test10/bundle-order-startup.dot | 3 + + pengine/test10/bundle-order-startup.exp | 17 +- + pengine/test10/bundle-order-startup.scores | 6 +- + pengine/test10/bundle-order-stop-clone.scores | 12 +- + pengine/test10/bundle-order-stop.scores | 4 +- + pengine/test10/guest-node-host-dies.dot | 4 +- + pengine/test10/guest-node-host-dies.exp | 4 +- + pengine/test10/remote-fence-unclean-3.scores | 12 +- + pengine/test10/whitebox-asymmetric.dot | 5 + + pengine/test10/whitebox-asymmetric.exp | 55 ++++- + pengine/test10/whitebox-asymmetric.summary | 4 +- + pengine/test10/whitebox-fail3.dot | 3 - + pengine/test10/whitebox-fail3.exp | 15 +- + pengine/test10/whitebox-fail3.summary | 4 +- + pengine/test10/whitebox-imply-stop-on-fence.dot | 2 +- + pengine/test10/whitebox-imply-stop-on-fence.exp | 6 +- + pengine/test10/whitebox-migrate1.dot | 2 +- + pengine/test10/whitebox-nested-group.dot | 21 -- + pengine/test10/whitebox-nested-group.exp | 66 +---- + pengine/test10/whitebox-nested-group.summary | 16 +- + 39 files changed, 536 insertions(+), 416 deletions(-) + +diff --git a/pengine/test10/bundle-nested-colocation.dot b/pengine/test10/bundle-nested-colocation.dot +index 2a2d71a..1a197c1 100644 +--- a/pengine/test10/bundle-nested-colocation.dot ++++ b/pengine/test10/bundle-nested-colocation.dot +@@ -2,16 +2,19 @@ digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "rabbitmq-bundle-0_monitor_60000 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq-bundle-0_monitor_60000 overcloud-controller-0" [ style = bold] ++"rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq:0_monitor_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-1_monitor_60000 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq-bundle-1_monitor_60000 overcloud-controller-1" [ style = bold] ++"rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq:1_monitor_0 rabbitmq-bundle-1" [ style = bold] + "rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style = bold] + "rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style = bold] + "rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-2_monitor_60000 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq-bundle-2_monitor_60000 overcloud-controller-2" [ style = bold] ++"rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq:2_monitor_0 rabbitmq-bundle-2" [ style = bold] + "rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq:2_monitor_10000 rabbitmq-bundle-2" [ style = bold] + "rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq:2_start_0 rabbitmq-bundle-2" [ style = bold] + "rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] +@@ -70,6 +73,8 @@ digraph "g" { + "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-1_start_0 overcloud-rabbit-1" [ style = bold] + "rabbitmq-bundle_start_0" -> "rabbitmq-bundle-docker-2_start_0 overcloud-rabbit-2" [ style = bold] + "rabbitmq-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq:0_monitor_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq:0_monitor_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] + "rabbitmq:0_post_notify_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] +@@ -77,6 +82,8 @@ digraph "g" { + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:1_monitor_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq:1_monitor_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] + "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] + "rabbitmq:1_post_notify_start_0 rabbitmq-bundle-1" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] + "rabbitmq:1_post_notify_start_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] +@@ -84,6 +91,8 @@ digraph "g" { + "rabbitmq:1_start_0 rabbitmq-bundle-1" -> "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style = bold] + "rabbitmq:1_start_0 rabbitmq-bundle-1" -> "rabbitmq:2_start_0 rabbitmq-bundle-2" [ style = bold] + "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq:2_monitor_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_start_0" [ style = bold] ++"rabbitmq:2_monitor_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] + "rabbitmq:2_monitor_10000 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] + "rabbitmq:2_post_notify_start_0 rabbitmq-bundle-2" -> "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style = bold] + "rabbitmq:2_post_notify_start_0 rabbitmq-bundle-2" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/bundle-nested-colocation.exp b/pengine/test10/bundle-nested-colocation.exp +index 916c44b..b91ced5 100644 +--- a/pengine/test10/bundle-nested-colocation.exp ++++ b/pengine/test10/bundle-nested-colocation.exp +@@ -1,39 +1,39 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -43,48 +43,61 @@ + + + +- ++ + + +- ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -94,51 +107,64 @@ + + + +- ++ ++ ++ ++ + + +- ++ + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -148,118 +174,140 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ ++ ++ ++ + + +- ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +@@ -268,14 +316,14 @@ + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -288,22 +336,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -314,16 +362,16 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +@@ -332,14 +380,14 @@ + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -352,22 +400,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -378,16 +426,16 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +@@ -396,14 +444,14 @@ + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -416,22 +464,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -442,9 +490,9 @@ + + + +- ++ + +- ++ + + + +@@ -459,19 +507,19 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +@@ -479,13 +527,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bundle-nested-colocation.scores b/pengine/test10/bundle-nested-colocation.scores +index c79e0ff..83d776d 100644 +--- a/pengine/test10/bundle-nested-colocation.scores ++++ b/pengine/test10/bundle-nested-colocation.scores +@@ -201,7 +201,7 @@ container_color: rabbitmq:0 allocation score on overcloud-galera-2: 0 + container_color: rabbitmq:0 allocation score on overcloud-rabbit-0: 0 + container_color: rabbitmq:0 allocation score on overcloud-rabbit-1: 0 + container_color: rabbitmq:0 allocation score on overcloud-rabbit-2: 0 +-container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: -INFINITY ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 500 + container_color: rabbitmq:0 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: rabbitmq:0 allocation score on rabbitmq-bundle-2: -INFINITY + container_color: rabbitmq:1 allocation score on overcloud-controller-0: 0 +@@ -214,7 +214,7 @@ container_color: rabbitmq:1 allocation score on overcloud-rabbit-0: 0 + container_color: rabbitmq:1 allocation score on overcloud-rabbit-1: 0 + container_color: rabbitmq:1 allocation score on overcloud-rabbit-2: 0 + container_color: rabbitmq:1 allocation score on rabbitmq-bundle-0: -INFINITY +-container_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: -INFINITY ++container_color: rabbitmq:1 allocation score on rabbitmq-bundle-1: 500 + container_color: rabbitmq:1 allocation score on rabbitmq-bundle-2: -INFINITY + container_color: rabbitmq:2 allocation score on overcloud-controller-0: 0 + container_color: rabbitmq:2 allocation score on overcloud-controller-1: 0 +@@ -227,7 +227,7 @@ container_color: rabbitmq:2 allocation score on overcloud-rabbit-1: 0 + container_color: rabbitmq:2 allocation score on overcloud-rabbit-2: 0 + container_color: rabbitmq:2 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: rabbitmq:2 allocation score on rabbitmq-bundle-1: -INFINITY +-container_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: -INFINITY ++container_color: rabbitmq:2 allocation score on rabbitmq-bundle-2: 500 + native_color: galera-bundle-docker-0 allocation score on overcloud-controller-0: -INFINITY + native_color: galera-bundle-docker-0 allocation score on overcloud-controller-1: -INFINITY + native_color: galera-bundle-docker-0 allocation score on overcloud-controller-2: -INFINITY +diff --git a/pengine/test10/bundle-nested-colocation.summary b/pengine/test10/bundle-nested-colocation.summary +index b2cec51..7c708d2 100644 +--- a/pengine/test10/bundle-nested-colocation.summary ++++ b/pengine/test10/bundle-nested-colocation.summary +@@ -35,7 +35,6 @@ Executing cluster transition: + * Pseudo action: rabbitmq-bundle_start_0 + * Pseudo action: all_stopped + * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 +- * Pseudo action: rabbitmq-bundle-clone_start_0 + * Resource action: rabbitmq-bundle-docker-0 start on overcloud-rabbit-0 + * Resource action: rabbitmq-bundle-docker-0 monitor=60000 on overcloud-rabbit-0 + * Resource action: rabbitmq-bundle-0 start on overcloud-controller-0 +@@ -45,13 +44,17 @@ Executing cluster transition: + * Resource action: rabbitmq-bundle-docker-2 start on overcloud-rabbit-2 + * Resource action: rabbitmq-bundle-docker-2 monitor=60000 on overcloud-rabbit-2 + * Resource action: rabbitmq-bundle-2 start on overcloud-controller-2 ++ * Resource action: rabbitmq:0 monitor on rabbitmq-bundle-0 ++ * Resource action: rabbitmq:1 monitor on rabbitmq-bundle-1 ++ * Resource action: rabbitmq:2 monitor on rabbitmq-bundle-2 ++ * Pseudo action: rabbitmq-bundle-clone_start_0 ++ * Resource action: rabbitmq-bundle-0 monitor=60000 on overcloud-controller-0 ++ * Resource action: rabbitmq-bundle-1 monitor=60000 on overcloud-controller-1 ++ * Resource action: rabbitmq-bundle-2 monitor=60000 on overcloud-controller-2 + * Resource action: rabbitmq:0 start on rabbitmq-bundle-0 + * Resource action: rabbitmq:1 start on rabbitmq-bundle-1 + * Resource action: rabbitmq:2 start on rabbitmq-bundle-2 + * Pseudo action: rabbitmq-bundle-clone_running_0 +- * Resource action: rabbitmq-bundle-0 monitor=60000 on overcloud-controller-0 +- * Resource action: rabbitmq-bundle-1 monitor=60000 on overcloud-controller-1 +- * Resource action: rabbitmq-bundle-2 monitor=60000 on overcloud-controller-2 + * Pseudo action: rabbitmq-bundle-clone_post_notify_running_0 + * Resource action: rabbitmq:0 notify on rabbitmq-bundle-0 + * Resource action: rabbitmq:1 notify on rabbitmq-bundle-1 +diff --git a/pengine/test10/bundle-order-fencing.scores b/pengine/test10/bundle-order-fencing.scores +index 9037624..25c3e79 100644 +--- a/pengine/test10/bundle-order-fencing.scores ++++ b/pengine/test10/bundle-order-fencing.scores +@@ -183,7 +183,7 @@ container_color: galera-bundle-master allocation score on rabbitmq-bundle-2: -IN + container_color: galera:0 allocation score on controller-0: 0 + container_color: galera:0 allocation score on controller-1: 0 + container_color: galera:0 allocation score on controller-2: 0 +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: INFINITY + container_color: galera:0 allocation score on galera-bundle-1: -INFINITY + container_color: galera:0 allocation score on galera-bundle-2: -INFINITY + container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY +@@ -193,7 +193,7 @@ container_color: galera:1 allocation score on controller-0: 0 + container_color: galera:1 allocation score on controller-1: 0 + container_color: galera:1 allocation score on controller-2: 0 + container_color: galera:1 allocation score on galera-bundle-0: -INFINITY +-container_color: galera:1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:1 allocation score on galera-bundle-1: INFINITY + container_color: galera:1 allocation score on galera-bundle-2: -INFINITY + container_color: galera:1 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY +@@ -203,7 +203,7 @@ container_color: galera:2 allocation score on controller-1: 0 + container_color: galera:2 allocation score on controller-2: 0 + container_color: galera:2 allocation score on galera-bundle-0: -INFINITY + container_color: galera:2 allocation score on galera-bundle-1: -INFINITY +-container_color: galera:2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:2 allocation score on galera-bundle-2: INFINITY + container_color: galera:2 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera:2 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: galera:2 allocation score on rabbitmq-bundle-2: -INFINITY +@@ -714,7 +714,7 @@ container_color: redis:0 allocation score on galera-bundle-2: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: INFINITY + container_color: redis:0 allocation score on redis-bundle-1: -INFINITY + container_color: redis:0 allocation score on redis-bundle-2: -INFINITY + container_color: redis:1 allocation score on controller-0: 0 +@@ -727,7 +727,7 @@ container_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY + container_color: redis:1 allocation score on redis-bundle-0: -INFINITY +-container_color: redis:1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-1: INFINITY + container_color: redis:1 allocation score on redis-bundle-2: -INFINITY + container_color: redis:2 allocation score on controller-0: 0 + container_color: redis:2 allocation score on controller-1: 0 +@@ -740,7 +740,7 @@ container_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY + container_color: redis:2 allocation score on redis-bundle-0: -INFINITY + container_color: redis:2 allocation score on redis-bundle-1: -INFINITY +-container_color: redis:2 allocation score on redis-bundle-2: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-2: INFINITY + galera:0 promotion score on galera-bundle-0: -1 + galera:1 promotion score on galera-bundle-1: 100 + galera:2 promotion score on galera-bundle-2: 100 +diff --git a/pengine/test10/bundle-order-partial-start-2.dot b/pengine/test10/bundle-order-partial-start-2.dot +index 163be1f..d6608bb 100644 +--- a/pengine/test10/bundle-order-partial-start-2.dot ++++ b/pengine/test10/bundle-order-partial-start-2.dot +@@ -2,6 +2,7 @@ digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold] ++"galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_0 galera-bundle-0" [ style = bold] + "galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] + "galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] + "galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] +@@ -28,6 +29,8 @@ digraph "g" { + "galera-bundle_start_0" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] + "galera-bundle_start_0" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"galera:0_monitor_0 galera-bundle-0" -> "galera-bundle-master_start_0" [ style = bold] ++"galera:0_monitor_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] + "galera:0_monitor_20000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] + "galera:0_monitor_30000 galera-bundle-0" [ style=bold color="green" fontcolor="black"] + "galera:0_start_0 galera-bundle-0" -> "galera-bundle-master_running_0" [ style = bold] +diff --git a/pengine/test10/bundle-order-partial-start-2.exp b/pengine/test10/bundle-order-partial-start-2.exp +index 087ed6c..d17819c 100644 +--- a/pengine/test10/bundle-order-partial-start-2.exp ++++ b/pengine/test10/bundle-order-partial-start-2.exp +@@ -1,49 +1,49 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -52,49 +52,49 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -102,145 +102,161 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + + + +@@ -249,29 +265,29 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -281,7 +297,7 @@ + + + +- ++ + + + +@@ -290,277 +306,277 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +@@ -568,10 +584,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start-2.scores b/pengine/test10/bundle-order-partial-start-2.scores +index f5e86de..b4a441d 100644 +--- a/pengine/test10/bundle-order-partial-start-2.scores ++++ b/pengine/test10/bundle-order-partial-start-2.scores +@@ -29,7 +29,7 @@ container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY + container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on undercloud: 0 +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: 500 + container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera:0 allocation score on undercloud: 0 + container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY +@@ -112,7 +112,7 @@ container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 + container_color: rabbitmq-bundle-clone allocation score on undercloud: 0 + container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY +-container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 500 + container_color: rabbitmq:0 allocation score on undercloud: 0 + container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY + container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY +@@ -132,7 +132,7 @@ container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINI + container_color: redis-bundle-master allocation score on undercloud: 0 + container_color: redis:0 allocation score on galera-bundle-0: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: INFINITY + container_color: redis:0 allocation score on undercloud: 0 + galera:0 promotion score on galera-bundle-0: -1 + native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY +diff --git a/pengine/test10/bundle-order-partial-start-2.summary b/pengine/test10/bundle-order-partial-start-2.summary +index 50933a2..f9748d2 100644 +--- a/pengine/test10/bundle-order-partial-start-2.summary ++++ b/pengine/test10/bundle-order-partial-start-2.summary +@@ -62,11 +62,12 @@ Executing cluster transition: + * Resource action: rabbitmq:0 monitor=10000 on rabbitmq-bundle-0 + * Resource action: redis monitor=20000 on redis-bundle-0 + * Pseudo action: galera-bundle_start_0 +- * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera-bundle-docker-0 start on undercloud + * Resource action: galera-bundle-docker-0 monitor=60000 on undercloud + * Resource action: galera-bundle-0 start on undercloud + * Resource action: galera-bundle-0 monitor=60000 on undercloud ++ * Resource action: galera:0 monitor on galera-bundle-0 ++ * Pseudo action: galera-bundle-master_start_0 + * Resource action: galera:0 start on galera-bundle-0 + * Pseudo action: galera-bundle-master_running_0 + * Pseudo action: galera-bundle_running_0 +diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot +index 958e7f4..0e121a4 100644 +--- a/pengine/test10/bundle-order-partial-start.dot ++++ b/pengine/test10/bundle-order-partial-start.dot +@@ -6,6 +6,7 @@ digraph "g" { + "galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] + "galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_monitor_0 undercloud" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] +diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp +index 8aad157..98922d5 100644 +--- a/pengine/test10/bundle-order-partial-start.exp ++++ b/pengine/test10/bundle-order-partial-start.exp +@@ -207,6 +207,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start.scores b/pengine/test10/bundle-order-partial-start.scores +index ea702d0..1c50e66 100644 +--- a/pengine/test10/bundle-order-partial-start.scores ++++ b/pengine/test10/bundle-order-partial-start.scores +@@ -29,7 +29,7 @@ container_color: galera-bundle-docker-0 allocation score on undercloud: 0 + container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on undercloud: 0 +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: 500 + container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera:0 allocation score on undercloud: 0 + container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY +@@ -112,7 +112,7 @@ container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 + container_color: rabbitmq-bundle-clone allocation score on undercloud: 0 + container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: INFINITY +-container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 500 + container_color: rabbitmq:0 allocation score on undercloud: 0 + container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY + container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY +@@ -132,7 +132,7 @@ container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINI + container_color: redis-bundle-master allocation score on undercloud: 0 + container_color: redis:0 allocation score on galera-bundle-0: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: INFINITY + container_color: redis:0 allocation score on undercloud: 0 + galera:0 promotion score on galera-bundle-0: -1 + native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY +diff --git a/pengine/test10/bundle-order-partial-stop.scores b/pengine/test10/bundle-order-partial-stop.scores +index a662f42..528842e 100644 +--- a/pengine/test10/bundle-order-partial-stop.scores ++++ b/pengine/test10/bundle-order-partial-stop.scores +@@ -29,7 +29,7 @@ container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY + container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on undercloud: 0 +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: INFINITY + container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera:0 allocation score on undercloud: 0 + container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY +@@ -132,7 +132,7 @@ container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINI + container_color: redis-bundle-master allocation score on undercloud: 0 + container_color: redis:0 allocation score on galera-bundle-0: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: INFINITY + container_color: redis:0 allocation score on undercloud: 0 + galera:0 promotion score on galera-bundle-0: 100 + native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY +diff --git a/pengine/test10/bundle-order-startup-clone-2.dot b/pengine/test10/bundle-order-startup-clone-2.dot +index 8726909..d0b0f03 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.dot ++++ b/pengine/test10/bundle-order-startup-clone-2.dot +@@ -20,14 +20,17 @@ digraph "g" { + "galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] + "galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] + "galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] + "galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] + "galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] + "galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] + "galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = bold] +@@ -38,14 +41,17 @@ digraph "g" { + "galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] + "galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] + "galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-1" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] + "galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] + "galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-2" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] + "galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] + "galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-1_monitor_0 metal-3" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-1_start_0 metal-2" -> "galera-bundle-1_start_0 metal-2" [ style = bold] +@@ -56,14 +62,17 @@ digraph "g" { + "galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] + "galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] + "galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-1" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] + "galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] + "galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-2" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = bold] + "galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-1_start_0 metal-2" [ style = bold] + "galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-docker-2_start_0 metal-3" [ style = bold] ++"galera-bundle-docker-2_monitor_0 metal-3" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-2_start_0 metal-3" -> "galera-bundle-2_start_0 metal-3" [ style = bold] +@@ -179,14 +188,17 @@ digraph "g" { + "redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_start_0 metal-1" -> "redis-bundle-0_start_0 metal-1" [ style = bold] +@@ -198,14 +210,17 @@ digraph "g" { + "redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] + "redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] + "redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-1" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] + "redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] + "redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-2" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] + "redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] + "redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-1_monitor_0 metal-3" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-1_start_0 metal-2" -> "redis-bundle-1_start_0 metal-2" [ style = bold] +@@ -217,14 +232,17 @@ digraph "g" { + "redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] + "redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] + "redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-1" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] + "redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] + "redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-2" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-1" [ style = bold] + "redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-1_start_0 metal-2" [ style = bold] + "redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-docker-2_start_0 metal-3" [ style = bold] ++"redis-bundle-docker-2_monitor_0 metal-3" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-2_start_0 metal-3" -> "redis-bundle-2_start_0 metal-3" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup-clone-2.exp b/pengine/test10/bundle-order-startup-clone-2.exp +index 7d18d29..b6f90d4 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.exp ++++ b/pengine/test10/bundle-order-startup-clone-2.exp +@@ -444,6 +444,33 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -1523,6 +1550,33 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-startup-clone-2.scores b/pengine/test10/bundle-order-startup-clone-2.scores +index 493cd87..bcc4905 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.scores ++++ b/pengine/test10/bundle-order-startup-clone-2.scores +@@ -143,7 +143,7 @@ container_color: galera-bundle-master allocation score on metal-1: 0 + container_color: galera-bundle-master allocation score on metal-2: 0 + container_color: galera-bundle-master allocation score on metal-3: 0 + container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: 0 +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: 500 + container_color: galera:0 allocation score on galera-bundle-1: -INFINITY + container_color: galera:0 allocation score on galera-bundle-2: -INFINITY + container_color: galera:0 allocation score on metal-1: 0 +@@ -151,7 +151,7 @@ container_color: galera:0 allocation score on metal-2: 0 + container_color: galera:0 allocation score on metal-3: 0 + container_color: galera:0 allocation score on rabbitmq-bundle-0: 0 + container_color: galera:1 allocation score on galera-bundle-0: -INFINITY +-container_color: galera:1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:1 allocation score on galera-bundle-1: 500 + container_color: galera:1 allocation score on galera-bundle-2: -INFINITY + container_color: galera:1 allocation score on metal-1: 0 + container_color: galera:1 allocation score on metal-2: 0 +@@ -159,7 +159,7 @@ container_color: galera:1 allocation score on metal-3: 0 + container_color: galera:1 allocation score on rabbitmq-bundle-0: 0 + container_color: galera:2 allocation score on galera-bundle-0: -INFINITY + container_color: galera:2 allocation score on galera-bundle-1: -INFINITY +-container_color: galera:2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:2 allocation score on galera-bundle-2: 500 + container_color: galera:2 allocation score on metal-1: 0 + container_color: galera:2 allocation score on metal-2: 0 + container_color: galera:2 allocation score on metal-3: 0 +@@ -363,7 +363,7 @@ container_color: redis:0 allocation score on metal-1: 0 + container_color: redis:0 allocation score on metal-2: 0 + container_color: redis:0 allocation score on metal-3: 0 + container_color: redis:0 allocation score on rabbitmq-bundle-0: 0 +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: 500 + container_color: redis:0 allocation score on redis-bundle-1: -INFINITY + container_color: redis:0 allocation score on redis-bundle-2: -INFINITY + container_color: redis:1 allocation score on galera-bundle-0: -INFINITY +@@ -374,7 +374,7 @@ container_color: redis:1 allocation score on metal-2: 0 + container_color: redis:1 allocation score on metal-3: 0 + container_color: redis:1 allocation score on rabbitmq-bundle-0: 0 + container_color: redis:1 allocation score on redis-bundle-0: -INFINITY +-container_color: redis:1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-1: 500 + container_color: redis:1 allocation score on redis-bundle-2: -INFINITY + container_color: redis:2 allocation score on galera-bundle-0: -INFINITY + container_color: redis:2 allocation score on galera-bundle-1: -INFINITY +@@ -385,7 +385,7 @@ container_color: redis:2 allocation score on metal-3: 0 + container_color: redis:2 allocation score on rabbitmq-bundle-0: 0 + container_color: redis:2 allocation score on redis-bundle-0: -INFINITY + container_color: redis:2 allocation score on redis-bundle-1: -INFINITY +-container_color: redis:2 allocation score on redis-bundle-2: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-2: 500 + galera:0 promotion score on galera-bundle-0: -1 + galera:1 promotion score on galera-bundle-1: -1 + galera:2 promotion score on galera-bundle-2: -1 +diff --git a/pengine/test10/bundle-order-startup-clone.dot b/pengine/test10/bundle-order-startup-clone.dot +index 219b676..b791c10 100644 +--- a/pengine/test10/bundle-order-startup-clone.dot ++++ b/pengine/test10/bundle-order-startup-clone.dot +@@ -6,10 +6,13 @@ digraph "g" { + "galera-bundle-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = dashed] + "galera-bundle-0_start_0 metal-1" [ style=dashed color="red" fontcolor="black"] + "galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed] ++"galera-bundle-docker-0_monitor_0 metal-1" -> "galera-bundle-master_start_0" [ style = dashed] + "galera-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed] ++"galera-bundle-docker-0_monitor_0 metal-2" -> "galera-bundle-master_start_0" [ style = dashed] + "galera-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-docker-0_start_0 metal-1" [ style = dashed] ++"galera-bundle-docker-0_monitor_0 metal-3" -> "galera-bundle-master_start_0" [ style = dashed] + "galera-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_60000 metal-1" [ style=dashed color="red" fontcolor="black"] + "galera-bundle-docker-0_start_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = dashed] +@@ -53,10 +56,13 @@ digraph "g" { + "redis-bundle-0_start_0 metal-2" -> "redis:0_start_0 redis-bundle-0" [ style = bold] + "redis-bundle-0_start_0 metal-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-1" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-2" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-docker-0_start_0 metal-2" [ style = bold] ++"redis-bundle-docker-0_monitor_0 metal-3" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_start_0 metal-2" -> "redis-bundle-0_start_0 metal-2" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup-clone.exp b/pengine/test10/bundle-order-startup-clone.exp +index 8c8ba6aa..5be495e 100644 +--- a/pengine/test10/bundle-order-startup-clone.exp ++++ b/pengine/test10/bundle-order-startup-clone.exp +@@ -258,6 +258,15 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-startup-clone.scores b/pengine/test10/bundle-order-startup-clone.scores +index d907861..4361fc4 100644 +--- a/pengine/test10/bundle-order-startup-clone.scores ++++ b/pengine/test10/bundle-order-startup-clone.scores +@@ -61,7 +61,7 @@ container_color: galera-bundle-master allocation score on metal-1: 0 + container_color: galera-bundle-master allocation score on metal-2: 0 + container_color: galera-bundle-master allocation score on metal-3: 0 + container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: 0 +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: 500 + container_color: galera:0 allocation score on metal-1: 0 + container_color: galera:0 allocation score on metal-2: 0 + container_color: galera:0 allocation score on metal-3: 0 +@@ -115,7 +115,7 @@ container_color: redis:0 allocation score on metal-1: 0 + container_color: redis:0 allocation score on metal-2: 0 + container_color: redis:0 allocation score on metal-3: 0 + container_color: redis:0 allocation score on rabbitmq-bundle-0: 0 +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: 500 + galera:0 promotion score on galera-bundle-0: -1 + native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY + native_color: galera-bundle-0 allocation score on metal-1: 10000 +diff --git a/pengine/test10/bundle-order-startup.dot b/pengine/test10/bundle-order-startup.dot +index 3ec10fa..98f1c3c 100644 +--- a/pengine/test10/bundle-order-startup.dot ++++ b/pengine/test10/bundle-order-startup.dot +@@ -6,6 +6,7 @@ digraph "g" { + "galera-bundle-0_start_0 undercloud" -> "galera:0_start_0 galera-bundle-0" [ style = bold] + "galera-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_0 undercloud" -> "galera-bundle-docker-0_start_0 undercloud" [ style = bold] ++"galera-bundle-docker-0_monitor_0 undercloud" -> "galera-bundle-master_start_0" [ style = bold] + "galera-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_start_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] +@@ -104,6 +105,7 @@ digraph "g" { + "rabbitmq-bundle-clone_start_0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq-bundle-clone_start_0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-docker-0_monitor_0 undercloud" -> "rabbitmq-bundle-clone_start_0" [ style = bold] + "rabbitmq-bundle-docker-0_monitor_0 undercloud" -> "rabbitmq-bundle-docker-0_start_0 undercloud" [ style = bold] + "rabbitmq-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] +@@ -129,6 +131,7 @@ digraph "g" { + "redis-bundle-0_start_0 undercloud" -> "redis:0_start_0 redis-bundle-0" [ style = bold] + "redis-bundle-0_start_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_0 undercloud" -> "redis-bundle-docker-0_start_0 undercloud" [ style = bold] ++"redis-bundle-docker-0_monitor_0 undercloud" -> "redis-bundle-master_start_0" [ style = bold] + "redis-bundle-docker-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_start_0 undercloud" -> "redis-bundle-0_start_0 undercloud" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp +index ec1053c..bcef087 100644 +--- a/pengine/test10/bundle-order-startup.exp ++++ b/pengine/test10/bundle-order-startup.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +@@ -123,6 +123,9 @@ + + + ++ ++ ++ + + + +@@ -268,6 +271,9 @@ + + + ++ ++ ++ + + + +@@ -338,7 +344,7 @@ + + + +- ++ + + + +@@ -417,7 +423,7 @@ + + + +- ++ + + + +@@ -479,6 +485,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bundle-order-startup.scores b/pengine/test10/bundle-order-startup.scores +index 565b47e..d49dd6e 100644 +--- a/pengine/test10/bundle-order-startup.scores ++++ b/pengine/test10/bundle-order-startup.scores +@@ -29,7 +29,7 @@ container_color: galera-bundle-docker-0 allocation score on undercloud: 0 + container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on undercloud: 0 +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: 500 + container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera:0 allocation score on undercloud: 0 + container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY +@@ -112,7 +112,7 @@ container_color: rabbitmq-bundle-clone allocation score on rabbitmq-bundle-0: 0 + container_color: rabbitmq-bundle-clone allocation score on undercloud: 0 + container_color: rabbitmq-bundle-docker-0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: rabbitmq-bundle-docker-0 allocation score on undercloud: 0 +-container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 0 ++container_color: rabbitmq:0 allocation score on rabbitmq-bundle-0: 500 + container_color: rabbitmq:0 allocation score on undercloud: 0 + container_color: redis-bundle allocation score on galera-bundle-0: -INFINITY + container_color: redis-bundle allocation score on rabbitmq-bundle-0: -INFINITY +@@ -132,7 +132,7 @@ container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINI + container_color: redis-bundle-master allocation score on undercloud: 0 + container_color: redis:0 allocation score on galera-bundle-0: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: 500 + container_color: redis:0 allocation score on undercloud: 0 + galera:0 promotion score on galera-bundle-0: -1 + native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY +diff --git a/pengine/test10/bundle-order-stop-clone.scores b/pengine/test10/bundle-order-stop-clone.scores +index df53f32..f8ec05c 100644 +--- a/pengine/test10/bundle-order-stop-clone.scores ++++ b/pengine/test10/bundle-order-stop-clone.scores +@@ -143,7 +143,7 @@ container_color: galera-bundle-master allocation score on metal-1: 0 + container_color: galera-bundle-master allocation score on metal-2: 0 + container_color: galera-bundle-master allocation score on metal-3: 0 + container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: 0 +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: INFINITY + container_color: galera:0 allocation score on galera-bundle-1: -INFINITY + container_color: galera:0 allocation score on galera-bundle-2: -INFINITY + container_color: galera:0 allocation score on metal-1: 0 +@@ -151,7 +151,7 @@ container_color: galera:0 allocation score on metal-2: 0 + container_color: galera:0 allocation score on metal-3: 0 + container_color: galera:0 allocation score on rabbitmq-bundle-0: 0 + container_color: galera:1 allocation score on galera-bundle-0: -INFINITY +-container_color: galera:1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:1 allocation score on galera-bundle-1: INFINITY + container_color: galera:1 allocation score on galera-bundle-2: -INFINITY + container_color: galera:1 allocation score on metal-1: 0 + container_color: galera:1 allocation score on metal-2: 0 +@@ -159,7 +159,7 @@ container_color: galera:1 allocation score on metal-3: 0 + container_color: galera:1 allocation score on rabbitmq-bundle-0: 0 + container_color: galera:2 allocation score on galera-bundle-0: -INFINITY + container_color: galera:2 allocation score on galera-bundle-1: -INFINITY +-container_color: galera:2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:2 allocation score on galera-bundle-2: INFINITY + container_color: galera:2 allocation score on metal-1: 0 + container_color: galera:2 allocation score on metal-2: 0 + container_color: galera:2 allocation score on metal-3: 0 +@@ -363,7 +363,7 @@ container_color: redis:0 allocation score on metal-1: 0 + container_color: redis:0 allocation score on metal-2: 0 + container_color: redis:0 allocation score on metal-3: 0 + container_color: redis:0 allocation score on rabbitmq-bundle-0: 0 +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: INFINITY + container_color: redis:0 allocation score on redis-bundle-1: -INFINITY + container_color: redis:0 allocation score on redis-bundle-2: -INFINITY + container_color: redis:1 allocation score on galera-bundle-0: -INFINITY +@@ -374,7 +374,7 @@ container_color: redis:1 allocation score on metal-2: 0 + container_color: redis:1 allocation score on metal-3: 0 + container_color: redis:1 allocation score on rabbitmq-bundle-0: 0 + container_color: redis:1 allocation score on redis-bundle-0: -INFINITY +-container_color: redis:1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-1: INFINITY + container_color: redis:1 allocation score on redis-bundle-2: -INFINITY + container_color: redis:2 allocation score on galera-bundle-0: -INFINITY + container_color: redis:2 allocation score on galera-bundle-1: -INFINITY +@@ -385,7 +385,7 @@ container_color: redis:2 allocation score on metal-3: 0 + container_color: redis:2 allocation score on rabbitmq-bundle-0: 0 + container_color: redis:2 allocation score on redis-bundle-0: -INFINITY + container_color: redis:2 allocation score on redis-bundle-1: -INFINITY +-container_color: redis:2 allocation score on redis-bundle-2: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-2: INFINITY + galera:0 promotion score on galera-bundle-0: -1 + galera:1 promotion score on galera-bundle-1: -1 + galera:2 promotion score on galera-bundle-2: -1 +diff --git a/pengine/test10/bundle-order-stop.scores b/pengine/test10/bundle-order-stop.scores +index a662f42..528842e 100644 +--- a/pengine/test10/bundle-order-stop.scores ++++ b/pengine/test10/bundle-order-stop.scores +@@ -29,7 +29,7 @@ container_color: galera-bundle-docker-0 allocation score on undercloud: INFINITY + container_color: galera-bundle-master allocation score on galera-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on undercloud: 0 +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: INFINITY + container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera:0 allocation score on undercloud: 0 + container_color: haproxy-bundle allocation score on galera-bundle-0: -INFINITY +@@ -132,7 +132,7 @@ container_color: redis-bundle-master allocation score on redis-bundle-0: -INFINI + container_color: redis-bundle-master allocation score on undercloud: 0 + container_color: redis:0 allocation score on galera-bundle-0: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: INFINITY + container_color: redis:0 allocation score on undercloud: 0 + galera:0 promotion score on galera-bundle-0: 100 + native_color: galera-bundle-0 allocation score on galera-bundle-0: -INFINITY +diff --git a/pengine/test10/guest-node-host-dies.dot b/pengine/test10/guest-node-host-dies.dot +index 83f38c4..2082d07 100644 +--- a/pengine/test10/guest-node-host-dies.dot ++++ b/pengine/test10/guest-node-host-dies.dot +@@ -13,6 +13,7 @@ digraph "g" { + "container1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] + "container1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "container1_stop_0 rhel7-1" -> "container1_start_0 rhel7-2" [ style = bold] ++"container1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] + "container1_stop_0 rhel7-1" -> "stonith 'reboot' lxc1" [ style = bold] + "container1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] + "container2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold] +@@ -20,6 +21,7 @@ digraph "g" { + "container2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] + "container2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "container2_stop_0 rhel7-1" -> "container2_start_0 rhel7-3" [ style = bold] ++"container2_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] + "container2_stop_0 rhel7-1" -> "stonith 'reboot' lxc2" [ style = bold] + "container2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] + "lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] +@@ -83,7 +85,6 @@ digraph "g" { + "lxc1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "lxc1_stop_0 rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] + "lxc1_stop_0 rhel7-1" -> "lxc1_start_0 rhel7-2" [ style = bold] +-"lxc1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] + "lxc1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] + "lxc2_monitor_0 rhel7-2" -> "lxc2_start_0 rhel7-3" [ style = bold] + "lxc2_monitor_0 rhel7-2" -> "lxc2_stop_0 rhel7-1" [ style = bold] +@@ -102,7 +103,6 @@ digraph "g" { + "lxc2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "lxc2_stop_0 rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold] + "lxc2_stop_0 rhel7-1" -> "lxc2_start_0 rhel7-3" [ style = bold] +-"lxc2_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] + "lxc2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] + "rsc_rhel7-1_monitor_5000 rhel7-5" [ style=bold color="green" fontcolor="black"] + "rsc_rhel7-1_start_0 rhel7-5" -> "rsc_rhel7-1_monitor_5000 rhel7-5" [ style = bold] +diff --git a/pengine/test10/guest-node-host-dies.exp b/pengine/test10/guest-node-host-dies.exp +index 01fe678..06362e2 100644 +--- a/pengine/test10/guest-node-host-dies.exp ++++ b/pengine/test10/guest-node-host-dies.exp +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/remote-fence-unclean-3.scores b/pengine/test10/remote-fence-unclean-3.scores +index 77d71d6..dba2026 100644 +--- a/pengine/test10/remote-fence-unclean-3.scores ++++ b/pengine/test10/remote-fence-unclean-3.scores +@@ -199,7 +199,7 @@ container_color: galera-bundle-master allocation score on overcloud-novacompute- + container_color: galera-bundle-master allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera-bundle-master allocation score on rabbitmq-bundle-1: -INFINITY + container_color: galera-bundle-master allocation score on rabbitmq-bundle-2: -INFINITY +-container_color: galera:0 allocation score on galera-bundle-0: -INFINITY ++container_color: galera:0 allocation score on galera-bundle-0: INFINITY + container_color: galera:0 allocation score on galera-bundle-1: -INFINITY + container_color: galera:0 allocation score on galera-bundle-2: -INFINITY + container_color: galera:0 allocation score on overcloud-controller-0: 0 +@@ -210,7 +210,7 @@ container_color: galera:0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: galera:0 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: galera:0 allocation score on rabbitmq-bundle-2: -INFINITY + container_color: galera:1 allocation score on galera-bundle-0: -INFINITY +-container_color: galera:1 allocation score on galera-bundle-1: -INFINITY ++container_color: galera:1 allocation score on galera-bundle-1: INFINITY + container_color: galera:1 allocation score on galera-bundle-2: -INFINITY + container_color: galera:1 allocation score on overcloud-controller-0: 0 + container_color: galera:1 allocation score on overcloud-controller-1: 0 +@@ -221,7 +221,7 @@ container_color: galera:1 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: galera:1 allocation score on rabbitmq-bundle-2: -INFINITY + container_color: galera:2 allocation score on galera-bundle-0: -INFINITY + container_color: galera:2 allocation score on galera-bundle-1: -INFINITY +-container_color: galera:2 allocation score on galera-bundle-2: -INFINITY ++container_color: galera:2 allocation score on galera-bundle-2: INFINITY + container_color: galera:2 allocation score on overcloud-controller-0: 0 + container_color: galera:2 allocation score on overcloud-controller-1: 0 + container_color: galera:2 allocation score on overcloud-controller-2: 0 +@@ -836,7 +836,7 @@ container_color: redis:0 allocation score on overcloud-novacompute-0: 0 + container_color: redis:0 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: redis:0 allocation score on rabbitmq-bundle-2: -INFINITY +-container_color: redis:0 allocation score on redis-bundle-0: -INFINITY ++container_color: redis:0 allocation score on redis-bundle-0: INFINITY + container_color: redis:0 allocation score on redis-bundle-1: -INFINITY + container_color: redis:0 allocation score on redis-bundle-2: -INFINITY + container_color: redis:1 allocation score on galera-bundle-0: -INFINITY +@@ -850,7 +850,7 @@ container_color: redis:1 allocation score on rabbitmq-bundle-0: -INFINITY + container_color: redis:1 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: redis:1 allocation score on rabbitmq-bundle-2: -INFINITY + container_color: redis:1 allocation score on redis-bundle-0: -INFINITY +-container_color: redis:1 allocation score on redis-bundle-1: -INFINITY ++container_color: redis:1 allocation score on redis-bundle-1: INFINITY + container_color: redis:1 allocation score on redis-bundle-2: -INFINITY + container_color: redis:2 allocation score on galera-bundle-0: -INFINITY + container_color: redis:2 allocation score on galera-bundle-1: -INFINITY +@@ -864,7 +864,7 @@ container_color: redis:2 allocation score on rabbitmq-bundle-1: -INFINITY + container_color: redis:2 allocation score on rabbitmq-bundle-2: -INFINITY + container_color: redis:2 allocation score on redis-bundle-0: -INFINITY + container_color: redis:2 allocation score on redis-bundle-1: -INFINITY +-container_color: redis:2 allocation score on redis-bundle-2: -INFINITY ++container_color: redis:2 allocation score on redis-bundle-2: INFINITY + galera:0 promotion score on galera-bundle-0: 100 + galera:1 promotion score on galera-bundle-1: 100 + galera:2 promotion score on galera-bundle-2: 100 +diff --git a/pengine/test10/whitebox-asymmetric.dot b/pengine/test10/whitebox-asymmetric.dot +index da2f3e3..529caa6 100644 +--- a/pengine/test10/whitebox-asymmetric.dot ++++ b/pengine/test10/whitebox-asymmetric.dot +@@ -1,14 +1,19 @@ + digraph "g" { + "18node2_monitor_30000 18builder" [ style=bold color="green" fontcolor="black"] + "18node2_start_0 18builder" -> "18node2_monitor_30000 18builder" [ style = bold] ++"18node2_start_0 18builder" -> "nfs_mount_monitor_0 18node2" [ style = bold] + "18node2_start_0 18builder" -> "nfs_mount_monitor_10000 18node2" [ style = bold] + "18node2_start_0 18builder" -> "nfs_mount_start_0 18node2" [ style = bold] + "18node2_start_0 18builder" -> "vg_tags_dup_monitor_0 18node2" [ style = bold] ++"18node2_start_0 18builder" -> "vg_tags_monitor_0 18node2" [ style = bold] + "18node2_start_0 18builder" -> "webserver_monitor_0 18node2" [ style = bold] + "18node2_start_0 18builder" [ style=bold color="green" fontcolor="black"] ++"nfs_mount_monitor_0 18node2" -> "nfs_mount_start_0 18node2" [ style = bold] ++"nfs_mount_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] + "nfs_mount_monitor_10000 18node2" [ style=bold color="green" fontcolor="black"] + "nfs_mount_start_0 18node2" -> "nfs_mount_monitor_10000 18node2" [ style = bold] + "nfs_mount_start_0 18node2" [ style=bold color="green" fontcolor="black"] + "vg_tags_dup_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] ++"vg_tags_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] + "webserver_monitor_0 18node2" [ style=bold color="green" fontcolor="black"] + } +diff --git a/pengine/test10/whitebox-asymmetric.exp b/pengine/test10/whitebox-asymmetric.exp +index db71443..3f105a1 100644 +--- a/pengine/test10/whitebox-asymmetric.exp ++++ b/pengine/test10/whitebox-asymmetric.exp +@@ -8,68 +8,97 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ ++ ++ ++ + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +diff --git a/pengine/test10/whitebox-asymmetric.summary b/pengine/test10/whitebox-asymmetric.summary +index 8da9996..15bf9e7 100644 +--- a/pengine/test10/whitebox-asymmetric.summary ++++ b/pengine/test10/whitebox-asymmetric.summary +@@ -18,9 +18,11 @@ Transition Summary: + Executing cluster transition: + * Resource action: 18node2 start on 18builder + * Resource action: webserver monitor on 18node2 +- * Resource action: nfs_mount start on 18node2 ++ * Resource action: nfs_mount monitor on 18node2 ++ * Resource action: vg_tags monitor on 18node2 + * Resource action: vg_tags_dup monitor on 18node2 + * Resource action: 18node2 monitor=30000 on 18builder ++ * Resource action: nfs_mount start on 18node2 + * Resource action: nfs_mount monitor=10000 on 18node2 + + Revised cluster status: +diff --git a/pengine/test10/whitebox-fail3.dot b/pengine/test10/whitebox-fail3.dot +index 6f608d1..9814f66 100644 +--- a/pengine/test10/whitebox-fail3.dot ++++ b/pengine/test10/whitebox-fail3.dot +@@ -1,8 +1,5 @@ + digraph "g" { + "18builder_monitor_0 dvossel-laptop2" -> "18builder_start_0 dvossel-laptop2" [ style = bold] +-"18builder_monitor_0 dvossel-laptop2" -> "FAKE_start_0 18builder" [ style = bold] +-"18builder_monitor_0 dvossel-laptop2" -> "W-master_start_0" [ style = bold] +-"18builder_monitor_0 dvossel-laptop2" -> "X-master_start_0" [ style = bold] + "18builder_monitor_0 dvossel-laptop2" [ style=bold color="green" fontcolor="black"] + "18builder_monitor_30000 dvossel-laptop2" [ style=bold color="green" fontcolor="black"] + "18builder_start_0 dvossel-laptop2" -> "18builder_monitor_30000 dvossel-laptop2" [ style = bold] +diff --git a/pengine/test10/whitebox-fail3.exp b/pengine/test10/whitebox-fail3.exp +index cddd7dd..ec28f12 100644 +--- a/pengine/test10/whitebox-fail3.exp ++++ b/pengine/test10/whitebox-fail3.exp +@@ -17,9 +17,6 @@ + + + +- +- +- + + + +@@ -95,11 +92,7 @@ + + + +- +- +- +- +- ++ + + + +@@ -157,11 +150,7 @@ + + + +- +- +- +- +- ++ + + + +diff --git a/pengine/test10/whitebox-fail3.summary b/pengine/test10/whitebox-fail3.summary +index 374fb69..eded099 100644 +--- a/pengine/test10/whitebox-fail3.summary ++++ b/pengine/test10/whitebox-fail3.summary +@@ -22,10 +22,10 @@ Transition Summary: + Executing cluster transition: + * Resource action: vm start on dvossel-laptop2 + * Resource action: FAKE stop on dvossel-laptop2 +- * Resource action: 18builder monitor on dvossel-laptop2 +- * Pseudo action: all_stopped + * Pseudo action: W-master_start_0 + * Pseudo action: X-master_start_0 ++ * Resource action: 18builder monitor on dvossel-laptop2 ++ * Pseudo action: all_stopped + * Resource action: 18builder start on dvossel-laptop2 + * Resource action: FAKE start on 18builder + * Resource action: W start on 18builder +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.dot b/pengine/test10/whitebox-imply-stop-on-fence.dot +index 029e9f1..1c41ad2 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.dot ++++ b/pengine/test10/whitebox-imply-stop-on-fence.dot +@@ -15,6 +15,7 @@ + "R-lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] + "R-lxc-02_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "R-lxc-02_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] ++"R-lxc-02_kiff-01_stop_0 kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] + "R-lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] + "all_stopped" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] +@@ -63,7 +64,6 @@ + "lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_stop_0 kiff-01" [ style = bold] + "lxc-02_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "lxc-02_kiff-01_stop_0 kiff-01" -> "lxc-02_kiff-01_start_0 kiff-02" [ style = bold] +-"lxc-02_kiff-01_stop_0 kiff-01" -> "vm-fs_start_0 lxc-01_kiff-01" [ style = bold] + "lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] + "shared0-clone_stop_0" -> "shared0-clone_stopped_0" [ style = bold] + "shared0-clone_stop_0" -> "shared0_stop_0 kiff-01" [ style = bold] +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp +index b5700c0..9a469f2 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.exp ++++ b/pengine/test10/whitebox-imply-stop-on-fence.exp +@@ -374,13 +374,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-migrate1.dot b/pengine/test10/whitebox-migrate1.dot +index 53d37e0..29874e6 100644 +--- a/pengine/test10/whitebox-migrate1.dot ++++ b/pengine/test10/whitebox-migrate1.dot +@@ -17,6 +17,7 @@ + "remote-rsc_start_0 rhel7-node3" -> "rhel7-node1_migrate_to_0 rhel7-node2" [ style = dashed] + "remote-rsc_start_0 rhel7-node3" -> "rhel7-node1_start_0 rhel7-node3" [ style = bold] + "remote-rsc_start_0 rhel7-node3" [ style=bold color="green" fontcolor="orange"] ++"remote-rsc_stop_0 rhel7-node2" -> "FAKE3_start_0 rhel7-node2" [ style = dashed] + "remote-rsc_stop_0 rhel7-node2" -> "all_stopped" [ style = bold] + "remote-rsc_stop_0 rhel7-node2" -> "remote-rsc_start_0 rhel7-node3" [ style = bold] + "remote-rsc_stop_0 rhel7-node2" [ style=bold color="green" fontcolor="black"] +@@ -32,7 +33,6 @@ + "rhel7-node1_monitor_30000 rhel7-node3" [ style=bold color="green" fontcolor="black"] + "rhel7-node1_start_0 rhel7-node3" -> "rhel7-node1_monitor_30000 rhel7-node3" [ style = bold] + "rhel7-node1_start_0 rhel7-node3" [ style=bold color="green" fontcolor="orange"] +-"rhel7-node1_stop_0 rhel7-node2" -> "FAKE3_start_0 rhel7-node2" [ style = dashed] + "rhel7-node1_stop_0 rhel7-node2" -> "all_stopped" [ style = bold] + "rhel7-node1_stop_0 rhel7-node2" -> "remote-rsc_migrate_to_0 rhel7-node2" [ style = dashed] + "rhel7-node1_stop_0 rhel7-node2" -> "remote-rsc_stop_0 rhel7-node2" [ style = bold] +diff --git a/pengine/test10/whitebox-nested-group.dot b/pengine/test10/whitebox-nested-group.dot +index e5749ec..9e1abce 100644 +--- a/pengine/test10/whitebox-nested-group.dot ++++ b/pengine/test10/whitebox-nested-group.dot +@@ -1,30 +1,9 @@ + digraph "g" { + "c7auto4_monitor_0 c7auto1" -> "c7auto4_start_0 c7auto1" [ style = bold] +-"c7auto4_monitor_0 c7auto1" -> "fake1_start_0 c7auto3" [ style = bold] +-"c7auto4_monitor_0 c7auto1" -> "fake2_start_0 c7auto4" [ style = bold] +-"c7auto4_monitor_0 c7auto1" -> "fake3_start_0 c7auto2" [ style = bold] +-"c7auto4_monitor_0 c7auto1" -> "fake4_start_0 c7auto3" [ style = bold] +-"c7auto4_monitor_0 c7auto1" -> "fake5_start_0 c7auto4" [ style = bold] +-"c7auto4_monitor_0 c7auto1" -> "fake_clone_start_0" [ style = bold] +-"c7auto4_monitor_0 c7auto1" -> "fake_group_start_0" [ style = bold] + "c7auto4_monitor_0 c7auto1" [ style=bold color="green" fontcolor="black"] + "c7auto4_monitor_0 c7auto2" -> "c7auto4_start_0 c7auto1" [ style = bold] +-"c7auto4_monitor_0 c7auto2" -> "fake1_start_0 c7auto3" [ style = bold] +-"c7auto4_monitor_0 c7auto2" -> "fake2_start_0 c7auto4" [ style = bold] +-"c7auto4_monitor_0 c7auto2" -> "fake3_start_0 c7auto2" [ style = bold] +-"c7auto4_monitor_0 c7auto2" -> "fake4_start_0 c7auto3" [ style = bold] +-"c7auto4_monitor_0 c7auto2" -> "fake5_start_0 c7auto4" [ style = bold] +-"c7auto4_monitor_0 c7auto2" -> "fake_clone_start_0" [ style = bold] +-"c7auto4_monitor_0 c7auto2" -> "fake_group_start_0" [ style = bold] + "c7auto4_monitor_0 c7auto2" [ style=bold color="green" fontcolor="black"] + "c7auto4_monitor_0 c7auto3" -> "c7auto4_start_0 c7auto1" [ style = bold] +-"c7auto4_monitor_0 c7auto3" -> "fake1_start_0 c7auto3" [ style = bold] +-"c7auto4_monitor_0 c7auto3" -> "fake2_start_0 c7auto4" [ style = bold] +-"c7auto4_monitor_0 c7auto3" -> "fake3_start_0 c7auto2" [ style = bold] +-"c7auto4_monitor_0 c7auto3" -> "fake4_start_0 c7auto3" [ style = bold] +-"c7auto4_monitor_0 c7auto3" -> "fake5_start_0 c7auto4" [ style = bold] +-"c7auto4_monitor_0 c7auto3" -> "fake_clone_start_0" [ style = bold] +-"c7auto4_monitor_0 c7auto3" -> "fake_group_start_0" [ style = bold] + "c7auto4_monitor_0 c7auto3" [ style=bold color="green" fontcolor="black"] + "c7auto4_monitor_30000 c7auto1" [ style=bold color="green" fontcolor="black"] + "c7auto4_start_0 c7auto1" -> "c7auto4_monitor_30000 c7auto1" [ style = bold] +diff --git a/pengine/test10/whitebox-nested-group.exp b/pengine/test10/whitebox-nested-group.exp +index 5625c92..388c4d4 100644 +--- a/pengine/test10/whitebox-nested-group.exp ++++ b/pengine/test10/whitebox-nested-group.exp +@@ -24,20 +24,11 @@ + + + +- +- +- + + + +- +- +- + + +- +- +- + + + +@@ -95,21 +86,12 @@ + + + +- +- +- + + + +- +- +- + + + +- +- +- + + + +@@ -169,20 +151,11 @@ + + + +- +- +- + + + +- +- +- + + +- +- +- + + + +@@ -237,20 +210,11 @@ + + + +- +- +- + + + +- +- +- + + +- +- +- + + + +@@ -308,21 +272,12 @@ + + + +- +- +- + + + +- +- +- + + + +- +- +- + + + +@@ -532,20 +487,11 @@ + + + +- +- +- + + + +- +- +- + + +- +- +- + + + +@@ -572,17 +518,7 @@ + + + +- +- +- +- +- +- +- +- +- +- +- ++ + + + +diff --git a/pengine/test10/whitebox-nested-group.summary b/pengine/test10/whitebox-nested-group.summary +index b03b357..ca9c47f 100644 +--- a/pengine/test10/whitebox-nested-group.summary ++++ b/pengine/test10/whitebox-nested-group.summary +@@ -47,6 +47,8 @@ Executing cluster transition: + * Resource action: fake:0 monitor on c7auto2 + * Resource action: fake:1 monitor on c7auto3 + * Resource action: fake:3 monitor on c7auto1 ++ * Pseudo action: fake_clone_start_0 ++ * Pseudo action: fake_group_start_0 + * Resource action: fake_fs monitor on c7auto3 + * Resource action: fake_fs monitor on c7auto2 + * Resource action: fake_fs monitor on c7auto1 +@@ -56,8 +58,9 @@ Executing cluster transition: + * Resource action: fake1 start on c7auto3 + * Resource action: fake3 start on c7auto2 + * Resource action: fake4 start on c7auto3 +- * Pseudo action: fake_clone_start_0 +- * Pseudo action: fake_group_start_0 ++ * Resource action: fake:0 start on c7auto2 ++ * Resource action: fake:1 start on c7auto3 ++ * Resource action: fake:3 start on c7auto1 + * Resource action: fake_fs start on c7auto1 + * Resource action: container start on c7auto1 + * Resource action: c7auto4 start on c7auto1 +@@ -66,10 +69,10 @@ Executing cluster transition: + * Resource action: fake3 monitor=10000 on c7auto2 + * Resource action: fake4 monitor=10000 on c7auto3 + * Resource action: fake5 start on c7auto4 +- * Resource action: fake:0 start on c7auto2 +- * Resource action: fake:1 start on c7auto3 ++ * Resource action: fake:0 monitor=10000 on c7auto2 ++ * Resource action: fake:1 monitor=10000 on c7auto3 + * Resource action: fake:2 start on c7auto4 +- * Resource action: fake:3 start on c7auto1 ++ * Resource action: fake:3 monitor=10000 on c7auto1 + * Pseudo action: fake_clone_running_0 + * Pseudo action: fake_group_running_0 + * Resource action: fake_fs monitor=10000 on c7auto1 +@@ -77,10 +80,7 @@ Executing cluster transition: + * Resource action: c7auto4 monitor=30000 on c7auto1 + * Resource action: fake2 monitor=10000 on c7auto4 + * Resource action: fake5 monitor=10000 on c7auto4 +- * Resource action: fake:0 monitor=10000 on c7auto2 +- * Resource action: fake:1 monitor=10000 on c7auto3 + * Resource action: fake:2 monitor=10000 on c7auto4 +- * Resource action: fake:3 monitor=10000 on c7auto1 + + Revised cluster status: + Online: [ c7auto1 c7auto2 c7auto3 ] +-- +1.8.3.1 + + +From 6a0dd3429c13343a7177f43cf4c82c352d39f672 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 26 Sep 2017 21:28:53 +1000 +Subject: [PATCH 18/21] Fix: PE: We most definitely need to probe connection + resources + +--- + pengine/container.c | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/pengine/container.c b/pengine/container.c +index 0e556bb..a205c53 100644 +--- a/pengine/container.c ++++ b/pengine/container.c +@@ -885,8 +885,7 @@ container_create_probe(resource_t * rsc, node_t * node, action_t * complete, + } + } + } +- if(FALSE && tuple->remote) { +- // TODO: Needed? ++ if(tuple->remote) { + any_created |= tuple->remote->cmds->create_probe(tuple->remote, node, complete, force, data_set); + } + } +-- +1.8.3.1 + + +From 5d27b590abc98d70ecbca3829569ee2de5e0e75d Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 26 Sep 2017 21:30:37 +1000 +Subject: [PATCH 19/21] Test: PE: We most definitely need to probe connection + resources + +--- + pengine/test10/bundle-nested-colocation.dot | 36 + + pengine/test10/bundle-nested-colocation.exp | 402 +++++-- + pengine/test10/bundle-nested-colocation.summary | 18 + + pengine/test10/bundle-order-fencing.dot | 24 + + pengine/test10/bundle-order-fencing.exp | 1056 +++++++++-------- + pengine/test10/bundle-order-fencing.summary | 12 + + pengine/test10/bundle-order-partial-start.dot | 2 + + pengine/test10/bundle-order-partial-start.exp | 236 ++-- + pengine/test10/bundle-order-partial-start.summary | 1 + + pengine/test10/bundle-order-startup-clone-2.dot | 36 + + pengine/test10/bundle-order-startup-clone-2.exp | 1182 ++++++++++++-------- + .../test10/bundle-order-startup-clone-2.summary | 18 + + pengine/test10/bundle-order-startup-clone.dot | 12 + + pengine/test10/bundle-order-startup-clone.exp | 247 ++-- + pengine/test10/bundle-order-startup-clone.summary | 6 + + pengine/test10/bundle-order-startup.dot | 6 + + pengine/test10/bundle-order-startup.exp | 486 ++++---- + pengine/test10/bundle-order-startup.summary | 3 + + pengine/test10/bundle-order-stop-clone.dot | 16 + + pengine/test10/bundle-order-stop-clone.exp | 248 ++-- + pengine/test10/bundle-order-stop-clone.summary | 12 + + pengine/test10/remote-fence-unclean-3.dot | 18 + + pengine/test10/remote-fence-unclean-3.exp | 176 ++- + pengine/test10/remote-fence-unclean-3.summary | 18 + + 24 files changed, 2727 insertions(+), 1544 deletions(-) + +diff --git a/pengine/test10/bundle-nested-colocation.dot b/pengine/test10/bundle-nested-colocation.dot +index 1a197c1..baa80e2 100644 +--- a/pengine/test10/bundle-nested-colocation.dot ++++ b/pengine/test10/bundle-nested-colocation.dot +@@ -1,17 +1,53 @@ + digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-0_monitor_0 overcloud-controller-0" -> "rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style = bold] ++"rabbitmq-bundle-0_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_monitor_0 overcloud-controller-1" -> "rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style = bold] ++"rabbitmq-bundle-0_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_monitor_0 overcloud-controller-2" -> "rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style = bold] ++"rabbitmq-bundle-0_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_monitor_0 overcloud-galera-0" -> "rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style = bold] ++"rabbitmq-bundle-0_monitor_0 overcloud-galera-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_monitor_0 overcloud-galera-1" -> "rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style = bold] ++"rabbitmq-bundle-0_monitor_0 overcloud-galera-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_monitor_0 overcloud-galera-2" -> "rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style = bold] ++"rabbitmq-bundle-0_monitor_0 overcloud-galera-2" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-0_monitor_60000 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq-bundle-0_monitor_60000 overcloud-controller-0" [ style = bold] + "rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq:0_monitor_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-0_start_0 overcloud-controller-0" -> "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style = bold] + "rabbitmq-bundle-0_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_0 overcloud-controller-0" -> "rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style = bold] ++"rabbitmq-bundle-1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_0 overcloud-controller-1" -> "rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style = bold] ++"rabbitmq-bundle-1_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_0 overcloud-controller-2" -> "rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style = bold] ++"rabbitmq-bundle-1_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_0 overcloud-galera-0" -> "rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style = bold] ++"rabbitmq-bundle-1_monitor_0 overcloud-galera-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_0 overcloud-galera-1" -> "rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style = bold] ++"rabbitmq-bundle-1_monitor_0 overcloud-galera-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_0 overcloud-galera-2" -> "rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style = bold] ++"rabbitmq-bundle-1_monitor_0 overcloud-galera-2" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-1_monitor_60000 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq-bundle-1_monitor_60000 overcloud-controller-1" [ style = bold] + "rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq:1_monitor_0 rabbitmq-bundle-1" [ style = bold] + "rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq:1_monitor_10000 rabbitmq-bundle-1" [ style = bold] + "rabbitmq-bundle-1_start_0 overcloud-controller-1" -> "rabbitmq:1_start_0 rabbitmq-bundle-1" [ style = bold] + "rabbitmq-bundle-1_start_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_0 overcloud-controller-0" -> "rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style = bold] ++"rabbitmq-bundle-2_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_0 overcloud-controller-1" -> "rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style = bold] ++"rabbitmq-bundle-2_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_0 overcloud-controller-2" -> "rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style = bold] ++"rabbitmq-bundle-2_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_0 overcloud-galera-0" -> "rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style = bold] ++"rabbitmq-bundle-2_monitor_0 overcloud-galera-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_0 overcloud-galera-1" -> "rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style = bold] ++"rabbitmq-bundle-2_monitor_0 overcloud-galera-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_0 overcloud-galera-2" -> "rabbitmq-bundle-2_start_0 overcloud-controller-2" [ style = bold] ++"rabbitmq-bundle-2_monitor_0 overcloud-galera-2" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-2_monitor_60000 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq-bundle-2_monitor_60000 overcloud-controller-2" [ style = bold] + "rabbitmq-bundle-2_start_0 overcloud-controller-2" -> "rabbitmq:2_monitor_0 rabbitmq-bundle-2" [ style = bold] +diff --git a/pengine/test10/bundle-nested-colocation.exp b/pengine/test10/bundle-nested-colocation.exp +index b91ced5..266d4c0 100644 +--- a/pengine/test10/bundle-nested-colocation.exp ++++ b/pengine/test10/bundle-nested-colocation.exp +@@ -1,39 +1,39 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -43,61 +43,61 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -107,64 +107,64 @@ + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -174,80 +174,80 @@ + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -255,52 +255,52 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -316,10 +316,10 @@ + + + +- ++ + + +- ++ + + + +@@ -338,20 +338,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -360,18 +360,90 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +@@ -380,14 +452,14 @@ + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -400,22 +472,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -424,18 +496,90 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +@@ -444,14 +588,14 @@ + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -464,22 +608,22 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -488,11 +632,83 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -507,19 +723,19 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +@@ -527,13 +743,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bundle-nested-colocation.summary b/pengine/test10/bundle-nested-colocation.summary +index 7c708d2..2cc1c7c 100644 +--- a/pengine/test10/bundle-nested-colocation.summary ++++ b/pengine/test10/bundle-nested-colocation.summary +@@ -30,8 +30,26 @@ Transition Summary: + Executing cluster transition: + * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 + * Resource action: rabbitmq-bundle-docker-0 stop on overcloud-rabbit-0 ++ * Resource action: rabbitmq-bundle-0 monitor on overcloud-galera-2 ++ * Resource action: rabbitmq-bundle-0 monitor on overcloud-galera-1 ++ * Resource action: rabbitmq-bundle-0 monitor on overcloud-galera-0 ++ * Resource action: rabbitmq-bundle-0 monitor on overcloud-controller-2 ++ * Resource action: rabbitmq-bundle-0 monitor on overcloud-controller-1 ++ * Resource action: rabbitmq-bundle-0 monitor on overcloud-controller-0 + * Resource action: rabbitmq-bundle-docker-1 stop on overcloud-rabbit-1 ++ * Resource action: rabbitmq-bundle-1 monitor on overcloud-galera-2 ++ * Resource action: rabbitmq-bundle-1 monitor on overcloud-galera-1 ++ * Resource action: rabbitmq-bundle-1 monitor on overcloud-galera-0 ++ * Resource action: rabbitmq-bundle-1 monitor on overcloud-controller-2 ++ * Resource action: rabbitmq-bundle-1 monitor on overcloud-controller-1 ++ * Resource action: rabbitmq-bundle-1 monitor on overcloud-controller-0 + * Resource action: rabbitmq-bundle-docker-2 stop on overcloud-rabbit-2 ++ * Resource action: rabbitmq-bundle-2 monitor on overcloud-galera-2 ++ * Resource action: rabbitmq-bundle-2 monitor on overcloud-galera-1 ++ * Resource action: rabbitmq-bundle-2 monitor on overcloud-galera-0 ++ * Resource action: rabbitmq-bundle-2 monitor on overcloud-controller-2 ++ * Resource action: rabbitmq-bundle-2 monitor on overcloud-controller-1 ++ * Resource action: rabbitmq-bundle-2 monitor on overcloud-controller-0 + * Pseudo action: rabbitmq-bundle_start_0 + * Pseudo action: all_stopped + * Pseudo action: rabbitmq-bundle-clone_confirmed-pre_notify_start_0 +diff --git a/pengine/test10/bundle-order-fencing.dot b/pengine/test10/bundle-order-fencing.dot +index 0db9605..cff477b 100644 +--- a/pengine/test10/bundle-order-fencing.dot ++++ b/pengine/test10/bundle-order-fencing.dot +@@ -6,6 +6,12 @@ digraph "g" { + "all_stopped" -> "stonith-fence_ipmilan-5254000dcb3f_start_0 controller-2" [ style = bold] + "all_stopped" -> "stonith-fence_ipmilan-5254003e8e97_start_0 controller-1" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-0_monitor_0 controller-1" -> "galera-bundle-0_start_0 controller-2" [ style = dashed] ++"galera-bundle-0_monitor_0 controller-1" -> "galera-bundle-0_stop_0 controller-0" [ style = bold] ++"galera-bundle-0_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_monitor_0 controller-2" -> "galera-bundle-0_start_0 controller-2" [ style = dashed] ++"galera-bundle-0_monitor_0 controller-2" -> "galera-bundle-0_stop_0 controller-0" [ style = bold] ++"galera-bundle-0_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_monitor_60000 controller-2" [ style=dashed color="red" fontcolor="black"] + "galera-bundle-0_start_0 controller-2" -> "galera-bundle-0_monitor_60000 controller-2" [ style = dashed] + "galera-bundle-0_start_0 controller-2" -> "galera_monitor_20000 galera-bundle-0" [ style = dashed] +@@ -16,6 +22,8 @@ digraph "g" { + "galera-bundle-0_stop_0 controller-0" -> "galera-bundle-0_start_0 controller-2" [ style = dashed] + "galera-bundle-0_stop_0 controller-0" -> "galera-bundle-docker-0_stop_0 controller-0" [ style = bold] + "galera-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-1_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] + "galera-bundle-docker-0_stop_0 controller-0" -> "galera-bundle_stopped_0" [ style = bold] + "galera-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] +@@ -94,6 +102,12 @@ digraph "g" { + "ip-192.168.24.7_stop_0 controller-0" -> "all_stopped" [ style = bold] + "ip-192.168.24.7_stop_0 controller-0" -> "ip-192.168.24.7_start_0 controller-2" [ style = bold] + "ip-192.168.24.7_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-0_monitor_0 controller-1" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed] ++"rabbitmq-bundle-0_monitor_0 controller-1" -> "rabbitmq-bundle-0_stop_0 controller-0" [ style = bold] ++"rabbitmq-bundle-0_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_monitor_0 controller-2" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed] ++"rabbitmq-bundle-0_monitor_0 controller-2" -> "rabbitmq-bundle-0_stop_0 controller-0" [ style = bold] ++"rabbitmq-bundle-0_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-0_monitor_60000 controller-1" [ style=dashed color="red" fontcolor="black"] + "rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq-bundle-0_monitor_60000 controller-1" [ style = dashed] + "rabbitmq-bundle-0_start_0 controller-1" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] +@@ -103,6 +117,8 @@ digraph "g" { + "rabbitmq-bundle-0_stop_0 controller-0" -> "rabbitmq-bundle-0_start_0 controller-1" [ style = dashed] + "rabbitmq-bundle-0_stop_0 controller-0" -> "rabbitmq-bundle-docker-0_stop_0 controller-0" [ style = bold] + "rabbitmq-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-1_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq-bundle_running_0" [ style = bold] + "rabbitmq-bundle-clone_confirmed-post_notify_running_0" -> "rabbitmq_monitor_10000 rabbitmq-bundle-0" [ style = dashed] + "rabbitmq-bundle-clone_confirmed-post_notify_running_0" [ style=bold color="green" fontcolor="orange"] +@@ -189,6 +205,12 @@ digraph "g" { + "rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_stopped_0" [ style = bold] + "rabbitmq_stop_0 rabbitmq-bundle-0" -> "rabbitmq_start_0 rabbitmq-bundle-0" [ style = dashed] + "rabbitmq_stop_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-0_monitor_0 controller-1" -> "redis-bundle-0_start_0 controller-1" [ style = dashed] ++"redis-bundle-0_monitor_0 controller-1" -> "redis-bundle-0_stop_0 controller-0" [ style = bold] ++"redis-bundle-0_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 controller-2" -> "redis-bundle-0_start_0 controller-1" [ style = dashed] ++"redis-bundle-0_monitor_0 controller-2" -> "redis-bundle-0_stop_0 controller-0" [ style = bold] ++"redis-bundle-0_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-0_monitor_60000 controller-1" [ style=dashed color="red" fontcolor="black"] + "redis-bundle-0_start_0 controller-1" -> "redis-bundle-0_monitor_60000 controller-1" [ style = dashed] + "redis-bundle-0_start_0 controller-1" -> "redis_monitor_45000 redis-bundle-0" [ style = dashed] +@@ -199,6 +221,8 @@ digraph "g" { + "redis-bundle-0_stop_0 controller-0" -> "redis-bundle-0_start_0 controller-1" [ style = dashed] + "redis-bundle-0_stop_0 controller-0" -> "redis-bundle-docker-0_stop_0 controller-0" [ style = bold] + "redis-bundle-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-1_monitor_0 controller-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_monitor_0 controller-1" [ style=bold color="green" fontcolor="black"] + "redis-bundle-docker-0_stop_0 controller-0" -> "all_stopped" [ style = bold] + "redis-bundle-docker-0_stop_0 controller-0" -> "redis-bundle_stopped_0" [ style = bold] + "redis-bundle-docker-0_stop_0 controller-0" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/bundle-order-fencing.exp b/pengine/test10/bundle-order-fencing.exp +index d072d85..6782060 100644 +--- a/pengine/test10/bundle-order-fencing.exp ++++ b/pengine/test10/bundle-order-fencing.exp +@@ -1,229 +1,229 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -231,136 +231,136 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -369,16 +369,16 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -386,137 +386,179 @@ + + + ++ ++ ++ ++ ++ ++ + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -525,16 +567,16 @@ + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -542,251 +584,293 @@ + + + ++ ++ ++ ++ ++ ++ + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -799,14 +883,14 @@ + + + +- ++ + + +- ++ + + + +- ++ + + + +@@ -815,7 +899,7 @@ + + + +- ++ + + + +@@ -824,517 +908,517 @@ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -1343,16 +1427,16 @@ + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -1360,40 +1444,82 @@ + + + ++ ++ ++ ++ ++ ++ + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -1402,42 +1528,42 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -1446,42 +1572,42 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +@@ -1490,13 +1616,13 @@ + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -1505,26 +1631,26 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -1534,34 +1660,34 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -1571,21 +1697,21 @@ + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + +@@ -1598,9 +1724,9 @@ + + + +- ++ + +- ++ + + + +@@ -1613,9 +1739,9 @@ + + + +- ++ + +- ++ + + + +@@ -1628,9 +1754,9 @@ + + + +- ++ + +- ++ + + + +@@ -1639,264 +1765,264 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +@@ -1904,68 +2030,68 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-fencing.summary b/pengine/test10/bundle-order-fencing.summary +index e2128cb..e78c531 100644 +--- a/pengine/test10/bundle-order-fencing.summary ++++ b/pengine/test10/bundle-order-fencing.summary +@@ -56,9 +56,21 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: rabbitmq-bundle-clone_pre_notify_stop_0 ++ * Resource action: rabbitmq-bundle-0 monitor on controller-2 ++ * Resource action: rabbitmq-bundle-0 monitor on controller-1 ++ * Resource action: rabbitmq-bundle-1 monitor on controller-2 ++ * Resource action: rabbitmq-bundle-2 monitor on controller-1 ++ * Resource action: galera-bundle-0 monitor on controller-2 ++ * Resource action: galera-bundle-0 monitor on controller-1 ++ * Resource action: galera-bundle-1 monitor on controller-2 ++ * Resource action: galera-bundle-2 monitor on controller-1 + * Resource action: redis cancel=45000 on redis-bundle-1 + * Resource action: redis cancel=60000 on redis-bundle-1 + * Pseudo action: redis-bundle-master_pre_notify_demote_0 ++ * Resource action: redis-bundle-0 monitor on controller-2 ++ * Resource action: redis-bundle-0 monitor on controller-1 ++ * Resource action: redis-bundle-1 monitor on controller-2 ++ * Resource action: redis-bundle-2 monitor on controller-1 + * Pseudo action: stonith-fence_ipmilan-5254003e8e97_stop_0 + * Pseudo action: stonith-fence_ipmilan-5254000dcb3f_stop_0 + * Pseudo action: haproxy-bundle_stop_0 +diff --git a/pengine/test10/bundle-order-partial-start.dot b/pengine/test10/bundle-order-partial-start.dot +index 0e121a4..1106b28 100644 +--- a/pengine/test10/bundle-order-partial-start.dot ++++ b/pengine/test10/bundle-order-partial-start.dot +@@ -1,4 +1,6 @@ + digraph "g" { ++"galera-bundle-0_monitor_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] ++"galera-bundle-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold] + "galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] +diff --git a/pengine/test10/bundle-order-partial-start.exp b/pengine/test10/bundle-order-partial-start.exp +index 98922d5..e883a3b 100644 +--- a/pengine/test10/bundle-order-partial-start.exp ++++ b/pengine/test10/bundle-order-partial-start.exp +@@ -1,43 +1,43 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -52,49 +52,49 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -102,22 +102,22 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -126,82 +126,82 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -210,26 +210,26 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -239,7 +239,7 @@ + + + +- ++ + + + +@@ -254,291 +254,303 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ ++ ++ ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + +diff --git a/pengine/test10/bundle-order-partial-start.summary b/pengine/test10/bundle-order-partial-start.summary +index 9045545..fb8e7de 100644 +--- a/pengine/test10/bundle-order-partial-start.summary ++++ b/pengine/test10/bundle-order-partial-start.summary +@@ -32,6 +32,7 @@ Executing cluster transition: + * Resource action: rabbitmq:0 monitor on rabbitmq-bundle-0 + * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 + * Resource action: galera-bundle-docker-0 monitor on undercloud ++ * Resource action: galera-bundle-0 monitor on undercloud + * Pseudo action: redis-bundle-master_pre_notify_promote_0 + * Resource action: haproxy-bundle-docker-0 monitor on undercloud + * Pseudo action: haproxy-bundle_start_0 +diff --git a/pengine/test10/bundle-order-startup-clone-2.dot b/pengine/test10/bundle-order-startup-clone-2.dot +index d0b0f03..6026530 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.dot ++++ b/pengine/test10/bundle-order-startup-clone-2.dot +@@ -1,16 +1,34 @@ + digraph "g" { ++"galera-bundle-0_monitor_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = bold] ++"galera-bundle-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_monitor_0 metal-2" -> "galera-bundle-0_start_0 metal-1" [ style = bold] ++"galera-bundle-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_monitor_0 metal-3" -> "galera-bundle-0_start_0 metal-1" [ style = bold] ++"galera-bundle-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = bold] + "galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] + "galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_30000 galera-bundle-0" [ style = bold] + "galera-bundle-0_start_0 metal-1" -> "galera:0_start_0 galera-bundle-0" [ style = bold] + "galera-bundle-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_monitor_0 metal-1" -> "galera-bundle-1_start_0 metal-2" [ style = bold] ++"galera-bundle-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_monitor_0 metal-2" -> "galera-bundle-1_start_0 metal-2" [ style = bold] ++"galera-bundle-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_monitor_0 metal-3" -> "galera-bundle-1_start_0 metal-2" [ style = bold] ++"galera-bundle-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] + "galera-bundle-1_start_0 metal-2" -> "galera-bundle-1_monitor_60000 metal-2" [ style = bold] + "galera-bundle-1_start_0 metal-2" -> "galera:1_monitor_20000 galera-bundle-1" [ style = bold] + "galera-bundle-1_start_0 metal-2" -> "galera:1_monitor_30000 galera-bundle-1" [ style = bold] + "galera-bundle-1_start_0 metal-2" -> "galera:1_start_0 galera-bundle-1" [ style = bold] + "galera-bundle-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_monitor_0 metal-1" -> "galera-bundle-2_start_0 metal-3" [ style = bold] ++"galera-bundle-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_monitor_0 metal-2" -> "galera-bundle-2_start_0 metal-3" [ style = bold] ++"galera-bundle-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_monitor_0 metal-3" -> "galera-bundle-2_start_0 metal-3" [ style = bold] ++"galera-bundle-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-2_start_0 metal-3" -> "galera-bundle-2_monitor_60000 metal-3" [ style = bold] + "galera-bundle-2_start_0 metal-3" -> "galera:2_monitor_20000 galera-bundle-2" [ style = bold] +@@ -167,18 +185,36 @@ digraph "g" { + "haproxy-bundle_start_0" -> "haproxy-bundle-docker-1_start_0 metal-2" [ style = bold] + "haproxy-bundle_start_0" -> "haproxy-bundle-docker-2_start_0 metal-3" [ style = bold] + "haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-0_monitor_0 metal-1" -> "redis-bundle-0_start_0 metal-1" [ style = bold] ++"redis-bundle-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 metal-2" -> "redis-bundle-0_start_0 metal-1" [ style = bold] ++"redis-bundle-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 metal-3" -> "redis-bundle-0_start_0 metal-1" [ style = bold] ++"redis-bundle-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-0_monitor_60000 metal-1" [ style=bold color="green" fontcolor="black"] + "redis-bundle-0_start_0 metal-1" -> "redis-bundle-0_monitor_60000 metal-1" [ style = bold] + "redis-bundle-0_start_0 metal-1" -> "redis:0_monitor_20000 redis-bundle-0" [ style = bold] + "redis-bundle-0_start_0 metal-1" -> "redis:0_promote_0 redis-bundle-0" [ style = bold] + "redis-bundle-0_start_0 metal-1" -> "redis:0_start_0 redis-bundle-0" [ style = bold] + "redis-bundle-0_start_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-1_monitor_0 metal-1" -> "redis-bundle-1_start_0 metal-2" [ style = bold] ++"redis-bundle-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-1_monitor_0 metal-2" -> "redis-bundle-1_start_0 metal-2" [ style = bold] ++"redis-bundle-1_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-1_monitor_0 metal-3" -> "redis-bundle-1_start_0 metal-2" [ style = bold] ++"redis-bundle-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-1_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-1_start_0 metal-2" -> "redis-bundle-1_monitor_60000 metal-2" [ style = bold] + "redis-bundle-1_start_0 metal-2" -> "redis:1_monitor_20000 redis-bundle-1" [ style = bold] + "redis-bundle-1_start_0 metal-2" -> "redis:1_promote_0 redis-bundle-1" [ style = bold] + "redis-bundle-1_start_0 metal-2" -> "redis:1_start_0 redis-bundle-1" [ style = bold] + "redis-bundle-1_start_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_monitor_0 metal-1" -> "redis-bundle-2_start_0 metal-3" [ style = bold] ++"redis-bundle-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_monitor_0 metal-2" -> "redis-bundle-2_start_0 metal-3" [ style = bold] ++"redis-bundle-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_monitor_0 metal-3" -> "redis-bundle-2_start_0 metal-3" [ style = bold] ++"redis-bundle-2_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-2_monitor_60000 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-2_start_0 metal-3" -> "redis-bundle-2_monitor_60000 metal-3" [ style = bold] + "redis-bundle-2_start_0 metal-3" -> "redis:2_monitor_20000 redis-bundle-2" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup-clone-2.exp b/pengine/test10/bundle-order-startup-clone-2.exp +index b6f90d4..ae499c0 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.exp ++++ b/pengine/test10/bundle-order-startup-clone-2.exp +@@ -1,43 +1,43 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -52,49 +52,49 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -103,49 +103,49 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -154,55 +154,55 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -210,28 +210,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -240,205 +240,205 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -447,50 +447,50 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -500,37 +500,37 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -539,7 +539,7 @@ + + + +- ++ + + + +@@ -557,46 +557,82 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -606,103 +642,139 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + +@@ -712,1305 +784,1449 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-startup-clone-2.summary b/pengine/test10/bundle-order-startup-clone-2.summary +index a2c2146..07e6f00 100644 +--- a/pengine/test10/bundle-order-startup-clone-2.summary ++++ b/pengine/test10/bundle-order-startup-clone-2.summary +@@ -52,12 +52,21 @@ Executing cluster transition: + * Resource action: galera-bundle-docker-0 monitor on metal-3 + * Resource action: galera-bundle-docker-0 monitor on metal-2 + * Resource action: galera-bundle-docker-0 monitor on metal-1 ++ * Resource action: galera-bundle-0 monitor on metal-3 ++ * Resource action: galera-bundle-0 monitor on metal-2 ++ * Resource action: galera-bundle-0 monitor on metal-1 + * Resource action: galera-bundle-docker-1 monitor on metal-3 + * Resource action: galera-bundle-docker-1 monitor on metal-2 + * Resource action: galera-bundle-docker-1 monitor on metal-1 ++ * Resource action: galera-bundle-1 monitor on metal-3 ++ * Resource action: galera-bundle-1 monitor on metal-2 ++ * Resource action: galera-bundle-1 monitor on metal-1 + * Resource action: galera-bundle-docker-2 monitor on metal-3 + * Resource action: galera-bundle-docker-2 monitor on metal-2 + * Resource action: galera-bundle-docker-2 monitor on metal-1 ++ * Resource action: galera-bundle-2 monitor on metal-3 ++ * Resource action: galera-bundle-2 monitor on metal-2 ++ * Resource action: galera-bundle-2 monitor on metal-1 + * Resource action: haproxy-bundle-docker-0 monitor on metal-3 + * Resource action: haproxy-bundle-docker-0 monitor on metal-2 + * Resource action: haproxy-bundle-docker-0 monitor on metal-1 +@@ -71,12 +80,21 @@ Executing cluster transition: + * Resource action: redis-bundle-docker-0 monitor on metal-3 + * Resource action: redis-bundle-docker-0 monitor on metal-2 + * Resource action: redis-bundle-docker-0 monitor on metal-1 ++ * Resource action: redis-bundle-0 monitor on metal-3 ++ * Resource action: redis-bundle-0 monitor on metal-2 ++ * Resource action: redis-bundle-0 monitor on metal-1 + * Resource action: redis-bundle-docker-1 monitor on metal-3 + * Resource action: redis-bundle-docker-1 monitor on metal-2 + * Resource action: redis-bundle-docker-1 monitor on metal-1 ++ * Resource action: redis-bundle-1 monitor on metal-3 ++ * Resource action: redis-bundle-1 monitor on metal-2 ++ * Resource action: redis-bundle-1 monitor on metal-1 + * Resource action: redis-bundle-docker-2 monitor on metal-3 + * Resource action: redis-bundle-docker-2 monitor on metal-2 + * Resource action: redis-bundle-docker-2 monitor on metal-1 ++ * Resource action: redis-bundle-2 monitor on metal-3 ++ * Resource action: redis-bundle-2 monitor on metal-2 ++ * Resource action: redis-bundle-2 monitor on metal-1 + * Pseudo action: redis-bundle_start_0 + * Pseudo action: haproxy-bundle_start_0 + * Pseudo action: storage-clone_confirmed-pre_notify_start_0 +diff --git a/pengine/test10/bundle-order-startup-clone.dot b/pengine/test10/bundle-order-startup-clone.dot +index b791c10..39f3635 100644 +--- a/pengine/test10/bundle-order-startup-clone.dot ++++ b/pengine/test10/bundle-order-startup-clone.dot +@@ -1,4 +1,10 @@ + digraph "g" { ++"galera-bundle-0_monitor_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = dashed] ++"galera-bundle-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_monitor_0 metal-2" -> "galera-bundle-0_start_0 metal-1" [ style = dashed] ++"galera-bundle-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_monitor_0 metal-3" -> "galera-bundle-0_start_0 metal-1" [ style = dashed] ++"galera-bundle-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_monitor_60000 metal-1" [ style=dashed color="red" fontcolor="black"] + "galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = dashed] + "galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed] +@@ -49,6 +55,12 @@ digraph "g" { + "haproxy-bundle_running_0" [ style=bold color="green" fontcolor="orange"] + "haproxy-bundle_start_0" -> "haproxy-bundle-docker-0_start_0 metal-2" [ style = bold] + "haproxy-bundle_start_0" [ style=bold color="green" fontcolor="orange"] ++"redis-bundle-0_monitor_0 metal-1" -> "redis-bundle-0_start_0 metal-2" [ style = bold] ++"redis-bundle-0_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 metal-2" -> "redis-bundle-0_start_0 metal-2" [ style = bold] ++"redis-bundle-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 metal-3" -> "redis-bundle-0_start_0 metal-2" [ style = bold] ++"redis-bundle-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "redis-bundle-0_monitor_60000 metal-2" [ style=bold color="green" fontcolor="black"] + "redis-bundle-0_start_0 metal-2" -> "redis-bundle-0_monitor_60000 metal-2" [ style = bold] + "redis-bundle-0_start_0 metal-2" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup-clone.exp b/pengine/test10/bundle-order-startup-clone.exp +index 5be495e..9e3dd9b 100644 +--- a/pengine/test10/bundle-order-startup-clone.exp ++++ b/pengine/test10/bundle-order-startup-clone.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -55,351 +55,414 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + +diff --git a/pengine/test10/bundle-order-startup-clone.summary b/pengine/test10/bundle-order-startup-clone.summary +index c2b46de..1346bda 100644 +--- a/pengine/test10/bundle-order-startup-clone.summary ++++ b/pengine/test10/bundle-order-startup-clone.summary +@@ -31,6 +31,9 @@ Executing cluster transition: + * Resource action: galera-bundle-docker-0 monitor on metal-3 + * Resource action: galera-bundle-docker-0 monitor on metal-2 + * Resource action: galera-bundle-docker-0 monitor on metal-1 ++ * Resource action: galera-bundle-0 monitor on metal-3 ++ * Resource action: galera-bundle-0 monitor on metal-2 ++ * Resource action: galera-bundle-0 monitor on metal-1 + * Resource action: haproxy-bundle-docker-0 monitor on metal-3 + * Resource action: haproxy-bundle-docker-0 monitor on metal-2 + * Resource action: haproxy-bundle-docker-0 monitor on metal-1 +@@ -38,6 +41,9 @@ Executing cluster transition: + * Resource action: redis-bundle-docker-0 monitor on metal-3 + * Resource action: redis-bundle-docker-0 monitor on metal-2 + * Resource action: redis-bundle-docker-0 monitor on metal-1 ++ * Resource action: redis-bundle-0 monitor on metal-3 ++ * Resource action: redis-bundle-0 monitor on metal-2 ++ * Resource action: redis-bundle-0 monitor on metal-1 + * Pseudo action: redis-bundle_start_0 + * Pseudo action: haproxy-bundle_start_0 + * Resource action: haproxy-bundle-docker-0 start on metal-2 +diff --git a/pengine/test10/bundle-order-startup.dot b/pengine/test10/bundle-order-startup.dot +index 98f1c3c..213fa4c 100644 +--- a/pengine/test10/bundle-order-startup.dot ++++ b/pengine/test10/bundle-order-startup.dot +@@ -1,4 +1,6 @@ + digraph "g" { ++"galera-bundle-0_monitor_0 undercloud" -> "galera-bundle-0_start_0 undercloud" [ style = bold] ++"galera-bundle-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_start_0 undercloud" -> "galera-bundle-0_monitor_60000 undercloud" [ style = bold] + "galera-bundle-0_start_0 undercloud" -> "galera:0_monitor_20000 galera-bundle-0" [ style = bold] +@@ -84,6 +86,8 @@ digraph "g" { + "openstack-cinder-volume_running_0" [ style=bold color="green" fontcolor="orange"] + "openstack-cinder-volume_start_0" -> "openstack-cinder-volume-docker-0_start_0 undercloud" [ style = bold] + "openstack-cinder-volume_start_0" [ style=bold color="green" fontcolor="orange"] ++"rabbitmq-bundle-0_monitor_0 undercloud" -> "rabbitmq-bundle-0_start_0 undercloud" [ style = bold] ++"rabbitmq-bundle-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq-bundle-0_monitor_60000 undercloud" [ style = bold] + "rabbitmq-bundle-0_start_0 undercloud" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] +@@ -124,6 +128,8 @@ digraph "g" { + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq-bundle-clone_running_0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" -> "rabbitmq:0_monitor_10000 rabbitmq-bundle-0" [ style = bold] + "rabbitmq:0_start_0 rabbitmq-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 undercloud" -> "redis-bundle-0_start_0 undercloud" [ style = bold] ++"redis-bundle-0_monitor_0 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-0_monitor_60000 undercloud" [ style=bold color="green" fontcolor="black"] + "redis-bundle-0_start_0 undercloud" -> "redis-bundle-0_monitor_60000 undercloud" [ style = bold] + "redis-bundle-0_start_0 undercloud" -> "redis:0_monitor_45000 redis-bundle-0" [ style = bold] +diff --git a/pengine/test10/bundle-order-startup.exp b/pengine/test10/bundle-order-startup.exp +index bcef087..77e67bd 100644 +--- a/pengine/test10/bundle-order-startup.exp ++++ b/pengine/test10/bundle-order-startup.exp +@@ -1,100 +1,100 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -102,22 +102,22 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -126,29 +126,29 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -158,7 +158,7 @@ + + + +- ++ + + + +@@ -173,809 +173,845 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ ++ ++ ++ + + + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ + + + +- ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + +diff --git a/pengine/test10/bundle-order-startup.summary b/pengine/test10/bundle-order-startup.summary +index 8da71d0..8e6b685 100644 +--- a/pengine/test10/bundle-order-startup.summary ++++ b/pengine/test10/bundle-order-startup.summary +@@ -41,9 +41,12 @@ Transition Summary: + Executing cluster transition: + * Pseudo action: rabbitmq-bundle-clone_pre_notify_start_0 + * Resource action: rabbitmq-bundle-docker-0 monitor on undercloud ++ * Resource action: rabbitmq-bundle-0 monitor on undercloud + * Resource action: galera-bundle-docker-0 monitor on undercloud ++ * Resource action: galera-bundle-0 monitor on undercloud + * Pseudo action: redis-bundle-master_pre_notify_start_0 + * Resource action: redis-bundle-docker-0 monitor on undercloud ++ * Resource action: redis-bundle-0 monitor on undercloud + * Resource action: ip-192.168.122.254 monitor on undercloud + * Resource action: ip-192.168.122.250 monitor on undercloud + * Resource action: ip-192.168.122.249 monitor on undercloud +diff --git a/pengine/test10/bundle-order-stop-clone.dot b/pengine/test10/bundle-order-stop-clone.dot +index 6545fb8..4d03564 100644 +--- a/pengine/test10/bundle-order-stop-clone.dot ++++ b/pengine/test10/bundle-order-stop-clone.dot +@@ -1,5 +1,11 @@ + digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange"] ++"galera-bundle-0_monitor_0 metal-2" -> "galera-bundle-0_start_0 metal-1" [ style = dashed] ++"galera-bundle-0_monitor_0 metal-2" -> "galera-bundle-0_stop_0 metal-1" [ style = bold] ++"galera-bundle-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_monitor_0 metal-3" -> "galera-bundle-0_start_0 metal-1" [ style = dashed] ++"galera-bundle-0_monitor_0 metal-3" -> "galera-bundle-0_stop_0 metal-1" [ style = bold] ++"galera-bundle-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] + "galera-bundle-0_monitor_60000 metal-1" [ style=dashed color="red" fontcolor="black"] + "galera-bundle-0_start_0 metal-1" -> "galera-bundle-0_monitor_60000 metal-1" [ style = dashed] + "galera-bundle-0_start_0 metal-1" -> "galera:0_monitor_20000 galera-bundle-0" [ style = dashed] +@@ -10,6 +16,10 @@ digraph "g" { + "galera-bundle-0_stop_0 metal-1" -> "galera-bundle-0_start_0 metal-1" [ style = dashed] + "galera-bundle-0_stop_0 metal-1" -> "galera-bundle-docker-0_stop_0 metal-1" [ style = bold] + "galera-bundle-0_stop_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "galera-bundle-docker-0_stop_0 metal-1" -> "all_stopped" [ style = bold] + "galera-bundle-docker-0_stop_0 metal-1" -> "galera-bundle_stopped_0" [ style = bold] + "galera-bundle-docker-0_stop_0 metal-1" [ style=bold color="green" fontcolor="black"] +@@ -45,6 +55,12 @@ digraph "g" { + "galera:0_stop_0 galera-bundle-0" -> "galera-bundle-master_stopped_0" [ style = bold] + "galera:0_stop_0 galera-bundle-0" -> "galera:0_start_0 galera-bundle-0" [ style = dashed] + "galera:0_stop_0 galera-bundle-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-1_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-1_monitor_0 metal-3" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_monitor_0 metal-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_monitor_0 metal-2" [ style=bold color="green" fontcolor="black"] + "storage-clone_confirmed-post_notify_stopped_0" -> "all_stopped" [ style = bold] + "storage-clone_confirmed-post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] + "storage-clone_confirmed-pre_notify_stop_0" -> "storage-clone_post_notify_stopped_0" [ style = bold] +diff --git a/pengine/test10/bundle-order-stop-clone.exp b/pengine/test10/bundle-order-stop-clone.exp +index e78e235..a1106aa 100644 +--- a/pengine/test10/bundle-order-stop-clone.exp ++++ b/pengine/test10/bundle-order-stop-clone.exp +@@ -1,139 +1,139 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -141,123 +141,123 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -267,58 +267,172 @@ + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + +- ++ + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + +- ++ + + + + + +- ++ + + + +- ++ + + + +@@ -326,19 +440,19 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bundle-order-stop-clone.summary b/pengine/test10/bundle-order-stop-clone.summary +index 70e0f21..d988c06 100644 +--- a/pengine/test10/bundle-order-stop-clone.summary ++++ b/pengine/test10/bundle-order-stop-clone.summary +@@ -28,6 +28,18 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: storage-clone_pre_notify_stop_0 ++ * Resource action: galera-bundle-0 monitor on metal-3 ++ * Resource action: galera-bundle-0 monitor on metal-2 ++ * Resource action: galera-bundle-1 monitor on metal-3 ++ * Resource action: galera-bundle-1 monitor on metal-1 ++ * Resource action: galera-bundle-2 monitor on metal-2 ++ * Resource action: galera-bundle-2 monitor on metal-1 ++ * Resource action: redis-bundle-0 monitor on metal-3 ++ * Resource action: redis-bundle-0 monitor on metal-2 ++ * Resource action: redis-bundle-1 monitor on metal-3 ++ * Resource action: redis-bundle-1 monitor on metal-1 ++ * Resource action: redis-bundle-2 monitor on metal-2 ++ * Resource action: redis-bundle-2 monitor on metal-1 + * Pseudo action: galera-bundle_stop_0 + * Resource action: storage:0 notify on metal-1 + * Resource action: storage:1 notify on metal-2 +diff --git a/pengine/test10/remote-fence-unclean-3.dot b/pengine/test10/remote-fence-unclean-3.dot +index 14adaef..dfee1d9 100644 +--- a/pengine/test10/remote-fence-unclean-3.dot ++++ b/pengine/test10/remote-fence-unclean-3.dot +@@ -10,8 +9,26 @@ digraph "g" { + "fence1_monitor_60000 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] + "fence1_start_0 overcloud-controller-0" -> "fence1_monitor_60000 overcloud-controller-0" [ style = bold] + "fence1_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-0_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-1_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"galera-bundle-2_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] + "overcloud-novacompute-0_stop_0 overcloud-controller-0" -> "all_stopped" [ style = bold] + "overcloud-novacompute-0_stop_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-0_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-1_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"rabbitmq-bundle-2_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-0_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-1_monitor_0 overcloud-controller-2" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] ++"redis-bundle-2_monitor_0 overcloud-controller-1" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' overcloud-novacompute-0" -> "stonith_complete" [ style = bold] + "stonith 'reboot' overcloud-novacompute-0" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] +diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp +index 64e5a62..76e5e89 100644 +--- a/pengine/test10/remote-fence-unclean-3.exp ++++ b/pengine/test10/remote-fence-unclean-3.exp +@@ -1,20 +1,20 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + +@@ -27,16 +27,16 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -72,29 +72,191 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -246,7 +269,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary +index ec54d8e..7f2b2d3 100644 +--- a/pengine/test10/remote-fence-unclean-3.summary ++++ b/pengine/test10/remote-fence-unclean-3.summary +@@ -43,6 +43,24 @@ Executing cluster transition: + * Resource action: fence1 monitor on overcloud-controller-1 + * Resource action: fence1 monitor on overcloud-controller-0 + * Resource action: overcloud-novacompute-0 stop on overcloud-controller-0 ++ * Resource action: rabbitmq-bundle-0 monitor on overcloud-controller-2 ++ * Resource action: rabbitmq-bundle-0 monitor on overcloud-controller-1 ++ * Resource action: rabbitmq-bundle-1 monitor on overcloud-controller-2 ++ * Resource action: rabbitmq-bundle-1 monitor on overcloud-controller-0 ++ * Resource action: rabbitmq-bundle-2 monitor on overcloud-controller-1 ++ * Resource action: rabbitmq-bundle-2 monitor on overcloud-controller-0 ++ * Resource action: galera-bundle-0 monitor on overcloud-controller-2 ++ * Resource action: galera-bundle-0 monitor on overcloud-controller-1 ++ * Resource action: galera-bundle-1 monitor on overcloud-controller-2 ++ * Resource action: galera-bundle-1 monitor on overcloud-controller-0 ++ * Resource action: galera-bundle-2 monitor on overcloud-controller-1 ++ * Resource action: galera-bundle-2 monitor on overcloud-controller-0 ++ * Resource action: redis-bundle-0 monitor on overcloud-controller-2 ++ * Resource action: redis-bundle-0 monitor on overcloud-controller-1 ++ * Resource action: redis-bundle-1 monitor on overcloud-controller-2 ++ * Resource action: redis-bundle-1 monitor on overcloud-controller-0 ++ * Resource action: redis-bundle-2 monitor on overcloud-controller-1 ++ * Resource action: redis-bundle-2 monitor on overcloud-controller-0 + * Fencing overcloud-novacompute-0 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped +-- +1.8.3.1 + + +From 140a37e6eaa35a4248abe254641965410e2f833d Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Fri, 22 Sep 2017 12:36:01 +1000 +Subject: [PATCH 20/21] crm_resource: Now that we only clean up on known nodes + we dont need this optimisation + +--- + tools/crm_resource_runtime.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c +index 210ea7d..c280c59 100644 +--- a/tools/crm_resource_runtime.c ++++ b/tools/crm_resource_runtime.c +@@ -619,7 +619,7 @@ cli_resource_delete(crm_ipc_t *crmd_channel, const char *host_uname, + resource_t *child = (resource_t *) lpc->data; + + rc = cli_resource_delete(cib_conn, crmd_channel, host_uname, child, data_set); +- if(rc != pcmk_ok || (pe_rsc_is_clone(rsc) && is_not_set(rsc->flags, pe_rsc_unique))) { ++ if(rc != pcmk_ok) { + return rc; + } + } +-- +1.8.3.1 + + +From f3e82cec10999738437e566e5092513de528c6e3 Mon Sep 17 00:00:00 2001 +From: Andrew Beekhof +Date: Tue, 26 Sep 2017 22:14:20 +1000 +Subject: [PATCH 21/21] Log: Reduce verbosity of developer logging + +--- + lib/common/strings.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/lib/common/strings.c b/lib/common/strings.c +index 5856c16..573a14b 100644 +--- a/lib/common/strings.c ++++ b/lib/common/strings.c +@@ -415,12 +415,12 @@ crm_compress_string(const char *data, int length, int max, char **result, unsign + #ifdef CLOCK_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, &after_t); + +- crm_info("Compressed %d bytes into %d (ratio %d:1) in %dms", ++ crm_trace("Compressed %d bytes into %d (ratio %d:1) in %ldms", + length, *result_len, length / (*result_len), + (after_t.tv_sec - before_t.tv_sec) * 1000 + (after_t.tv_nsec - + before_t.tv_nsec) / 1000000); + #else +- crm_info("Compressed %d bytes into %d (ratio %d:1)", ++ crm_trace("Compressed %d bytes into %d (ratio %d:1)", + length, *result_len, length / (*result_len)); + #endif + +-- +1.8.3.1 + diff --git a/SOURCES/094-attrd_updater.patch b/SOURCES/094-attrd_updater.patch new file mode 100644 index 0000000..5cd540d --- /dev/null +++ b/SOURCES/094-attrd_updater.patch @@ -0,0 +1,717 @@ +From 3d0b886202c155902470a0552d73343dbc0f0178 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 27 Sep 2017 12:05:50 -0500 +Subject: [PATCH 1/6] Log: attrd: "peer peer loss" -> "peer loss" + +--- + attrd/commands.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/attrd/commands.c b/attrd/commands.c +index dffbea3..27d0b73 100644 +--- a/attrd/commands.c ++++ b/attrd/commands.c +@@ -919,7 +919,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *da + } + } else { + /* Remove all attribute values associated with lost nodes */ +- attrd_peer_remove(peer->uname, FALSE, "peer loss"); ++ attrd_peer_remove(peer->uname, FALSE, "loss"); + if (peer_writer && safe_str_eq(peer->uname, peer_writer)) { + free(peer_writer); + peer_writer = NULL; +-- +1.8.3.1 + + +From 91105a32e24b5addaf281595c35892bd0ceb0e44 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 27 Sep 2017 12:11:49 -0500 +Subject: [PATCH 2/6] Refactor: pengine,libpe_status: prefix resource discovery + values + +now that they're publicly exposed +--- + include/crm/pengine/status.h | 9 ++++----- + lib/pengine/container.c | 6 +++--- + pengine/allocate.c | 2 +- + pengine/clone.c | 2 +- + pengine/native.c | 8 ++++---- + pengine/pengine.h | 2 +- + pengine/utils.c | 6 +++--- + 7 files changed, 17 insertions(+), 18 deletions(-) + +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index 16724a7..9b4e944 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -402,11 +402,10 @@ enum pe_link_state { + pe_link_dup, + }; + +- +-enum rsc_discover_e { +- discover_always = 0, +- discover_never, +- discover_exclusive, ++enum pe_discover_e { ++ pe_discover_always = 0, ++ pe_discover_never, ++ pe_discover_exclusive, + }; + + /* *INDENT-OFF* */ +diff --git a/lib/pengine/container.c b/lib/pengine/container.c +index 15234e3..ccbbcb6 100644 +--- a/lib/pengine/container.c ++++ b/lib/pengine/container.c +@@ -499,7 +499,7 @@ disallow_node(resource_t *rsc, const char *uname) + + if (match) { + ((pe_node_t *) match)->weight = -INFINITY; +- ((pe_node_t *) match)->rsc_discover_mode = discover_never; ++ ((pe_node_t *) match)->rsc_discover_mode = pe_discover_never; + } + if (rsc->children) { + GListPtr child; +@@ -573,7 +573,7 @@ create_remote_resource( + } else { + node->weight = -INFINITY; + } +- node->rsc_discover_mode = discover_never; ++ node->rsc_discover_mode = pe_discover_never; + + /* unpack_remote_nodes() ensures that each remote node and guest node + * has a pe_node_t entry. Ideally, it would do the same for bundle nodes. +@@ -598,7 +598,7 @@ create_remote_resource( + + tuple->node = node_copy(node); + tuple->node->weight = 500; +- tuple->node->rsc_discover_mode = discover_exclusive; ++ tuple->node->rsc_discover_mode = pe_discover_exclusive; + + /* Ensure the node shows up as allowed and with the correct discovery set */ + g_hash_table_insert(tuple->child->allowed_nodes, (gpointer) tuple->node->details->id, node_copy(tuple->node)); +diff --git a/pengine/allocate.c b/pengine/allocate.c +index eef9a44..2a80f32 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -956,7 +956,7 @@ rsc_discover_filter(resource_t *rsc, node_t *node) + } + + match = g_hash_table_lookup(rsc->allowed_nodes, node->details->id); +- if (match && match->rsc_discover_mode != discover_exclusive) { ++ if (match && match->rsc_discover_mode != pe_discover_exclusive) { + match->weight = -INFINITY; + } + } +diff --git a/pengine/clone.c b/pengine/clone.c +index 41cb5cb..b10f0f3 100644 +--- a/pengine/clone.c ++++ b/pengine/clone.c +@@ -1359,7 +1359,7 @@ clone_create_probe(resource_t * rsc, node_t * node, action_t * complete, + + if (rsc->exclusive_discover) { + node_t *allowed = g_hash_table_lookup(rsc->allowed_nodes, node->details->id); +- if (allowed && allowed->rsc_discover_mode != discover_exclusive) { ++ if (allowed && allowed->rsc_discover_mode != pe_discover_exclusive) { + /* exclusive discover is enabled and this node is not marked + * as a node this resource should be discovered on + * +diff --git a/pengine/native.c b/pengine/native.c +index 3f52452..08f4c8a 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -2105,7 +2105,7 @@ native_rsc_location(resource_t * rsc, rsc_to_node_t * constraint) + } + + if (other_node->rsc_discover_mode < constraint->discover_mode) { +- if (constraint->discover_mode == discover_exclusive) { ++ if (constraint->discover_mode == pe_discover_exclusive) { + rsc->exclusive_discover = TRUE; + } + /* exclusive > never > always... always is default */ +@@ -2860,7 +2860,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + /* exclusive discover is enabled and this node is not in the allowed list. */ + pe_rsc_trace(rsc, "Skipping probe for %s on node %s, A", rsc->id, node->details->id); + return FALSE; +- } else if (allowed->rsc_discover_mode != discover_exclusive) { ++ } else if (allowed->rsc_discover_mode != pe_discover_exclusive) { + /* exclusive discover is enabled and this node is not marked + * as a node this resource should be discovered on */ + pe_rsc_trace(rsc, "Skipping probe for %s on node %s, B", rsc->id, node->details->id); +@@ -2868,7 +2868,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + } + } + +- if(allowed == NULL && node->rsc_discover_mode == discover_never) { ++ if(allowed == NULL && node->rsc_discover_mode == pe_discover_never) { + /* If this node was allowed to host this resource it would + * have been explicitly added to the 'allowed_nodes' list. + * However it wasn't and the node has discovery disabled, so +@@ -2878,7 +2878,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + return FALSE; + } + +- if (allowed && allowed->rsc_discover_mode == discover_never) { ++ if (allowed && allowed->rsc_discover_mode == pe_discover_never) { + /* this resource is marked as not needing to be discovered on this node */ + pe_rsc_trace(rsc, "Skipping probe for %s on node %s, discovery mode", rsc->id, node->details->id); + return FALSE; +diff --git a/pengine/pengine.h b/pengine/pengine.h +index 2c13258..c88f5cf 100644 +--- a/pengine/pengine.h ++++ b/pengine/pengine.h +@@ -80,7 +80,7 @@ struct rsc_to_node_s { + resource_t *rsc_lh; + + enum rsc_role_e role_filter; +- enum rsc_discover_e discover_mode; ++ enum pe_discover_e discover_mode; + GListPtr node_list_rh; /* node_t* */ + }; + +diff --git a/pengine/utils.c b/pengine/utils.c +index 0cc6381..cae42a8 100644 +--- a/pengine/utils.c ++++ b/pengine/utils.c +@@ -83,11 +83,11 @@ rsc2node_new(const char *id, resource_t * rsc, + + + if (discover_mode == NULL || safe_str_eq(discover_mode, "always")) { +- new_con->discover_mode = discover_always; ++ new_con->discover_mode = pe_discover_always; + } else if (safe_str_eq(discover_mode, "never")) { +- new_con->discover_mode = discover_never; ++ new_con->discover_mode = pe_discover_never; + } else if (safe_str_eq(discover_mode, "exclusive")) { +- new_con->discover_mode = discover_exclusive; ++ new_con->discover_mode = pe_discover_exclusive; + rsc->exclusive_discover = TRUE; + } else { + pe_err("Invalid %s value %s in location constraint", XML_LOCATION_ATTR_DISCOVERY, discover_mode); +-- +1.8.3.1 + + +From 964b3fcfa90e1e1772476ee428c68f13057f7964 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 28 Sep 2017 17:47:16 -0500 +Subject: [PATCH 3/6] Refactor: pengine,libpe_status: prefix node attribute + functions + +since they're publicly exposed +--- + include/crm/pengine/internal.h | 4 ++-- + lib/pengine/common.c | 5 ++--- + lib/pengine/unpack.c | 22 +++++++++++----------- + lib/pengine/utils.c | 4 ++-- + pengine/allocate.c | 2 +- + pengine/constraints.c | 2 +- + pengine/master.c | 6 +++--- + pengine/native.c | 16 ++++++++-------- + tools/crm_mon.c | 2 +- + 9 files changed, 31 insertions(+), 32 deletions(-) + +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index d91f54a..70dd7c7 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -298,7 +298,7 @@ bool remote_id_conflict(const char *remote_name, pe_working_set_t *data); + void common_print(resource_t * rsc, const char *pre_text, const char *name, node_t *node, long options, void *print_data); + resource_t *find_container_child(const char *stem, resource_t * rsc, node_t *node); + bool fix_remote_addr(resource_t * rsc); +-const char *node_attribute_calculated(pe_node_t *node, const char *name, resource_t *rsc); +-const char *node_attribute_raw(pe_node_t *node, const char *name); ++const char *pe_node_attribute_calculated(pe_node_t *node, const char *name, resource_t *rsc); ++const char *pe_node_attribute_raw(pe_node_t *node, const char *name); + + #endif +diff --git a/lib/pengine/common.c b/lib/pengine/common.c +index 0e21aaa..9dd2472 100644 +--- a/lib/pengine/common.c ++++ b/lib/pengine/common.c +@@ -433,7 +433,7 @@ add_hash_param(GHashTable * hash, const char *name, const char *value) + } + + const char * +-node_attribute_calculated(pe_node_t *node, const char *name, resource_t *rsc) ++pe_node_attribute_calculated(pe_node_t *node, const char *name, resource_t *rsc) + { + const char *source; + +@@ -471,11 +471,10 @@ node_attribute_calculated(pe_node_t *node, const char *name, resource_t *rsc) + } + + const char * +-node_attribute_raw(pe_node_t *node, const char *name) ++pe_node_attribute_raw(pe_node_t *node, const char *name) + { + if(node == NULL) { + return NULL; + } + return g_hash_table_lookup(node->details->attrs, name); + } +- +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index a41f35e..48f9368 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -1008,7 +1008,7 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * + attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE); + add_node_attrs(attrs, this_node, TRUE, data_set); + +- shutdown = node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); ++ shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); + if (shutdown != NULL && safe_str_neq("0", shutdown)) { + crm_info("Node %s is shutting down", this_node->details->uname); + this_node->details->shutdown = TRUE; +@@ -1017,18 +1017,18 @@ unpack_handle_remote_attrs(node_t *this_node, xmlNode *state, pe_working_set_t * + } + } + +- if (crm_is_true(node_attribute_raw(this_node, "standby"))) { ++ if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) { + crm_info("Node %s is in standby-mode", this_node->details->uname); + this_node->details->standby = TRUE; + } + +- if (crm_is_true(node_attribute_raw(this_node, "maintenance")) || ++ if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance")) || + (rsc && !is_set(rsc->flags, pe_rsc_managed))) { + crm_info("Node %s is in maintenance-mode", this_node->details->uname); + this_node->details->maintenance = TRUE; + } + +- resource_discovery_enabled = node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY); ++ resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY); + if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) { + if (is_baremetal_remote_node(this_node) && is_not_set(data_set->flags, pe_flag_stonith_enabled)) { + crm_warn("ignoring %s attribute on baremetal remote node %s, disabling resource discovery requires stonith to be enabled.", +@@ -1184,17 +1184,17 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set) + attrs = find_xml_node(state, XML_TAG_TRANSIENT_NODEATTRS, FALSE); + add_node_attrs(attrs, this_node, TRUE, data_set); + +- if (crm_is_true(node_attribute_raw(this_node, "standby"))) { ++ if (crm_is_true(pe_node_attribute_raw(this_node, "standby"))) { + crm_info("Node %s is in standby-mode", this_node->details->uname); + this_node->details->standby = TRUE; + } + +- if (crm_is_true(node_attribute_raw(this_node, "maintenance"))) { ++ if (crm_is_true(pe_node_attribute_raw(this_node, "maintenance"))) { + crm_info("Node %s is in maintenance-mode", this_node->details->uname); + this_node->details->maintenance = TRUE; + } + +- resource_discovery_enabled = node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY); ++ resource_discovery_enabled = pe_node_attribute_raw(this_node, XML_NODE_ATTR_RSC_DISCOVERY); + if (resource_discovery_enabled && !crm_is_true(resource_discovery_enabled)) { + crm_warn("ignoring %s attribute on node %s, disabling resource discovery is not allowed on cluster nodes", + XML_NODE_ATTR_RSC_DISCOVERY, this_node->details->uname); +@@ -1280,7 +1280,7 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat + const char *is_peer = crm_element_value(node_state, XML_NODE_IS_PEER); + const char *in_cluster = crm_element_value(node_state, XML_NODE_IN_CLUSTER); + const char *exp_state = crm_element_value(node_state, XML_NODE_EXPECTED); +- const char *terminate = node_attribute_raw(this_node, "terminate"); ++ const char *terminate = pe_node_attribute_raw(this_node, "terminate"); + + /* + - XML_NODE_IN_CLUSTER ::= true|false +@@ -1454,7 +1454,7 @@ determine_online_status(xmlNode * node_state, node_t * this_node, pe_working_set + + this_node->details->shutdown = FALSE; + this_node->details->expected_up = FALSE; +- shutdown = node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); ++ shutdown = pe_node_attribute_raw(this_node, XML_CIB_ATTR_SHUTDOWN); + + if (shutdown != NULL && safe_str_neq("0", shutdown)) { + this_node->details->shutdown = TRUE; +@@ -3286,8 +3286,8 @@ add_node_attrs(xmlNode * xml_obj, node_t * node, gboolean overwrite, pe_working_ + unpack_instance_attributes(data_set->input, xml_obj, XML_TAG_ATTR_SETS, NULL, + node->details->attrs, NULL, overwrite, data_set->now); + +- if (node_attribute_raw(node, "#site-name") == NULL) { +- const char *site_name = node_attribute_raw(node, "site-name"); ++ if (pe_node_attribute_raw(node, "#site-name") == NULL) { ++ const char *site_name = pe_node_attribute_raw(node, "site-name"); + + if (site_name) { + /* Prefix '#' to the key */ +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index e2a845d..b58f05a 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1996,8 +1996,8 @@ fencing_action_digest_cmp(resource_t * rsc, node_t * node, pe_working_set_t * da + char *key = generate_op_key(rsc->id, STONITH_DIGEST_TASK, 0); + op_digest_cache_t *data = rsc_action_digest(rsc, STONITH_DIGEST_TASK, key, node, NULL, data_set); + +- const char *digest_all = node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); +- const char *digest_secure = node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); ++ const char *digest_all = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_ALL); ++ const char *digest_secure = pe_node_attribute_raw(node, CRM_ATTR_DIGESTS_SECURE); + + /* No 'reloads' for fencing device changes + * +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 2a80f32..62b141a 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -899,7 +899,7 @@ probe_resources(pe_working_set_t * data_set) + + for (GListPtr gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; +- const char *probed = node_attribute_raw(node, CRM_OP_PROBED); ++ const char *probed = pe_node_attribute_raw(node, CRM_OP_PROBED); + + if (is_container_remote_node(node)) { + /* Guest node probes and their ordering requirements are now functional */ +diff --git a/pengine/constraints.c b/pengine/constraints.c +index 9ee83af..aa204fe 100644 +--- a/pengine/constraints.c ++++ b/pengine/constraints.c +@@ -968,7 +968,7 @@ get_node_score(const char *rule, const char *score, gboolean raw, node_t * node, + score_f = char2score(score); + + } else { +- const char *attr_score = node_attribute_calculated(node, score, rsc); ++ const char *attr_score = pe_node_attribute_calculated(node, score, rsc); + + if (attr_score == NULL) { + crm_debug("Rule %s: node %s did not have a value for %s", +diff --git a/pengine/master.c b/pengine/master.c +index 66a531f..ddb92bc 100644 +--- a/pengine/master.c ++++ b/pengine/master.c +@@ -511,7 +457,7 @@ master_score(resource_t * rsc, node_t * node, int not_set_value) + attr_name = calloc(1, len); + sprintf(attr_name, "master-%s", name); + +- attr_value = node_attribute_calculated(node, attr_name, rsc); ++ attr_value = pe_node_attribute_calculated(node, attr_name, rsc); + if (attr_value != NULL) { + score = char2score(attr_value); + } +@@ -984,10 +984,10 @@ node_hash_update_one(GHashTable * hash, node_t * other, const char *attr, int sc + attr = "#" XML_ATTR_UNAME; + } + +- value = node_attribute_raw(other, attr); ++ value = pe_node_attribute_raw(other, attr); + g_hash_table_iter_init(&iter, hash); + while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { +- const char *tmp = node_attribute_raw(node, attr); ++ const char *tmp = pe_node_attribute_raw(node, attr); + + if (safe_str_eq(value, tmp)) { + crm_trace("%s: %d + %d", node->details->uname, node->weight, other->weight); +diff --git a/pengine/native.c b/pengine/native.c +index 08f4c8a..c87127f 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -203,7 +203,7 @@ node_list_attr_score(GHashTable * list, const char *attr, const char *value) + weight = -INFINITY; + } + if (weight > best_score || best_node == NULL) { +- const char *tmp = node_attribute_raw(node, attr); ++ const char *tmp = pe_node_attribute_raw(node, attr); + + if (safe_str_eq(value, tmp)) { + best_score = weight; +@@ -241,7 +241,7 @@ node_hash_update(GHashTable * list1, GHashTable * list2, const char *attr, float + CRM_LOG_ASSERT(node != NULL); + if(node == NULL) { continue; }; + +- score = node_list_attr_score(list2, attr, node_attribute_raw(node, attr)); ++ score = node_list_attr_score(list2, attr, pe_node_attribute_raw(node, attr)); + new_score = merge_weights(factor * score, node->weight); + + if (factor < 0 && score < 0) { +@@ -1640,8 +1640,8 @@ influence_priority(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * + return; + } + +- lh_value = node_attribute_raw(rsc_lh->allocated_to, attribute); +- rh_value = node_attribute_raw(rsc_rh->allocated_to, attribute); ++ lh_value = pe_node_attribute_raw(rsc_lh->allocated_to, attribute); ++ rh_value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute); + + if (!safe_str_eq(lh_value, rh_value)) { + if(constraint->score == INFINITY && constraint->role_lh == RSC_ROLE_MASTER) { +@@ -1679,7 +1679,7 @@ colocation_match(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * co + } + + if (rsc_rh->allocated_to) { +- value = node_attribute_raw(rsc_rh->allocated_to, attribute); ++ value = pe_node_attribute_raw(rsc_rh->allocated_to, attribute); + do_check = TRUE; + + } else if (constraint->score < 0) { +@@ -1693,7 +1693,7 @@ colocation_match(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * co + + g_hash_table_iter_init(&iter, work); + while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { +- tmp = node_attribute_raw(node, attribute); ++ tmp = pe_node_attribute_raw(node, attribute); + if (do_check && safe_str_eq(tmp, value)) { + if (constraint->score < INFINITY) { + pe_rsc_trace(rsc_lh, "%s: %s.%s += %d", constraint->id, rsc_lh->id, +@@ -2503,7 +2503,7 @@ StopRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * d + + if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { + action_t *unfence = pe_fence_op(current, "on", TRUE, NULL, data_set); +- const char *unfenced = node_attribute_raw(current, CRM_ATTR_UNFENCED); ++ const char *unfenced = pe_node_attribute_raw(current, CRM_ATTR_UNFENCED); + + order_actions(stop, unfence, pe_order_implies_first); + if (unfenced == NULL || safe_str_eq("0", unfenced)) { +@@ -2526,7 +2526,7 @@ StartRsc(resource_t * rsc, node_t * next, gboolean optional, pe_working_set_t * + + if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { + action_t *unfence = pe_fence_op(next, "on", TRUE, NULL, data_set); +- const char *unfenced = node_attribute_raw(next, CRM_ATTR_UNFENCED); ++ const char *unfenced = pe_node_attribute_raw(next, CRM_ATTR_UNFENCED); + + order_actions(unfence, start, pe_order_implies_then); + +diff --git a/tools/crm_mon.c b/tools/crm_mon.c +index cad468e..6a29115 100644 +--- a/tools/crm_mon.c ++++ b/tools/crm_mon.c +@@ -1726,7 +1726,7 @@ print_node_attribute(gpointer name, gpointer user_data) + const char *value = NULL; + struct mon_attr_data *data = (struct mon_attr_data *) user_data; + +- value = node_attribute_raw(data->node, name); ++ value = pe_node_attribute_raw(data->node, name); + + /* Print attribute name and value */ + switch (output_format) { +-- +1.8.3.1 + + +From 1309e0ae0ab7bb7491c8c0e2149762b90bf45857 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 28 Sep 2017 17:52:32 -0500 +Subject: [PATCH 4/6] Low: pengine: don't assert for nodes without unames + +It's possible data_set->nodes has a node with only an ID. + +This also has minor refactoring to avoid unnecessary memory allocation. +--- + pengine/notif.c | 27 +++++++++++++++------------ + 1 file changed, 15 insertions(+), 12 deletions(-) + +diff --git a/pengine/notif.c b/pengine/notif.c +index 76c5a4a..39d8c72 100644 +--- a/pengine/notif.c ++++ b/pengine/notif.c +@@ -103,7 +103,9 @@ expand_node_list(GListPtr list, char **uname, char **metal) + int existing_len = 0; + node_t *node = (node_t *) gIter->data; + +- CRM_ASSERT(node->details->uname); ++ if (node->details->uname == NULL) { ++ continue; ++ } + len = 2 + strlen(node->details->uname); + + if(node_list) { +@@ -125,7 +127,9 @@ expand_node_list(GListPtr list, char **uname, char **metal) + node = node->details->remote_rsc->container->running_on->data; + } + +- CRM_ASSERT(node->details->uname); ++ if (node->details->uname == NULL) { ++ continue; ++ } + len = 2 + strlen(node->details->uname); + metal_list = realloc_safe(metal_list, len + existing_len); + sprintf(metal_list + existing_len, "%s%s", existing_len == 0 ? "":" ", node->details->uname); +@@ -532,6 +536,7 @@ expand_notification_data(resource_t *rsc, notify_data_t * n_data, pe_working_set + char *rsc_list = NULL; + char *node_list = NULL; + char *metal_list = NULL; ++ const char *source = NULL; + GListPtr nodes = NULL; + + if (n_data->stop) { +@@ -609,17 +614,15 @@ expand_notification_data(resource_t *rsc, notify_data_t * n_data, pe_working_set + g_hash_table_insert(n_data->keys, strdup("notify_available_uname"), node_list); + g_list_free(nodes); + +- expand_node_list(data_set->nodes, &node_list, &metal_list); +- g_hash_table_insert(n_data->keys, strdup("notify_all_uname"), node_list); +- +- { +- const char *source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET); +- if(safe_str_eq("host", source)) { +- g_hash_table_insert(n_data->keys, strdup("notify_all_hosts"), metal_list); +- } else { +- free(metal_list); +- } ++ source = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_TARGET); ++ if (safe_str_eq("host", source)) { ++ expand_node_list(data_set->nodes, &node_list, &metal_list); ++ g_hash_table_insert(n_data->keys, strdup("notify_all_hosts"), ++ metal_list); ++ } else { ++ expand_node_list(data_set->nodes, &node_list, NULL); + } ++ g_hash_table_insert(n_data->keys, strdup("notify_all_uname"), node_list); + + if (required && n_data->pre) { + update_action_flags(n_data->pre, pe_action_optional | pe_action_clear, __FUNCTION__, __LINE__); +-- +1.8.3.1 + + +From 308387753ce8dd045a9c77fefc1ab2b36255c9bb Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 28 Sep 2017 17:54:40 -0500 +Subject: [PATCH 5/6] Low: pengine: don't move up to resource's parent if not + start action + +--- + pengine/graph.c | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/pengine/graph.c b/pengine/graph.c +index beaa077..8ee6aa8 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -441,17 +441,16 @@ update_colo_start_chain(action_t *action) + + if (is_not_set(action->flags, pe_action_runnable) && safe_str_eq(action->task, RSC_START)) { + rsc = uber_parent(action->rsc); ++ if (rsc->parent) { ++ // This is a bundle (uber_parent() stops _before_ the bundle) ++ rsc = rsc->parent; ++ } + } + + if (rsc == NULL || rsc->rsc_cons_lhs == NULL) { + return; + } + +- if(rsc->parent) { +- /* uber_parent() stops _before_ the bundle */ +- rsc = rsc->parent; +- } +- + /* if rsc has children, all the children need to have start set to + * unrunnable before we follow the colo chain for the parent. */ + for (gIter = rsc->children; gIter != NULL; gIter = gIter->next) { +-- +1.8.3.1 + + +From 39d1bc68a63122cc42b2dc87bed60653171dcacc Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Thu, 28 Sep 2017 18:03:04 -0500 +Subject: [PATCH 6/6] Low: tools: partially revert ccbdb2a2 to avoid + attrd_updater needing libcrmcluster + +Both approaches have drawbacks when used on remote nodes. + +If we keep the get_local_node_name() calls, attrd_udpater needs libcrmcluster, +which shouldn't be required on remote nodes. Also, on a corosync cluster for +example, the call on a remote node will spend 10+ seconds trying to connect to +corosync before finally using uname, which will be incorrect for remote nodes +whose name known to the cluster is not the same as uname. + +If we avoid get_local_node_name() (the approach here), there is not a clear way +to distinguish a query with a NULL node name meaning "let the remote proxy +callback add the name" from "query all nodes". However, this likely exists +already, and should be addressed separately. +--- + pacemaker.spec.in | 4 +--- + tools/Makefile.am | 2 +- + tools/attrd_updater.c | 7 ------- + 3 files changed, 2 insertions(+), 11 deletions(-) + +diff --git a/pacemaker.spec.in b/pacemaker.spec.in +index d0bc2c3..642af74 100644 +--- a/pacemaker.spec.in ++++ b/pacemaker.spec.in +@@ -601,7 +601,6 @@ exit 0 + %exclude %{_sbindir}/pacemaker_remoted + %{_libexecdir}/pacemaker/* + +-%{_sbindir}/attrd_updater + %{_sbindir}/crm_attribute + %{_sbindir}/crm_master + %{_sbindir}/crm_node +@@ -620,7 +619,6 @@ exit 0 + %doc %{_mandir}/man7/ocf_pacemaker_o2cb.* + %doc %{_mandir}/man7/ocf_pacemaker_remote.* + %doc %{_mandir}/man8/crm_attribute.* +-%doc %{_mandir}/man8/attrd_updater.* + %doc %{_mandir}/man8/crm_node.* + %doc %{_mandir}/man8/crm_master.* + %if %{with cman} +@@ -670,6 +668,7 @@ exit 0 + %config(noreplace) %{_sysconfdir}/init/crm_mon.conf + %endif + ++%{_sbindir}/attrd_updater + %{_sbindir}/cibadmin + %{_sbindir}/crm_diff + %{_sbindir}/crm_error +@@ -705,7 +704,6 @@ exit 0 + %exclude %{_mandir}/man7/ocf_pacemaker_o2cb.* + %exclude %{_mandir}/man7/ocf_pacemaker_remote.* + %doc %{_mandir}/man8/* +-%exclude %{_mandir}/man8/attrd_updater.* + %exclude %{_mandir}/man8/crm_attribute.* + %exclude %{_mandir}/man8/crm_node.* + %exclude %{_mandir}/man8/crm_master.* +diff --git a/tools/Makefile.am b/tools/Makefile.am +index a601b8d..3548035 100644 +--- a/tools/Makefile.am ++++ b/tools/Makefile.am +@@ -121,7 +121,7 @@ iso8601_SOURCES = test.iso8601.c + iso8601_LDADD = $(COMMONLIBS) + + attrd_updater_SOURCES = attrd_updater.c +-attrd_updater_LDADD = $(top_builddir)/lib/cluster/libcrmcluster.la $(COMMONLIBS) ++attrd_updater_LDADD = $(COMMONLIBS) + + crm_ticket_SOURCES = crm_ticket.c + crm_ticket_LDADD = $(top_builddir)/lib/pengine/libpe_rules.la \ +diff --git a/tools/attrd_updater.c b/tools/attrd_updater.c +index 5e8b837..1da166b 100644 +--- a/tools/attrd_updater.c ++++ b/tools/attrd_updater.c +@@ -32,7 +32,6 @@ + #include + + #include +-#include + + /* *INDENT-OFF* */ + static struct crm_option long_options[] = { +@@ -201,9 +200,6 @@ main(int argc, char **argv) + } else { + + attr_node = attrd_get_target(attr_node); +- if (attr_node == NULL) { +- attr_node = get_local_node_name(); +- } + crm_exit(do_update(command, attr_node, attr_name, attr_value, + attr_section, attr_set, attr_dampen, attr_options)); + } +@@ -354,9 +350,6 @@ do_query(const char *attr_name, const char *attr_node, gboolean query_all) + attr_node = NULL; + } else { + attr_node = attrd_get_target(attr_node); +- if (attr_node == NULL) { +- attr_node = get_local_node_name(); +- } + } + + /* Build and send attrd request, and get XML reply */ +-- +1.8.3.1 + diff --git a/SOURCES/095-attrd-client-leak.patch b/SOURCES/095-attrd-client-leak.patch new file mode 100644 index 0000000..42a5c52 --- /dev/null +++ b/SOURCES/095-attrd-client-leak.patch @@ -0,0 +1,60 @@ +From b41b23a7f3cd668512401a539fdf75f98d9a1d74 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Sat, 30 Sep 2017 14:51:43 -0500 +Subject: [PATCH] Low: libcrmcommon: avoid memory leak in attrd_get_target() + +--- + lib/common/utils.c | 24 +++++++++++++++++------- + 1 file changed, 17 insertions(+), 7 deletions(-) + +diff --git a/lib/common/utils.c b/lib/common/utils.c +index 290a661..4c02869 100644 +--- a/lib/common/utils.c ++++ b/lib/common/utils.c +@@ -2157,6 +2157,8 @@ attrd_clear_delegate(crm_ipc_t *ipc, const char *host, const char *resource, + } + #endif + ++#define LRM_TARGET_ENV "OCF_RESKEY_" CRM_META "_" XML_LRM_ATTR_TARGET ++ + const char * + attrd_get_target(const char *name) + { +@@ -2168,19 +2170,27 @@ attrd_get_target(const char *name) + return name; + + } else { +- const char *target = getenv(crm_meta_name(XML_RSC_ATTR_TARGET)); +- const char *host_pyhsical = getenv(crm_meta_name(PCMK_ENV_PHYSICAL_HOST)); +- const char *host_pcmk = getenv("OCF_RESKEY_" CRM_META "_" XML_LRM_ATTR_TARGET); ++ char *target_var = crm_meta_name(XML_RSC_ATTR_TARGET); ++ char *phys_var = crm_meta_name(PCMK_ENV_PHYSICAL_HOST); ++ const char *target = getenv(target_var); ++ const char *host_physical = getenv(phys_var); + + /* It is important we use the names by which the PE knows us */ +- if(safe_str_eq(target, "host") && host_pyhsical != NULL) { +- return host_pyhsical; ++ if (host_physical && safe_str_eq(target, "host")) { ++ name = host_physical; ++ ++ } else { ++ const char *host_pcmk = getenv(LRM_TARGET_ENV); + +- } else if(host_pcmk) { +- return host_pcmk; ++ if (host_pcmk) { ++ name = host_pcmk; ++ } + } ++ free(target_var); ++ free(phys_var); + } + + // TODO? Call get_local_node_name() if name == NULL ++ // (currently would require linkage against libcrmcluster) + return name; + } +-- +1.8.3.1 + diff --git a/SOURCES/096-digest-leak.patch b/SOURCES/096-digest-leak.patch new file mode 100644 index 0000000..03fb8c6 --- /dev/null +++ b/SOURCES/096-digest-leak.patch @@ -0,0 +1,33 @@ +From a82c9726299044f4d9d0cfac6ed78a959db726c2 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 2 Oct 2017 15:51:31 -0500 +Subject: [PATCH] Low: libpe_status: avoid memory leak in unfencing digests + +--- + lib/pengine/utils.c | 10 ++++++---- + 1 file changed, 6 insertions(+), 4 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index b58f05a..82b6809 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -2157,10 +2157,12 @@ pe_fence_op(node_t * node, const char *op, bool optional, const char *reason, pe + digests_secure+digests_secure_offset, max-digests_secure_offset, + "%s:%s:%s,", match->id, (const char*)g_hash_table_lookup(match->meta, XML_ATTR_TYPE), data->digest_secure_calc); + } +- add_hash_param(stonith_op->meta, strdup(XML_OP_ATTR_DIGESTS_ALL), +- digests_all); +- add_hash_param(stonith_op->meta, strdup(XML_OP_ATTR_DIGESTS_SECURE), +- digests_secure); ++ g_hash_table_insert(stonith_op->meta, ++ strdup(XML_OP_ATTR_DIGESTS_ALL), ++ digests_all); ++ g_hash_table_insert(stonith_op->meta, ++ strdup(XML_OP_ATTR_DIGESTS_SECURE), ++ digests_secure); + } + + } else { +-- +1.8.3.1 + diff --git a/SOURCES/regression-test-feature-set.patch b/SOURCES/regression-test-feature-set.patch new file mode 100644 index 0000000..2c4a364 --- /dev/null +++ b/SOURCES/regression-test-feature-set.patch @@ -0,0 +1,348 @@ +From fa856a94806e0d8ec849186ed7c01e317c93be45 Mon Sep 17 00:00:00 2001 +From: rpm-build +Date: Fri, 29 Sep 2017 15:50:12 -0500 +Subject: [PATCH] Test: pengine: lower test feature set for use with RHEL 7.4 + +--- + pengine/test10/remote-reconnect-delay.xml | 180 +++++++++++++++--------------- + 1 file changed, 90 insertions(+), 90 deletions(-) + +diff --git a/pengine/test10/remote-reconnect-delay.xml b/pengine/test10/remote-reconnect-delay.xml +index e9ed3e6..d544a2c 100644 +--- a/pengine/test10/remote-reconnect-delay.xml ++++ b/pengine/test10/remote-reconnect-delay.xml +@@ -1,4 +1,4 @@ +- ++ + + + +@@ -243,59 +243,59 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- +- ++ ++ + + +- ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- +- +- ++ ++ ++ + + +- +- ++ ++ + + + +@@ -310,60 +310,60 @@ + + + +- +- ++ ++ + + +- +- ++ ++ + + +- ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- ++ + + +- +- ++ ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +@@ -372,56 +372,56 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- ++ + + +- ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +@@ -442,58 +442,58 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- ++ + + +- ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- +- ++ ++ + + +- ++ + + +- ++ + + + +-- +1.8.3.1 + diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec index 3f421c8..c734229 100644 --- a/SPECS/pacemaker.spec +++ b/SPECS/pacemaker.spec @@ -143,7 +143,7 @@ Name: pacemaker Summary: Scalable High-Availability cluster resource manager Version: %{pcmkversion} -Release: %{pcmk_release}%{?dist}.2 +Release: %{pcmk_release}%{?dist}.4 %if %{defined _unitdir} License: GPLv2+ and LGPLv2+ %else @@ -237,10 +237,28 @@ Patch76: 076-quorum-loss.patch Patch77: 077-reenable-fence-device.patch Patch78: 078-fencing-memory.patch Patch79: 079-crm_report.patch +Patch80: 080-docker-location.patch +Patch81: 081-fence-logs.patch +Patch82: 082-unrecoverable-remotes.patch +Patch83: 083-unrecoverable-remotes-test.patch +Patch84: 084-remote-fence-test.patch +Patch85: 085-remote-fence-test2.patch +Patch86: 086-nested-containers.patch +Patch87: 087-nested-containers-test.patch +Patch88: 088-str-table.patch +Patch89: 089-unfencing.patch +Patch90: 090-remote-fence-loop.patch +Patch91: 091-compat.patch +Patch92: 092-remote-fencing.patch +Patch93: 093-bundle-fixes.patch +Patch94: 094-attrd_updater.patch +Patch95: 095-attrd-client-leak.patch +Patch96: 096-digest-leak.patch # patches that aren't from upstream Patch100: lrmd-protocol-version.patch Patch101: rhbz-url.patch +Patch102: regression-test-feature-set.patch BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) AutoReqProv: on @@ -906,6 +924,20 @@ exit 0 %attr(0644,root,root) %{_datadir}/pacemaker/nagios/plugins-metadata/* %changelog +* Mon Oct 2 2017 Ken Gaillot - 1.1.16-12.4 +- Avoid memory leak when unfencing is needed +- Resolves: rhbz#1491544 + +%changelog +* Mon Oct 2 2017 Ken Gaillot - 1.1.16-12.3 +- Allow unfencing of remote nodes +- Support clone notifications for bundle resources +- Support colocation constraints involving bundles +- Support container-attribute-target resource meta-attribute +- Resolves: rhbz#1491544 +- Resolves: rhbz#1497602 + +%changelog * Fri Aug 18 2017 Ken Gaillot - 1.1.16-12.2 - Allow crm_report to work when no log files are specified - Resolves: rhbz#1482852