diff --git a/SOURCES/015-node-terminology.patch b/SOURCES/015-node-terminology.patch new file mode 100644 index 0000000..12e0ac6 --- /dev/null +++ b/SOURCES/015-node-terminology.patch @@ -0,0 +1,158 @@ +From 2ba6773d0aea4ece57a69f5c0902ca865834aaf8 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 5 Aug 2016 10:23:46 -0500 +Subject: [PATCH] Low: pengine: use new "remote node" vs "guest node" + terminology in allocate.c + +--- + pengine/allocate.c | 69 ++++++++++++++++++++++++++++++------------------------ + 1 file changed, 39 insertions(+), 30 deletions(-) + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 999cc79..7464e9a 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -880,14 +880,14 @@ probe_resources(pe_working_set_t * data_set) + continue; + + } else if (is_remote_node(node) && node->details->shutdown) { +- /* Don't try and probe a remote node we're shutting down. +- * It causes constraint conflicts to try and run any sort of action +- * other that 'stop' on resources living within a remote-node when +- * it is being shutdown. */ ++ /* Don't probe a Pacemaker Remote node we're shutting down. ++ * It causes constraint conflicts to try to run any action ++ * other than "stop" on resources living within such a node when ++ * it is shutting down. */ + continue; + + } else if (is_container_remote_node(node)) { +- /* TODO enable container node probes once ordered probing is implemented. */ ++ /* TODO enable guest node probes once ordered probing is implemented */ + continue; + + } else if (node->details->rsc_discovery_enabled == FALSE) { +@@ -1158,9 +1158,10 @@ allocate_resources(pe_working_set_t * data_set) + continue; + } + pe_rsc_trace(rsc, "Allocating: %s", rsc->id); +- /* for remote node connection resources, always prefer the partial migration +- * target during resource allocation if the rsc is in the middle of a +- * migration */ ++ /* For remote node connection resources, always prefer the partial ++ * migration target during resource allocation, if the rsc is in the ++ * middle of a migration. ++ */ + rsc->cmds->allocate(rsc, rsc->partial_migration_target, data_set); + } + } +@@ -1368,7 +1369,10 @@ stage6(pe_working_set_t * data_set) + for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + +- /* remote-nodes associated with a container resource (such as a vm) are not fenced */ ++ /* Guest nodes are "fenced" by recovering their container resource. ++ * The container stop may be explicit, or implied by the fencing of the ++ * guest's host. ++ */ + if (is_container_remote_node(node)) { + /* Guest */ + if (need_stonith +@@ -1417,7 +1421,7 @@ stage6(pe_working_set_t * data_set) + } + + } else if (node->details->online && node->details->shutdown && +- /* TODO define what a shutdown op means for a baremetal remote node. ++ /* TODO define what a shutdown op means for a remote node. + * For now we do not send shutdown operations for remote nodes, but + * if we can come up with a good use for this in the future, we will. */ + is_remote_node(node) == FALSE) { +@@ -1694,10 +1698,11 @@ apply_remote_node_ordering(pe_working_set_t *data_set) + action->rsc->is_remote_node && + safe_str_eq(action->task, CRM_OP_CLEAR_FAILCOUNT)) { + +- /* if we are clearing the failcount of an actual remote node connect +- * resource, then make sure this happens before allowing the connection +- * to start if we are planning on starting the connection during this +- * transition */ ++ /* If we are clearing the failcount of an actual remote node ++ * connection resource, then make sure this happens before allowing ++ * the connection to start if we are planning on starting the ++ * connection during this transition. ++ */ + custom_action_order(action->rsc, + NULL, + action, +@@ -1710,10 +1715,10 @@ apply_remote_node_ordering(pe_working_set_t *data_set) + continue; + } + +- /* detect if the action occurs on a remote node. if so create +- * ordering constraints that guarantee the action occurs while +- * the remote node is active (after start, before stop...) things +- * like that */ ++ /* If the action occurs on a Pacemaker Remote node, create ++ * ordering constraints that guarantee the action occurs while the node ++ * is active (after start, before stop ... things like that). ++ */ + if (action->node == NULL || + is_remote_node(action->node) == FALSE || + action->node->details->remote_rsc == NULL || +@@ -1747,12 +1752,13 @@ apply_remote_node_ordering(pe_working_set_t *data_set) + * to build a constraint between a resource's demotion and + * the connection resource starting... because the connection + * resource can not start. The connection might already be up, +- * but the START action would not be allowed which in turn would +- * block the demotion of any resournces living in the remote-node. ++ * but the "start" action would not be allowed, which in turn would ++ * block the demotion of any resources living in the node. + * + * In this case, only build the constraint between the demotion and +- * the connection's stop action. This allows the connection and all the +- * resources within the remote-node to be torn down properly. */ ++ * the connection's "stop" action. This allows the connection and ++ * all the resources within the node to be torn down properly. ++ */ + if (remote_rsc->next_role == RSC_ROLE_STOPPED) { + custom_action_order(action->rsc, + NULL, +@@ -1780,10 +1786,11 @@ apply_remote_node_ordering(pe_working_set_t *data_set) + container && + is_set(container->flags, pe_rsc_failed)) { + +- /* when the container representing a remote node fails, the stop ++ /* When the container representing a guest node fails, the stop + * action for all the resources living in that container is implied +- * by the container stopping. This is similar to how fencing operations +- * work for cluster nodes. */ ++ * by the container stopping. This is similar to how fencing ++ * operations work for cluster nodes. ++ */ + pe_set_action_bit(action, pe_action_pseudo); + custom_action_order(container, + generate_op_key(container->id, RSC_STOP, 0), +@@ -1796,14 +1803,16 @@ apply_remote_node_ordering(pe_working_set_t *data_set) + } else if (safe_str_eq(action->task, "stop")) { + gboolean after_start = FALSE; + +- /* handle special case with baremetal remote where stop actions need to be +- * ordered after the connection resource starts somewhere else. */ ++ /* Handle special case with remote node where stop actions need to be ++ * ordered after the connection resource starts somewhere else. ++ */ + if (is_baremetal_remote_node(action->node)) { + node_t *cluster_node = remote_rsc->running_on ? remote_rsc->running_on->data : NULL; + +- /* if the current cluster node a baremetal connection resource +- * is residing on is unclean or went offline we can't process any +- * operations on that remote node until after it starts somewhere else. */ ++ /* If the cluster node the remote connection resource resides on ++ * is unclean or went offline, we can't process any operations ++ * on that remote node until after it starts elsewhere. ++ */ + if (cluster_node == NULL || + cluster_node->details->unclean == TRUE || + cluster_node->details->online == FALSE) { +-- +1.8.3.1 + diff --git a/SOURCES/016-guest-fencing.patch b/SOURCES/016-guest-fencing.patch new file mode 100644 index 0000000..d5e845c --- /dev/null +++ b/SOURCES/016-guest-fencing.patch @@ -0,0 +1,1644 @@ +From 73da74305b69b086f8bc7cae697063e2534a79f4 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 22 Nov 2016 16:37:07 -0600 +Subject: [PATCH 1/6] Low: pengine: remove unnecessary assert + +it was made obsolete with 1420ff88 +--- + lib/pengine/unpack.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index a9fbcc0..2ef9343 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -1402,6 +1402,10 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node) + resource_t *rsc = this_node->details->remote_rsc; + resource_t *container = NULL; + ++ /* If there is a node state entry for a (former) Pacemaker Remote node ++ * but no resource creating that node, the node's connection resource will ++ * be NULL. Consider it an offline remote node in that case. ++ */ + if (rsc == NULL) { + this_node->details->online = FALSE; + goto remote_online_done; +@@ -1409,8 +1413,6 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node) + + container = rsc->container; + +- CRM_ASSERT(rsc != NULL); +- + /* If the resource is currently started, mark it online. */ + if (rsc->role == RSC_ROLE_STARTED) { + crm_trace("Remote node %s is set to ONLINE. role == started", this_node->details->id); +-- +1.8.3.1 + + +From 5156074d560d85ee84de31b9d1e0bd893999fa4e Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 23 Nov 2016 13:40:47 -0600 +Subject: [PATCH 2/6] Log: pengine: improve trace messages for Pacemaker Remote + nodes + +--- + lib/pengine/unpack.c | 22 ++++++++++++++++------ + 1 file changed, 16 insertions(+), 6 deletions(-) + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 2ef9343..a49e108 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -70,10 +70,15 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason) + /* A guest node is fenced by marking its container as failed */ + if (is_container_remote_node(node)) { + resource_t *rsc = node->details->remote_rsc->container; ++ + if (is_set(rsc->flags, pe_rsc_failed) == FALSE) { + crm_warn("Guest node %s will be fenced (by recovering %s) %s", + node->details->uname, rsc->id, reason); +- /* node->details->unclean = TRUE; */ ++ ++ /* We don't mark the node as unclean, because that would prevent the ++ * node from running resources. We want to allow it to run resources ++ * in this transition if the recovery succeeds. ++ */ + node->details->remote_requires_reset = TRUE; + set_bit(rsc->flags, pe_rsc_failed); + } +@@ -1415,30 +1420,35 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node) + + /* If the resource is currently started, mark it online. */ + if (rsc->role == RSC_ROLE_STARTED) { +- crm_trace("Remote node %s is set to ONLINE. role == started", this_node->details->id); ++ crm_trace("%s node %s presumed ONLINE because connection resource is started", ++ (container? "Guest" : "Remote"), this_node->details->id); + this_node->details->online = TRUE; + } + + /* consider this node shutting down if transitioning start->stop */ + if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) { +- crm_trace("Remote node %s shutdown. transition from start to stop role", this_node->details->id); ++ crm_trace("%s node %s shutting down because connection resource is stopping", ++ (container? "Guest" : "Remote"), this_node->details->id); + this_node->details->shutdown = TRUE; + } + + /* Now check all the failure conditions. */ + if(container && is_set(container->flags, pe_rsc_failed)) { +- crm_trace("Remote node %s is set to UNCLEAN. rsc failed.", this_node->details->id); ++ crm_trace("Guest node %s UNCLEAN because guest resource failed", ++ this_node->details->id); + this_node->details->online = FALSE; + this_node->details->remote_requires_reset = TRUE; + + } else if(is_set(rsc->flags, pe_rsc_failed)) { +- crm_trace("Remote node %s is set to OFFLINE. rsc failed.", this_node->details->id); ++ crm_trace("%s node %s OFFLINE because connection resource failed", ++ (container? "Guest" : "Remote"), this_node->details->id); + this_node->details->online = FALSE; + + } else if (rsc->role == RSC_ROLE_STOPPED + || (container && container->role == RSC_ROLE_STOPPED)) { + +- crm_trace("Remote node %s is set to OFFLINE. node is stopped.", this_node->details->id); ++ crm_trace("%s node %s OFFLINE because its resource is stopped", ++ (container? "Guest" : "Remote"), this_node->details->id); + this_node->details->online = FALSE; + this_node->details->remote_requires_reset = FALSE; + } +-- +1.8.3.1 + + +From 225d20cacc5643e113d42159fc713071172d88da Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 22 Nov 2016 16:40:52 -0600 +Subject: [PATCH 3/6] Fix: pengine: guest node fencing doesn't require stonith + enabled + +Comments elsewhere say as much, but stage6() didn't get the memo +--- + lib/pengine/utils.c | 13 +++++++++++++ + pengine/allocate.c | 5 +---- + 2 files changed, 14 insertions(+), 4 deletions(-) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index cc97db1..6be9bb1 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -36,6 +36,19 @@ void unpack_operation(action_t * action, xmlNode * xml_obj, resource_t * contain + static xmlNode *find_rsc_op_entry_helper(resource_t * rsc, const char *key, + gboolean include_disabled); + ++/*! ++ * \internal ++ * \brief Check whether we can fence a particular node ++ * ++ * \param[in] data_set Working set for cluster ++ * \param[in] node Name of node to check ++ * ++ * \return TRUE if node can be fenced, FALSE otherwise ++ * ++ * \note This function should only be called for cluster nodes and baremetal ++ * remote nodes; guest nodes are fenced by stopping their container ++ * resource, so fence execution requirements do not apply to them. ++ */ + bool pe_can_fence(pe_working_set_t * data_set, node_t *node) + { + if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) { +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 82abd36..bdf03e5 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1374,10 +1374,7 @@ stage6(pe_working_set_t * data_set) + * guest's host. + */ + if (is_container_remote_node(node)) { +- /* Guest */ +- if (need_stonith +- && node->details->remote_requires_reset +- && pe_can_fence(data_set, node)) { ++ if (node->details->remote_requires_reset && need_stonith) { + resource_t *container = node->details->remote_rsc->container; + char *key = stop_key(container); + GListPtr stop_list = find_actions(container->actions, key, NULL); +-- +1.8.3.1 + + +From b11887869723f23a330af8b1b0e9ffd935b68ae0 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 23 Mar 2016 17:57:50 -0500 +Subject: [PATCH 4/6] Test: pengine: add regression test for when a guest + node's host goes away + +As of this commit, the PE handles this situation badly. Adding the test before +the fix allows the changes in behavior to be highlighted. +--- + pengine/regression.sh | 1 + + pengine/test10/guest-node-host-dies.dot | 107 ++++++ + pengine/test10/guest-node-host-dies.exp | 539 ++++++++++++++++++++++++++++ + pengine/test10/guest-node-host-dies.scores | 80 +++++ + pengine/test10/guest-node-host-dies.summary | 73 ++++ + pengine/test10/guest-node-host-dies.xml | 294 +++++++++++++++ + 6 files changed, 1094 insertions(+) + create mode 100644 pengine/test10/guest-node-host-dies.dot + create mode 100644 pengine/test10/guest-node-host-dies.exp + create mode 100644 pengine/test10/guest-node-host-dies.scores + create mode 100644 pengine/test10/guest-node-host-dies.summary + create mode 100644 pengine/test10/guest-node-host-dies.xml + +diff --git a/pengine/regression.sh b/pengine/regression.sh +index be1734b..1bc8e1e 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -817,6 +817,7 @@ do_test whitebox-unexpectedly-running "Recover container nodes the cluster did n + do_test whitebox-migrate1 "Migrate both container and connection resource" + do_test whitebox-imply-stop-on-fence "imply stop action on container node rsc when host node is fenced" + do_test whitebox-nested-group "Verify guest remote-node works nested in a group" ++do_test guest-node-host-dies "Verify guest node is recovered if host goes away" + + echo "" + do_test remote-startup-probes "Baremetal remote-node startup probes" +diff --git a/pengine/test10/guest-node-host-dies.dot b/pengine/test10/guest-node-host-dies.dot +new file mode 100644 +index 0000000..01858b3 +--- /dev/null ++++ b/pengine/test10/guest-node-host-dies.dot +@@ -0,0 +1,107 @@ ++digraph "g" { ++"Fencing_monitor_120000 rhel7-4" [ style=bold color="green" fontcolor="black"] ++"Fencing_start_0 rhel7-4" -> "Fencing_monitor_120000 rhel7-4" [ style = bold] ++"Fencing_start_0 rhel7-4" [ style=bold color="green" fontcolor="black"] ++"Fencing_stop_0 rhel7-4" -> "Fencing_start_0 rhel7-4" [ style = bold] ++"Fencing_stop_0 rhel7-4" -> "all_stopped" [ style = bold] ++"Fencing_stop_0 rhel7-4" [ style=bold color="green" fontcolor="black"] ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"container1_start_0 rhel7-2" -> "lxc1_start_0 rhel7-2" [ style = bold] ++"container1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] ++"container1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] ++"container1_stop_0 rhel7-1" -> "container1_start_0 rhel7-2" [ style = bold] ++"container1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] ++"container2_start_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold] ++"container2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] ++"container2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] ++"container2_stop_0 rhel7-1" -> "container2_start_0 rhel7-3" [ style = bold] ++"container2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] ++"lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc1" [ style = bold] ++"lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold] ++"lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold] ++"lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold] ++"lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc1" [ style = bold] ++"lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold] ++"lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold] ++"lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc1" [ style = bold] ++"lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc2" [ style = bold] ++"lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold] ++"lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc1" [ style = bold] ++"lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc2" [ style = bold] ++"lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold] ++"lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold] ++"lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms_demote_0 lxc1" -> "lxc-ms-master_demoted_0" [ style = bold] ++"lxc-ms_demote_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] ++"lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] ++"lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="black"] ++"lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] ++"lxc-ms_promote_0 lxc1" -> "lxc-ms-master_promoted_0" [ style = bold] ++"lxc-ms_promote_0 lxc1" [ style=bold color="green" fontcolor="black"] ++"lxc-ms_start_0 lxc1" -> "lxc-ms-master_running_0" [ style = bold] ++"lxc-ms_start_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] ++"lxc-ms_start_0 lxc1" [ style=bold color="green" fontcolor="black"] ++"lxc-ms_start_0 lxc2" -> "lxc-ms-master_running_0" [ style = bold] ++"lxc-ms_start_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] ++"lxc-ms_start_0 lxc2" [ style=bold color="green" fontcolor="black"] ++"lxc-ms_stop_0 lxc1" -> "all_stopped" [ style = bold] ++"lxc-ms_stop_0 lxc1" -> "lxc-ms-master_stopped_0" [ style = bold] ++"lxc-ms_stop_0 lxc1" -> "lxc-ms_start_0 lxc1" [ style = bold] ++"lxc-ms_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"] ++"lxc-ms_stop_0 lxc2" -> "all_stopped" [ style = bold] ++"lxc-ms_stop_0 lxc2" -> "lxc-ms-master_stopped_0" [ style = bold] ++"lxc-ms_stop_0 lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold] ++"lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"] ++"lxc1_monitor_30000 rhel7-2" [ style=bold color="green" fontcolor="black"] ++"lxc1_start_0 rhel7-2" -> "lxc-ms_promote_0 lxc1" [ style = bold] ++"lxc1_start_0 rhel7-2" -> "lxc-ms_start_0 lxc1" [ style = bold] ++"lxc1_start_0 rhel7-2" -> "lxc1_monitor_30000 rhel7-2" [ style = bold] ++"lxc1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] ++"lxc1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] ++"lxc1_stop_0 rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] ++"lxc1_stop_0 rhel7-1" -> "lxc1_start_0 rhel7-2" [ style = bold] ++"lxc1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] ++"lxc2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"] ++"lxc2_start_0 rhel7-3" -> "lxc-ms_monitor_10000 lxc2" [ style = bold] ++"lxc2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold] ++"lxc2_start_0 rhel7-3" -> "lxc2_monitor_30000 rhel7-3" [ style = bold] ++"lxc2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] ++"lxc2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] ++"lxc2_stop_0 rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold] ++"lxc2_stop_0 rhel7-1" -> "lxc2_start_0 rhel7-3" [ style = bold] ++"lxc2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] ++"rsc_rhel7-1_monitor_5000 rhel7-5" [ style=bold color="green" fontcolor="black"] ++"rsc_rhel7-1_start_0 rhel7-5" -> "rsc_rhel7-1_monitor_5000 rhel7-5" [ style = bold] ++"rsc_rhel7-1_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"] ++"rsc_rhel7-1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] ++"rsc_rhel7-1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] ++"rsc_rhel7-1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] ++"stonith 'reboot' rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] ++"stonith 'reboot' rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold] ++"stonith 'reboot' rhel7-1" -> "lxc-ms-master_stop_0" [ style = bold] ++"stonith 'reboot' rhel7-1" -> "lxc-ms_stop_0 lxc1" [ style = bold] ++"stonith 'reboot' rhel7-1" -> "lxc-ms_stop_0 lxc2" [ style = bold] ++"stonith 'reboot' rhel7-1" -> "lxc1_stop_0 rhel7-1" [ style = bold] ++"stonith 'reboot' rhel7-1" -> "lxc2_stop_0 rhel7-1" [ style = bold] ++"stonith 'reboot' rhel7-1" -> "rsc_rhel7-1_stop_0 rhel7-1" [ style = bold] ++"stonith 'reboot' rhel7-1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' rhel7-1" [ style=bold color="green" fontcolor="black"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "container1_start_0 rhel7-2" [ style = bold] ++"stonith_complete" -> "container2_start_0 rhel7-3" [ style = bold] ++"stonith_complete" -> "lxc-ms_promote_0 lxc1" [ style = bold] ++"stonith_complete" -> "lxc-ms_start_0 lxc1" [ style = bold] ++"stonith_complete" -> "lxc-ms_start_0 lxc2" [ style = bold] ++"stonith_complete" -> "lxc1_start_0 rhel7-2" [ style = bold] ++"stonith_complete" -> "lxc2_start_0 rhel7-3" [ style = bold] ++"stonith_complete" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] ++} +diff --git a/pengine/test10/guest-node-host-dies.exp b/pengine/test10/guest-node-host-dies.exp +new file mode 100644 +index 0000000..b3c24be +--- /dev/null ++++ b/pengine/test10/guest-node-host-dies.exp +@@ -0,0 +1,539 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/guest-node-host-dies.scores b/pengine/test10/guest-node-host-dies.scores +new file mode 100644 +index 0000000..0d7ad3f +--- /dev/null ++++ b/pengine/test10/guest-node-host-dies.scores +@@ -0,0 +1,80 @@ ++Allocation scores: ++clone_color: lxc-ms-master allocation score on lxc1: INFINITY ++clone_color: lxc-ms-master allocation score on lxc2: INFINITY ++clone_color: lxc-ms-master allocation score on rhel7-1: 0 ++clone_color: lxc-ms-master allocation score on rhel7-2: 0 ++clone_color: lxc-ms-master allocation score on rhel7-3: 0 ++clone_color: lxc-ms-master allocation score on rhel7-4: 0 ++clone_color: lxc-ms-master allocation score on rhel7-5: 0 ++clone_color: lxc-ms:0 allocation score on lxc1: INFINITY ++clone_color: lxc-ms:0 allocation score on lxc2: INFINITY ++clone_color: lxc-ms:0 allocation score on rhel7-1: 0 ++clone_color: lxc-ms:0 allocation score on rhel7-2: 0 ++clone_color: lxc-ms:0 allocation score on rhel7-3: 0 ++clone_color: lxc-ms:0 allocation score on rhel7-4: 0 ++clone_color: lxc-ms:0 allocation score on rhel7-5: 0 ++clone_color: lxc-ms:1 allocation score on lxc1: INFINITY ++clone_color: lxc-ms:1 allocation score on lxc2: INFINITY ++clone_color: lxc-ms:1 allocation score on rhel7-1: 0 ++clone_color: lxc-ms:1 allocation score on rhel7-2: 0 ++clone_color: lxc-ms:1 allocation score on rhel7-3: 0 ++clone_color: lxc-ms:1 allocation score on rhel7-4: 0 ++clone_color: lxc-ms:1 allocation score on rhel7-5: 0 ++lxc-ms:0 promotion score on lxc1: INFINITY ++lxc-ms:1 promotion score on lxc2: INFINITY ++native_color: Fencing allocation score on lxc1: -INFINITY ++native_color: Fencing allocation score on lxc2: -INFINITY ++native_color: Fencing allocation score on rhel7-1: 0 ++native_color: Fencing allocation score on rhel7-2: 0 ++native_color: Fencing allocation score on rhel7-3: 0 ++native_color: Fencing allocation score on rhel7-4: 0 ++native_color: Fencing allocation score on rhel7-5: 0 ++native_color: container1 allocation score on lxc1: -INFINITY ++native_color: container1 allocation score on lxc2: -INFINITY ++native_color: container1 allocation score on rhel7-1: -INFINITY ++native_color: container1 allocation score on rhel7-2: 0 ++native_color: container1 allocation score on rhel7-3: 0 ++native_color: container1 allocation score on rhel7-4: 0 ++native_color: container1 allocation score on rhel7-5: 0 ++native_color: container2 allocation score on lxc1: -INFINITY ++native_color: container2 allocation score on lxc2: -INFINITY ++native_color: container2 allocation score on rhel7-1: -INFINITY ++native_color: container2 allocation score on rhel7-2: 0 ++native_color: container2 allocation score on rhel7-3: 0 ++native_color: container2 allocation score on rhel7-4: 0 ++native_color: container2 allocation score on rhel7-5: 0 ++native_color: lxc-ms:0 allocation score on lxc1: INFINITY ++native_color: lxc-ms:0 allocation score on lxc2: INFINITY ++native_color: lxc-ms:0 allocation score on rhel7-1: -INFINITY ++native_color: lxc-ms:0 allocation score on rhel7-2: 0 ++native_color: lxc-ms:0 allocation score on rhel7-3: 0 ++native_color: lxc-ms:0 allocation score on rhel7-4: 0 ++native_color: lxc-ms:0 allocation score on rhel7-5: 0 ++native_color: lxc-ms:1 allocation score on lxc1: -INFINITY ++native_color: lxc-ms:1 allocation score on lxc2: INFINITY ++native_color: lxc-ms:1 allocation score on rhel7-1: -INFINITY ++native_color: lxc-ms:1 allocation score on rhel7-2: 0 ++native_color: lxc-ms:1 allocation score on rhel7-3: 0 ++native_color: lxc-ms:1 allocation score on rhel7-4: 0 ++native_color: lxc-ms:1 allocation score on rhel7-5: 0 ++native_color: lxc1 allocation score on lxc1: -INFINITY ++native_color: lxc1 allocation score on lxc2: -INFINITY ++native_color: lxc1 allocation score on rhel7-1: -INFINITY ++native_color: lxc1 allocation score on rhel7-2: 0 ++native_color: lxc1 allocation score on rhel7-3: -INFINITY ++native_color: lxc1 allocation score on rhel7-4: -INFINITY ++native_color: lxc1 allocation score on rhel7-5: -INFINITY ++native_color: lxc2 allocation score on lxc1: -INFINITY ++native_color: lxc2 allocation score on lxc2: -INFINITY ++native_color: lxc2 allocation score on rhel7-1: -INFINITY ++native_color: lxc2 allocation score on rhel7-2: -INFINITY ++native_color: lxc2 allocation score on rhel7-3: 0 ++native_color: lxc2 allocation score on rhel7-4: -INFINITY ++native_color: lxc2 allocation score on rhel7-5: -INFINITY ++native_color: rsc_rhel7-1 allocation score on lxc1: -INFINITY ++native_color: rsc_rhel7-1 allocation score on lxc2: -INFINITY ++native_color: rsc_rhel7-1 allocation score on rhel7-1: 100 ++native_color: rsc_rhel7-1 allocation score on rhel7-2: 0 ++native_color: rsc_rhel7-1 allocation score on rhel7-3: 0 ++native_color: rsc_rhel7-1 allocation score on rhel7-4: 0 ++native_color: rsc_rhel7-1 allocation score on rhel7-5: 0 +diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary +new file mode 100644 +index 0000000..8a1bfd4 +--- /dev/null ++++ b/pengine/test10/guest-node-host-dies.summary +@@ -0,0 +1,73 @@ ++ ++Current cluster status: ++Node rhel7-1 (1): UNCLEAN (offline) ++Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] ++Containers: [ lxc1:container1 lxc2:container2 ] ++ ++ Fencing (stonith:fence_xvm): Started rhel7-4 ++ rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-1 ( UNCLEAN ) ++ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-1 ( UNCLEAN ) ++ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-1 ( UNCLEAN ) ++ Master/Slave Set: lxc-ms-master [lxc-ms] ++ Masters: [ lxc1 ] ++ Slaves: [ lxc2 ] ++ ++Transition Summary: ++ * Restart Fencing (Started rhel7-4) ++ * Move rsc_rhel7-1 (Started rhel7-1 -> rhel7-5) ++ * Move container1 (Started rhel7-1 -> rhel7-2) ++ * Move container2 (Started rhel7-1 -> rhel7-3) ++ * Restart lxc-ms:0 (Master lxc1) ++ * Restart lxc-ms:1 (Slave lxc2) ++ * Move lxc1 (Started rhel7-1 -> rhel7-2) ++ * Move lxc2 (Started rhel7-1 -> rhel7-3) ++ ++Executing cluster transition: ++ * Resource action: Fencing stop on rhel7-4 ++ * Resource action: Fencing start on rhel7-4 ++ * Resource action: Fencing monitor=120000 on rhel7-4 ++ * Pseudo action: lxc-ms-master_demote_0 ++ * Fencing rhel7-1 (reboot) ++ * Pseudo action: stonith_complete ++ * Pseudo action: rsc_rhel7-1_stop_0 ++ * Resource action: lxc-ms demote on lxc1 ++ * Pseudo action: lxc-ms-master_demoted_0 ++ * Pseudo action: lxc-ms-master_stop_0 ++ * Pseudo action: lxc1_stop_0 ++ * Pseudo action: lxc2_stop_0 ++ * Resource action: rsc_rhel7-1 start on rhel7-5 ++ * Pseudo action: container1_stop_0 ++ * Pseudo action: container2_stop_0 ++ * Pseudo action: lxc-ms_stop_0 ++ * Pseudo action: lxc-ms_stop_0 ++ * Pseudo action: lxc-ms-master_stopped_0 ++ * Pseudo action: lxc-ms-master_start_0 ++ * Pseudo action: all_stopped ++ * Resource action: rsc_rhel7-1 monitor=5000 on rhel7-5 ++ * Resource action: container1 start on rhel7-2 ++ * Resource action: container2 start on rhel7-3 ++ * Resource action: lxc1 start on rhel7-2 ++ * Resource action: lxc2 start on rhel7-3 ++ * Resource action: lxc-ms start on lxc1 ++ * Resource action: lxc-ms start on lxc2 ++ * Resource action: lxc-ms monitor=10000 on lxc2 ++ * Pseudo action: lxc-ms-master_running_0 ++ * Resource action: lxc1 monitor=30000 on rhel7-2 ++ * Resource action: lxc2 monitor=30000 on rhel7-3 ++ * Pseudo action: lxc-ms-master_promote_0 ++ * Resource action: lxc-ms promote on lxc1 ++ * Pseudo action: lxc-ms-master_promoted_0 ++ ++Revised cluster status: ++Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] ++OFFLINE: [ rhel7-1 ] ++Containers: [ lxc1:container1 lxc2:container2 ] ++ ++ Fencing (stonith:fence_xvm): Started rhel7-4 ++ rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-5 ++ container1 (ocf::heartbeat:VirtualDomain): Started rhel7-2 ++ container2 (ocf::heartbeat:VirtualDomain): Started rhel7-3 ++ Master/Slave Set: lxc-ms-master [lxc-ms] ++ Masters: [ lxc1 ] ++ Slaves: [ lxc2 ] ++ +diff --git a/pengine/test10/guest-node-host-dies.xml b/pengine/test10/guest-node-host-dies.xml +new file mode 100644 +index 0000000..a840da1 +--- /dev/null ++++ b/pengine/test10/guest-node-host-dies.xml +@@ -0,0 +1,294 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +-- +1.8.3.1 + + +From beab7718e14a54f1b50d7c5ff4b0086e09332da3 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 15 Apr 2016 13:10:17 -0500 +Subject: [PATCH 5/6] Fix: pengine: create a pseudo-fence for guest node + recovery + +If a guest node needs to be recovered, the PE would previously order actions +in relation to the stop action for the guest's container resource, if one +was scheduled. + +This had problems: for implied stops due to fencing the guest's host, there +would be no stop action, so no ordering could be done; ordering in relation to +the stop action made stonith_constraints() mistakenly assume that the host node +(the node for the stop action) was the fence target, and thus mistakenly mark +the wrong stops/demotes as implied; and, clone notifications for fence events +would not get called for guest node recoveries, whether explicit or implied. + +Now, a fence pseudo-event is created for guest node recovery, regardless of +whether there is an explicit stop action scheduled for the container. This +addresses all those issues, and will allow the crmd to be able to detect +implied stops. + +This also allows us to simplify the implied stop/demote detection, since we +will check the pseudo-op for implied actions -- we don't need to check the +real fence op for implied actions on guest nodes. +--- + crmd/te_utils.c | 8 ++++++ + pengine/allocate.c | 82 ++++++++++++++++++++++++++++++++++++++++++++---------- + pengine/graph.c | 14 ++++------ + pengine/native.c | 48 ++------------------------------ + 4 files changed, 83 insertions(+), 69 deletions(-) + +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index 4c708a1..e7bf7ff 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -331,6 +331,14 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) + /* The DC always sends updates */ + send_stonith_update(NULL, st_event->target, uuid); + ++ /* @TODO Ideally, at this point, we'd check whether the fenced node ++ * hosted any guest nodes, and call remote_node_down() for them. ++ * Unfortunately, the crmd doesn't have a simple, reliable way to ++ * map hosts to guests. It might be possible to track this in the ++ * peer cache via crm_remote_peer_cache_refresh(). For now, we rely ++ * on the PE creating fence pseudo-events for the guests. ++ */ ++ + if (st_event->client_origin && safe_str_neq(st_event->client_origin, te_client_id)) { + + /* Abort the current transition graph if it wasn't us +diff --git a/pengine/allocate.c b/pengine/allocate.c +index bdf03e5..74b57fb 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1341,6 +1341,70 @@ any_managed_resources(pe_working_set_t * data_set) + return FALSE; + } + ++/*! ++ * \internal ++ * \brief Create pseudo-op for guest node fence, and order relative to it ++ * ++ * \param[in] node Guest node to fence ++ * \param[in] done STONITH_DONE operation ++ * \param[in] data_set Working set of CIB state ++ */ ++static void ++fence_guest(pe_node_t *node, pe_action_t *done, pe_working_set_t *data_set) ++{ ++ resource_t *container = node->details->remote_rsc->container; ++ pe_action_t *stop = NULL; ++ pe_action_t *stonith_op = NULL; ++ ++ /* The fence action is just a label; we don't do anything differently for ++ * off vs. reboot. We specify it explicitly, rather than let it default to ++ * cluster's default action, because we are not _initiating_ fencing -- we ++ * are creating a pseudo-event to describe fencing that is already occurring ++ * by other means (container recovery). ++ */ ++ const char *fence_action = "off"; ++ ++ /* Check whether guest's container resource is has any explicit stop or ++ * start (the stop may be implied by fencing of the guest's host). ++ */ ++ if (container) { ++ stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL); ++ ++ if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) { ++ fence_action = "reboot"; ++ } ++ } ++ ++ /* Create a fence pseudo-event, so we have an event to order actions ++ * against, and crmd can always detect it. ++ */ ++ stonith_op = pe_fence_op(node, fence_action, FALSE, data_set); ++ update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable); ++ ++ ++ /* We want to imply stops/demotes after the guest is stopped, not wait until ++ * it is restarted, so we always order pseudo-fencing after stop, not start ++ * (even though start might be closer to what is done for a real reboot). ++ */ ++ if (stop) { ++ order_actions(stop, stonith_op, ++ pe_order_runnable_left|pe_order_implies_then); ++ crm_info("Implying guest node %s is down (action %d) " ++ "after container %s is stopped (action %d)", ++ node->details->uname, stonith_op->id, ++ container->id, stop->id); ++ } else { ++ crm_info("Implying guest node %s is down (action %d) ", ++ node->details->uname, stonith_op->id); ++ } ++ ++ /* @TODO: Order pseudo-fence after any (optional) fence of guest's host */ ++ ++ /* Order/imply other actions relative to pseudo-fence as with real fence */ ++ stonith_constraints(node, stonith_op, data_set); ++ order_actions(stonith_op, done, pe_order_implies_then); ++} ++ + /* + * Create dependencies for stonith and shutdown operations + */ +@@ -1369,24 +1433,12 @@ stage6(pe_working_set_t * data_set) + for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) { + node_t *node = (node_t *) gIter->data; + +- /* Guest nodes are "fenced" by recovering their container resource. +- * The container stop may be explicit, or implied by the fencing of the +- * guest's host. ++ /* Guest nodes are "fenced" by recovering their container resource, ++ * so handle them separately. + */ + if (is_container_remote_node(node)) { + if (node->details->remote_requires_reset && need_stonith) { +- resource_t *container = node->details->remote_rsc->container; +- char *key = stop_key(container); +- GListPtr stop_list = find_actions(container->actions, key, NULL); +- +- crm_info("Implying node %s is down when container %s is stopped (%p)", +- node->details->uname, container->id, stop_list); +- if(stop_list) { +- stonith_constraints(node, stop_list->data, data_set); +- } +- +- g_list_free(stop_list); +- free(key); ++ fence_guest(node, done, data_set); + } + continue; + } +diff --git a/pengine/graph.c b/pengine/graph.c +index ee7c7c8..569cf6e 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -715,13 +715,7 @@ stonith_constraints(node_t * node, action_t * stonith_op, pe_working_set_t * dat + + CRM_CHECK(stonith_op != NULL, return FALSE); + for (r = data_set->resources; r != NULL; r = r->next) { +- resource_t *rsc = (resource_t *) r->data; +- +- if ((stonith_op->rsc == NULL) +- || ((stonith_op->rsc != rsc) && (stonith_op->rsc != rsc->container))) { +- +- rsc_stonith_ordering(rsc, stonith_op, data_set); +- } ++ rsc_stonith_ordering((resource_t *) r->data, stonith_op, data_set); + } + return TRUE; + } +@@ -888,7 +882,11 @@ action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set) + } + + if (safe_str_eq(action->task, CRM_OP_FENCE)) { +- action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); ++ /* All fences need node info; guest node fences are pseudo-events */ ++ action_xml = create_xml_node(NULL, ++ is_set(action->flags, pe_action_pseudo)? ++ XML_GRAPH_TAG_PSEUDO_EVENT : ++ XML_GRAPH_TAG_CRM_EVENT); + + } else if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) { + action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT); +diff --git a/pengine/native.c b/pengine/native.c +index 56a1434..ff4467b 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -2902,48 +2902,6 @@ native_start_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set + } + } + +-/* User data to pass to guest node iterator */ +-struct action_list_s { +- GListPtr search_list; /* list of actions to search */ +- GListPtr result_list; /* list of matching actions for this node */ +- const char *key; /* action key to match */ +-}; +- +-/*! +- * \internal +- * \brief Prepend a node's actions matching a key to a list +- * +- * \param[in] node Guest node +- * \param[in/out] data User data +- */ +-static void prepend_node_actions(const node_t *node, void *data) +-{ +- GListPtr actions; +- struct action_list_s *info = (struct action_list_s *) data; +- +- actions = find_actions(info->search_list, info->key, node); +- info->result_list = g_list_concat(actions, info->result_list); +-} +- +-static GListPtr +-find_fence_target_node_actions(GListPtr search_list, const char *key, node_t *fence_target, pe_working_set_t *data_set) +-{ +- struct action_list_s action_list; +- +- /* Actions on the target that match the key are implied by the fencing */ +- action_list.search_list = search_list; +- action_list.result_list = find_actions(search_list, key, fence_target); +- action_list.key = key; +- +- /* +- * If the target is a host for any guest nodes, actions on those nodes +- * that match the key are also implied by the fencing. +- */ +- pe_foreach_guest_node(data_set, fence_target, prepend_node_actions, &action_list); +- +- return action_list.result_list; +-} +- + static void + native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set) + { +@@ -2963,8 +2921,7 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_ + + /* Get a list of stop actions potentially implied by the fencing */ + key = stop_key(rsc); +- action_list = find_fence_target_node_actions(rsc->actions, key, target, +- data_set); ++ action_list = find_actions(rsc->actions, key, target); + free(key); + + for (gIter = action_list; gIter != NULL; gIter = gIter->next) { +@@ -3061,8 +3018,7 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_ + + /* Get a list of demote actions potentially implied by the fencing */ + key = demote_key(rsc); +- action_list = find_fence_target_node_actions(rsc->actions, key, target, +- data_set); ++ action_list = find_actions(rsc->actions, key, target); + free(key); + + for (gIter = action_list; gIter != NULL; gIter = gIter->next) { +-- +1.8.3.1 + + +From b7ce740edf3d71fcccead2288bf0ab11037f9672 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Wed, 23 Nov 2016 14:56:29 -0600 +Subject: [PATCH 6/6] Fix: pengine: consider guest node unclean if its host is + unclean + +--- + lib/pengine/unpack.c | 12 ++++++++++++ + 1 file changed, 12 insertions(+) + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index a49e108..6737273 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -1406,6 +1406,7 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node) + { + resource_t *rsc = this_node->details->remote_rsc; + resource_t *container = NULL; ++ pe_node_t *host = NULL; + + /* If there is a node state entry for a (former) Pacemaker Remote node + * but no resource creating that node, the node's connection resource will +@@ -1418,6 +1419,10 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node) + + container = rsc->container; + ++ if (container && (g_list_length(rsc->running_on) == 1)) { ++ host = rsc->running_on->data; ++ } ++ + /* If the resource is currently started, mark it online. */ + if (rsc->role == RSC_ROLE_STARTED) { + crm_trace("%s node %s presumed ONLINE because connection resource is started", +@@ -1451,6 +1456,13 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node) + (container? "Guest" : "Remote"), this_node->details->id); + this_node->details->online = FALSE; + this_node->details->remote_requires_reset = FALSE; ++ ++ } else if (host && (host->details->online == FALSE) ++ && host->details->unclean) { ++ crm_trace("Guest node %s UNCLEAN because host is unclean", ++ this_node->details->id); ++ this_node->details->online = FALSE; ++ this_node->details->remote_requires_reset = TRUE; + } + + remote_online_done: +-- +1.8.3.1 + diff --git a/SOURCES/017-guest-fencing-tests.patch b/SOURCES/017-guest-fencing-tests.patch new file mode 100644 index 0000000..62b57d9 --- /dev/null +++ b/SOURCES/017-guest-fencing-tests.patch @@ -0,0 +1,2301 @@ +From a9977a2dc1135d040088b90e1ea307b3ff71c0b5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 22 Mar 2016 16:43:53 -0500 +Subject: [PATCH] Test: pengine: update regression tests for guest node + pseudo-fencing + +--- + pengine/test10/bug-cl-5247.dot | 45 +++-- + pengine/test10/bug-cl-5247.summary | 18 +- + pengine/test10/bug-rh-1097457.dot | 23 ++- + pengine/test10/bug-rh-1097457.exp | 70 ++++++-- + pengine/test10/bug-rh-1097457.summary | 12 +- + pengine/test10/guest-node-host-dies.dot | 17 +- + pengine/test10/guest-node-host-dies.exp | 190 +++++++++++++-------- + pengine/test10/guest-node-host-dies.summary | 48 +++--- + pengine/test10/whitebox-fail1.dot | 15 +- + pengine/test10/whitebox-fail1.exp | 50 +++++- + pengine/test10/whitebox-fail1.summary | 12 +- + pengine/test10/whitebox-fail2.dot | 15 +- + pengine/test10/whitebox-fail2.exp | 50 +++++- + pengine/test10/whitebox-fail2.summary | 12 +- + pengine/test10/whitebox-imply-stop-on-fence.dot | 9 +- + pengine/test10/whitebox-imply-stop-on-fence.exp | 177 +++++++++++-------- + .../test10/whitebox-imply-stop-on-fence.summary | 22 +-- + pengine/test10/whitebox-ms-ordering.dot | 18 ++ + pengine/test10/whitebox-ms-ordering.exp | 76 +++++++++ + pengine/test10/whitebox-ms-ordering.summary | 3 + + pengine/test10/whitebox-unexpectedly-running.dot | 5 + + pengine/test10/whitebox-unexpectedly-running.exp | 30 ++++ + .../test10/whitebox-unexpectedly-running.summary | 2 + + 24 files changed, 748 insertions(+), 292 deletions(-) + +diff --git a/pengine/test10/bug-cl-5247.dot b/pengine/test10/bug-cl-5247.dot +index ed728ac..0ab7893 100644 +--- a/pengine/test10/bug-cl-5247.dot ++++ b/pengine/test10/bug-cl-5247.dot +@@ -1,4 +1,6 @@ + digraph "g" { ++"all_stopped" -> "prmStonith1-2_start_0 bl460g8n4" [ style = bold] ++"all_stopped" -> "prmStonith2-2_start_0 bl460g8n3" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "grpStonith1_running_0" [ style=bold color="green" fontcolor="orange"] + "grpStonith1_start_0" -> "grpStonith1_running_0" [ style = bold] +@@ -52,7 +54,7 @@ digraph "g" { + "msPostgresql_post_notify_demoted_0" -> "pgsql_post_notify_demoted_0 pgsr01" [ style = bold] + "msPostgresql_post_notify_demoted_0" [ style=bold color="green" fontcolor="orange"] + "msPostgresql_post_notify_stopped_0" -> "msPostgresql_confirmed-post_notify_stopped_0" [ style = bold] +-"msPostgresql_post_notify_stopped_0" -> "pgsql_post_notify_stop_0 pgsr01" [ style = bold] ++"msPostgresql_post_notify_stopped_0" -> "pgsql_post_notify_stonith_0 pgsr01" [ style = bold] + "msPostgresql_post_notify_stopped_0" [ style=bold color="green" fontcolor="orange"] + "msPostgresql_pre_notify_demote_0" -> "msPostgresql_confirmed-pre_notify_demote_0" [ style = bold] + "msPostgresql_pre_notify_demote_0" -> "pgsql_pre_notify_demote_0 pgsr01" [ style = bold] +@@ -65,21 +67,21 @@ digraph "g" { + "msPostgresql_stop_0" [ style=bold color="green" fontcolor="orange"] + "msPostgresql_stopped_0" -> "msPostgresql_post_notify_stopped_0" [ style = bold] + "msPostgresql_stopped_0" [ style=bold color="green" fontcolor="orange"] +-"pgsql_confirmed-post_notify_stop_0" -> "all_stopped" [ style = bold] +-"pgsql_confirmed-post_notify_stop_0" -> "pgsql_monitor_9000 pgsr01" [ style = bold] +-"pgsql_confirmed-post_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"pgsql_confirmed-post_notify_stonith_0" -> "all_stopped" [ style = bold] ++"pgsql_confirmed-post_notify_stonith_0" -> "pgsql_monitor_9000 pgsr01" [ style = bold] ++"pgsql_confirmed-post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] + "pgsql_demote_0 pgsr02" -> "msPostgresql_demoted_0" [ style = bold] + "pgsql_demote_0 pgsr02" -> "pgsql_stop_0 pgsr02" [ style = bold] + "pgsql_demote_0 pgsr02" [ style=bold color="green" fontcolor="orange"] + "pgsql_monitor_9000 pgsr01" [ style=bold color="green" fontcolor="black"] + "pgsql_post_notify_demoted_0 pgsr01" -> "msPostgresql_confirmed-post_notify_demoted_0" [ style = bold] + "pgsql_post_notify_demoted_0 pgsr01" [ style=bold color="green" fontcolor="black"] +-"pgsql_post_notify_stop_0 pgsr01" -> "msPostgresql_confirmed-post_notify_stopped_0" [ style = bold] +-"pgsql_post_notify_stop_0 pgsr01" -> "pgsql_confirmed-post_notify_stop_0" [ style = bold] +-"pgsql_post_notify_stop_0 pgsr01" [ style=bold color="green" fontcolor="black"] +-"pgsql_post_notify_stop_0" -> "pgsql_confirmed-post_notify_stop_0" [ style = bold] +-"pgsql_post_notify_stop_0" -> "pgsql_post_notify_stop_0 pgsr01" [ style = bold] +-"pgsql_post_notify_stop_0" [ style=bold color="green" fontcolor="orange"] ++"pgsql_post_notify_stonith_0 pgsr01" -> "msPostgresql_confirmed-post_notify_stopped_0" [ style = bold] ++"pgsql_post_notify_stonith_0 pgsr01" -> "pgsql_confirmed-post_notify_stonith_0" [ style = bold] ++"pgsql_post_notify_stonith_0 pgsr01" [ style=bold color="green" fontcolor="black"] ++"pgsql_post_notify_stonith_0" -> "pgsql_confirmed-post_notify_stonith_0" [ style = bold] ++"pgsql_post_notify_stonith_0" -> "pgsql_post_notify_stonith_0 pgsr01" [ style = bold] ++"pgsql_post_notify_stonith_0" [ style=bold color="green" fontcolor="orange"] + "pgsql_pre_notify_demote_0 pgsr01" -> "msPostgresql_confirmed-pre_notify_demote_0" [ style = bold] + "pgsql_pre_notify_demote_0 pgsr01" [ style=bold color="green" fontcolor="black"] + "pgsql_pre_notify_stop_0 pgsr01" -> "msPostgresql_confirmed-pre_notify_stop_0" [ style = bold] +@@ -91,13 +93,7 @@ digraph "g" { + "pgsr02_stop_0 bl460g8n4" -> "prmDB2_stop_0 bl460g8n4" [ style = bold] + "pgsr02_stop_0 bl460g8n4" [ style=bold color="green" fontcolor="black"] + "prmDB2_stop_0 bl460g8n4" -> "all_stopped" [ style = bold] +-"prmDB2_stop_0 bl460g8n4" -> "master-group_stop_0" [ style = bold] +-"prmDB2_stop_0 bl460g8n4" -> "msPostgresql_stop_0" [ style = bold] +-"prmDB2_stop_0 bl460g8n4" -> "pgsql_demote_0 pgsr02" [ style = bold] +-"prmDB2_stop_0 bl460g8n4" -> "pgsql_post_notify_stop_0" [ style = bold] +-"prmDB2_stop_0 bl460g8n4" -> "pgsql_stop_0 pgsr02" [ style = bold] +-"prmDB2_stop_0 bl460g8n4" -> "vip-master_stop_0 pgsr02" [ style = bold] +-"prmDB2_stop_0 bl460g8n4" -> "vip-rep_stop_0 pgsr02" [ style = bold] ++"prmDB2_stop_0 bl460g8n4" -> "stonith 'off' pgsr02" [ style = bold] + "prmDB2_stop_0 bl460g8n4" [ style=bold color="green" fontcolor="black"] + "prmStonith1-2_monitor_3600000 bl460g8n4" [ style=bold color="green" fontcolor="black"] + "prmStonith1-2_start_0 bl460g8n4" -> "grpStonith1_running_0" [ style = bold] +@@ -106,7 +102,7 @@ digraph "g" { + "prmStonith1-2_stop_0 bl460g8n4" -> "all_stopped" [ style = bold] + "prmStonith1-2_stop_0 bl460g8n4" -> "grpStonith1_stopped_0" [ style = bold] + "prmStonith1-2_stop_0 bl460g8n4" -> "prmStonith1-2_start_0 bl460g8n4" [ style = bold] +-"prmStonith1-2_stop_0 bl460g8n4" [ style=bold color="green" fontcolor="orange"] ++"prmStonith1-2_stop_0 bl460g8n4" [ style=bold color="green" fontcolor="black"] + "prmStonith2-2_monitor_3600000 bl460g8n3" [ style=bold color="green" fontcolor="black"] + "prmStonith2-2_start_0 bl460g8n3" -> "grpStonith2_running_0" [ style = bold] + "prmStonith2-2_start_0 bl460g8n3" -> "prmStonith2-2_monitor_3600000 bl460g8n3" [ style = bold] +@@ -115,6 +111,19 @@ digraph "g" { + "prmStonith2-2_stop_0 bl460g8n3" -> "grpStonith2_stopped_0" [ style = bold] + "prmStonith2-2_stop_0 bl460g8n3" -> "prmStonith2-2_start_0 bl460g8n3" [ style = bold] + "prmStonith2-2_stop_0 bl460g8n3" [ style=bold color="green" fontcolor="black"] ++"stonith 'off' pgsr02" -> "master-group_stop_0" [ style = bold] ++"stonith 'off' pgsr02" -> "msPostgresql_stop_0" [ style = bold] ++"stonith 'off' pgsr02" -> "pgsql_demote_0 pgsr02" [ style = bold] ++"stonith 'off' pgsr02" -> "pgsql_post_notify_stonith_0" [ style = bold] ++"stonith 'off' pgsr02" -> "pgsql_stop_0 pgsr02" [ style = bold] ++"stonith 'off' pgsr02" -> "stonith_complete" [ style = bold] ++"stonith 'off' pgsr02" -> "vip-master_stop_0 pgsr02" [ style = bold] ++"stonith 'off' pgsr02" -> "vip-rep_stop_0 pgsr02" [ style = bold] ++"stonith 'off' pgsr02" [ style=bold color="green" fontcolor="orange"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "vip-master_start_0 pgsr01" [ style = bold] ++"stonith_complete" -> "vip-rep_start_0 pgsr01" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] + "vip-master_monitor_10000 pgsr01" [ style=bold color="green" fontcolor="black"] + "vip-master_start_0 pgsr01" -> "master-group_running_0" [ style = bold] + "vip-master_start_0 pgsr01" -> "vip-master_monitor_10000 pgsr01" [ style = bold] +diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary +index 5564286..09dc301 100644 +--- a/pengine/test10/bug-cl-5247.summary ++++ b/pengine/test10/bug-cl-5247.summary +@@ -28,7 +28,7 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: grpStonith1_stop_0 +- * Pseudo action: prmStonith1-2_stop_0 ++ * Resource action: prmStonith1-2 stop on bl460g8n4 + * Pseudo action: grpStonith2_stop_0 + * Resource action: prmStonith2-2 stop on bl460g8n3 + * Pseudo action: msPostgresql_pre_notify_demote_0 +@@ -36,18 +36,14 @@ Executing cluster transition: + * Resource action: prmDB2 stop on bl460g8n4 + * Pseudo action: grpStonith1_stopped_0 + * Pseudo action: grpStonith1_start_0 +- * Resource action: prmStonith1-2 start on bl460g8n4 +- * Resource action: prmStonith1-2 monitor=3600000 on bl460g8n4 + * Pseudo action: grpStonith2_stopped_0 + * Pseudo action: grpStonith2_start_0 +- * Resource action: prmStonith2-2 start on bl460g8n3 +- * Resource action: prmStonith2-2 monitor=3600000 on bl460g8n3 +- * Pseudo action: pgsql_post_notify_stop_0 + * Resource action: pgsql notify on pgsr01 + * Pseudo action: msPostgresql_confirmed-pre_notify_demote_0 + * Pseudo action: msPostgresql_demote_0 +- * Pseudo action: grpStonith1_running_0 +- * Pseudo action: grpStonith2_running_0 ++ * Pseudo action: stonith-pgsr02-off on pgsr02 ++ * Pseudo action: stonith_complete ++ * Pseudo action: pgsql_post_notify_stop_0 + * Pseudo action: pgsql_demote_0 + * Pseudo action: msPostgresql_demoted_0 + * Pseudo action: msPostgresql_post_notify_demoted_0 +@@ -75,6 +71,12 @@ Executing cluster transition: + * Pseudo action: pgsql_notified_0 + * Resource action: pgsql monitor=9000 on pgsr01 + * Pseudo action: all_stopped ++ * Resource action: prmStonith1-2 start on bl460g8n4 ++ * Resource action: prmStonith1-2 monitor=3600000 on bl460g8n4 ++ * Resource action: prmStonith2-2 start on bl460g8n3 ++ * Resource action: prmStonith2-2 monitor=3600000 on bl460g8n3 ++ * Pseudo action: grpStonith1_running_0 ++ * Pseudo action: grpStonith2_running_0 + Using the original execution date of: 2015-08-12 02:53:40Z + + Revised cluster status: +diff --git a/pengine/test10/bug-rh-1097457.dot b/pengine/test10/bug-rh-1097457.dot +index 078d177..ece2834 100644 +--- a/pengine/test10/bug-rh-1097457.dot ++++ b/pengine/test10/bug-rh-1097457.dot +@@ -47,14 +47,9 @@ digraph "g" { + "VM2_start_0 lama3" -> "VM2_monitor_10000 lama3" [ style = bold] + "VM2_start_0 lama3" -> "lamaVM2_start_0 lama3" [ style = bold] + "VM2_start_0 lama3" [ style=bold color="green" fontcolor="black"] +-"VM2_stop_0 lama3" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] +-"VM2_stop_0 lama3" -> "FAKE4_stop_0 lamaVM2" [ style = bold] +-"VM2_stop_0 lama3" -> "FAKE6-clone_stop_0" [ style = bold] +-"VM2_stop_0 lama3" -> "FAKE6_stop_0 lamaVM2" [ style = bold] +-"VM2_stop_0 lama3" -> "FSlun3_stop_0 lamaVM2" [ style = bold] + "VM2_stop_0 lama3" -> "VM2_start_0 lama3" [ style = bold] + "VM2_stop_0 lama3" -> "all_stopped" [ style = bold] +-"VM2_stop_0 lama3" -> "lamaVM2-G4_stop_0" [ style = bold] ++"VM2_stop_0 lama3" -> "stonith 'reboot' lamaVM2" [ style = bold] + "VM2_stop_0 lama3" [ style=bold color="green" fontcolor="black"] + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "lamaVM2-G4_running_0" [ style=bold color="green" fontcolor="orange"] +@@ -83,4 +78,20 @@ digraph "g" { + "lamaVM2_stop_0 lama3" -> "all_stopped" [ style = bold] + "lamaVM2_stop_0 lama3" -> "lamaVM2_start_0 lama3" [ style = bold] + "lamaVM2_stop_0 lama3" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' lamaVM2" -> "FAKE4-IP_stop_0 lamaVM2" [ style = bold] ++"stonith 'reboot' lamaVM2" -> "FAKE4_stop_0 lamaVM2" [ style = bold] ++"stonith 'reboot' lamaVM2" -> "FAKE6-clone_stop_0" [ style = bold] ++"stonith 'reboot' lamaVM2" -> "FAKE6_stop_0 lamaVM2" [ style = bold] ++"stonith 'reboot' lamaVM2" -> "FSlun3_stop_0 lamaVM2" [ style = bold] ++"stonith 'reboot' lamaVM2" -> "lamaVM2-G4_stop_0" [ style = bold] ++"stonith 'reboot' lamaVM2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' lamaVM2" [ style=bold color="green" fontcolor="orange"] ++"stonith_complete" -> "FAKE4-IP_start_0 lamaVM2" [ style = bold] ++"stonith_complete" -> "FAKE4_start_0 lamaVM2" [ style = bold] ++"stonith_complete" -> "FAKE6_start_0 lamaVM2" [ style = bold] ++"stonith_complete" -> "FSlun3_start_0 lama2" [ style = bold] ++"stonith_complete" -> "VM2_start_0 lama3" [ style = bold] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "lamaVM2_start_0 lama3" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/bug-rh-1097457.exp b/pengine/test10/bug-rh-1097457.exp +index 94a4e8d..0c3430c 100644 +--- a/pengine/test10/bug-rh-1097457.exp ++++ b/pengine/test10/bug-rh-1097457.exp +@@ -10,6 +10,9 @@ + + + ++ ++ ++ + + + +@@ -65,6 +68,9 @@ + + + ++ ++ ++ + + + +@@ -75,10 +81,10 @@ + + + +- ++ + + +- ++ + + + +@@ -121,7 +127,7 @@ + + + +- ++ + + + +@@ -175,6 +181,9 @@ + + + ++ ++ ++ + + + +@@ -185,14 +194,14 @@ + + + +- +- +- + + + + + ++ ++ ++ + + + +@@ -231,6 +240,9 @@ + + + ++ ++ ++ + + + +@@ -241,10 +253,10 @@ + + + +- ++ + + +- ++ + + + +@@ -281,6 +293,9 @@ + + + ++ ++ ++ + + + +@@ -291,10 +306,10 @@ + + + +- ++ + + +- ++ + + + +@@ -337,7 +352,7 @@ + + + +- ++ + + + +@@ -382,6 +397,9 @@ + + + ++ ++ ++ + + + +@@ -411,6 +429,33 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -434,6 +479,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary +index d78d951..f8d2c14 100644 +--- a/pengine/test10/bug-rh-1097457.summary ++++ b/pengine/test10/bug-rh-1097457.summary +@@ -42,18 +42,20 @@ Transition Summary: + Executing cluster transition: + * Resource action: lamaVM2 stop on lama3 + * Resource action: VM2 stop on lama3 ++ * Pseudo action: stonith-lamaVM2-reboot on lamaVM2 ++ * Pseudo action: stonith_complete ++ * Resource action: VM2 start on lama3 ++ * Resource action: VM2 monitor=10000 on lama3 + * Pseudo action: lamaVM2-G4_stop_0 + * Pseudo action: FAKE4-IP_stop_0 + * Pseudo action: FAKE6-clone_stop_0 +- * Resource action: VM2 start on lama3 +- * Resource action: VM2 monitor=10000 on lama3 ++ * Resource action: lamaVM2 start on lama3 ++ * Resource action: lamaVM2 monitor=30000 on lama3 ++ * Resource action: FSlun3 monitor=10000 on lamaVM2 + * Pseudo action: FAKE4_stop_0 + * Pseudo action: FAKE6_stop_0 + * Pseudo action: FAKE6-clone_stopped_0 + * Pseudo action: FAKE6-clone_start_0 +- * Resource action: lamaVM2 start on lama3 +- * Resource action: lamaVM2 monitor=30000 on lama3 +- * Resource action: FSlun3 monitor=10000 on lamaVM2 + * Pseudo action: lamaVM2-G4_stopped_0 + * Resource action: FAKE6 start on lamaVM2 + * Resource action: FAKE6 monitor=30000 on lamaVM2 +diff --git a/pengine/test10/guest-node-host-dies.dot b/pengine/test10/guest-node-host-dies.dot +index 01858b3..c1ced94 100644 +--- a/pengine/test10/guest-node-host-dies.dot ++++ b/pengine/test10/guest-node-host-dies.dot +@@ -5,16 +5,19 @@ digraph "g" { + "Fencing_stop_0 rhel7-4" -> "Fencing_start_0 rhel7-4" [ style = bold] + "Fencing_stop_0 rhel7-4" -> "all_stopped" [ style = bold] + "Fencing_stop_0 rhel7-4" [ style=bold color="green" fontcolor="black"] ++"all_stopped" -> "Fencing_start_0 rhel7-4" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "container1_start_0 rhel7-2" -> "lxc1_start_0 rhel7-2" [ style = bold] + "container1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] + "container1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "container1_stop_0 rhel7-1" -> "container1_start_0 rhel7-2" [ style = bold] ++"container1_stop_0 rhel7-1" -> "stonith 'reboot' lxc1" [ style = bold] + "container1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] + "container2_start_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold] + "container2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"] + "container2_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "container2_stop_0 rhel7-1" -> "container2_start_0 rhel7-3" [ style = bold] ++"container2_stop_0 rhel7-1" -> "stonith 'reboot' lxc2" [ style = bold] + "container2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] + "lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold] + "lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc1" [ style = bold] +@@ -42,7 +45,7 @@ digraph "g" { + "lxc-ms_demote_0 lxc1" -> "lxc-ms-master_demoted_0" [ style = bold] + "lxc-ms_demote_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold] + "lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] +-"lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="black"] ++"lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="orange"] + "lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"] + "lxc-ms_promote_0 lxc1" -> "lxc-ms-master_promoted_0" [ style = bold] + "lxc-ms_promote_0 lxc1" [ style=bold color="green" fontcolor="black"] +@@ -84,11 +87,17 @@ digraph "g" { + "rsc_rhel7-1_stop_0 rhel7-1" -> "all_stopped" [ style = bold] + "rsc_rhel7-1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold] + "rsc_rhel7-1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"] ++"stonith 'reboot' lxc1" -> "lxc-ms-master_stop_0" [ style = bold] ++"stonith 'reboot' lxc1" -> "lxc-ms_demote_0 lxc1" [ style = bold] ++"stonith 'reboot' lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] ++"stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] ++"stonith 'reboot' lxc2" -> "lxc-ms-master_stop_0" [ style = bold] ++"stonith 'reboot' lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold] ++"stonith 'reboot' lxc2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' lxc2" [ style=bold color="green" fontcolor="orange"] + "stonith 'reboot' rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold] + "stonith 'reboot' rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold] +-"stonith 'reboot' rhel7-1" -> "lxc-ms-master_stop_0" [ style = bold] +-"stonith 'reboot' rhel7-1" -> "lxc-ms_stop_0 lxc1" [ style = bold] +-"stonith 'reboot' rhel7-1" -> "lxc-ms_stop_0 lxc2" [ style = bold] + "stonith 'reboot' rhel7-1" -> "lxc1_stop_0 rhel7-1" [ style = bold] + "stonith 'reboot' rhel7-1" -> "lxc2_stop_0 rhel7-1" [ style = bold] + "stonith 'reboot' rhel7-1" -> "rsc_rhel7-1_stop_0 rhel7-1" [ style = bold] +diff --git a/pengine/test10/guest-node-host-dies.exp b/pengine/test10/guest-node-host-dies.exp +index b3c24be..6bd7a60 100644 +--- a/pengine/test10/guest-node-host-dies.exp ++++ b/pengine/test10/guest-node-host-dies.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -10,14 +10,17 @@ + + + +- ++ + + + + + + +- ++ ++ ++ ++ + + + +@@ -30,33 +33,33 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -65,26 +68,26 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -93,7 +96,7 @@ + + + +- ++ + + + +@@ -102,20 +105,20 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -124,7 +127,7 @@ + + + +- ++ + + + +@@ -133,23 +136,23 @@ + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + +- ++ + + + +@@ -164,69 +167,72 @@ + + + +- ++ ++ + +- ++ + + + +- ++ + + +- ++ ++ ++ ++ + + +- ++ + + + + + +- +- ++ + +- ++ + + + +- ++ ++ ++ ++ ++ ++ ++ + + + + + +- +- ++ + +- ++ + + + +- +- +- +- ++ + + +- +- +- +- ++ + + + + + +- +- +- ++ ++ ++ ++ + + + +- ++ + + +- ++ + + + +@@ -239,7 +245,7 @@ + + + +- ++ + + + +@@ -254,17 +260,16 @@ + + + +- +- +- +- ++ ++ ++ + + + +- ++ + + +- ++ + + + +@@ -276,7 +281,7 @@ + + + +- ++ + + + +@@ -299,7 +304,7 @@ + + + +- ++ + + + +@@ -329,10 +334,10 @@ + + + +- ++ + + +- ++ + + + +@@ -350,7 +355,10 @@ + + + +- ++ ++ ++ ++ + + + +@@ -362,7 +370,7 @@ + + + +- ++ + + + +@@ -409,7 +417,7 @@ + + + +- ++ + + + +@@ -427,7 +435,7 @@ + + + +- ++ + + + +@@ -453,7 +461,7 @@ + + + +- ++ + + + +@@ -471,13 +479,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -490,40 +498,76 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + + + + +- ++ ++ ++ ++ ++ ++ ++ + + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary +index 8a1bfd4..83af5ce 100644 +--- a/pengine/test10/guest-node-host-dies.summary ++++ b/pengine/test10/guest-node-host-dies.summary +@@ -2,58 +2,58 @@ + Current cluster status: + Node rhel7-1 (1): UNCLEAN (offline) + Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] +-Containers: [ lxc1:container1 lxc2:container2 ] + + Fencing (stonith:fence_xvm): Started rhel7-4 + rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-1 ( UNCLEAN ) +- container1 (ocf::heartbeat:VirtualDomain): Started rhel7-1 ( UNCLEAN ) +- container2 (ocf::heartbeat:VirtualDomain): Started rhel7-1 ( UNCLEAN ) ++ container1 (ocf::heartbeat:VirtualDomain): FAILED rhel7-1 (UNCLEAN) ++ container2 (ocf::heartbeat:VirtualDomain): FAILED rhel7-1 (UNCLEAN) + Master/Slave Set: lxc-ms-master [lxc-ms] +- Masters: [ lxc1 ] +- Slaves: [ lxc2 ] ++ Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ] + + Transition Summary: + * Restart Fencing (Started rhel7-4) + * Move rsc_rhel7-1 (Started rhel7-1 -> rhel7-5) +- * Move container1 (Started rhel7-1 -> rhel7-2) +- * Move container2 (Started rhel7-1 -> rhel7-3) +- * Restart lxc-ms:0 (Master lxc1) +- * Restart lxc-ms:1 (Slave lxc2) ++ * Recover container1 (Started rhel7-1 -> rhel7-2) ++ * Recover container2 (Started rhel7-1 -> rhel7-3) ++ * Recover lxc-ms:0 (Master lxc1) ++ * Recover lxc-ms:1 (Slave lxc2) + * Move lxc1 (Started rhel7-1 -> rhel7-2) + * Move lxc2 (Started rhel7-1 -> rhel7-3) + + Executing cluster transition: + * Resource action: Fencing stop on rhel7-4 +- * Resource action: Fencing start on rhel7-4 +- * Resource action: Fencing monitor=120000 on rhel7-4 + * Pseudo action: lxc-ms-master_demote_0 + * Fencing rhel7-1 (reboot) +- * Pseudo action: stonith_complete + * Pseudo action: rsc_rhel7-1_stop_0 +- * Resource action: lxc-ms demote on lxc1 +- * Pseudo action: lxc-ms-master_demoted_0 +- * Pseudo action: lxc-ms-master_stop_0 + * Pseudo action: lxc1_stop_0 + * Pseudo action: lxc2_stop_0 +- * Resource action: rsc_rhel7-1 start on rhel7-5 + * Pseudo action: container1_stop_0 + * Pseudo action: container2_stop_0 ++ * Pseudo action: stonith-lxc2-reboot on lxc2 ++ * Pseudo action: stonith-lxc1-reboot on lxc1 ++ * Pseudo action: stonith_complete ++ * Resource action: rsc_rhel7-1 start on rhel7-5 ++ * Resource action: container1 start on rhel7-2 ++ * Resource action: container2 start on rhel7-3 ++ * Pseudo action: lxc-ms_demote_0 ++ * Pseudo action: lxc-ms-master_demoted_0 ++ * Pseudo action: lxc-ms-master_stop_0 ++ * Resource action: lxc1 start on rhel7-2 ++ * Resource action: lxc2 start on rhel7-3 ++ * Resource action: rsc_rhel7-1 monitor=5000 on rhel7-5 + * Pseudo action: lxc-ms_stop_0 + * Pseudo action: lxc-ms_stop_0 + * Pseudo action: lxc-ms-master_stopped_0 + * Pseudo action: lxc-ms-master_start_0 ++ * Resource action: lxc1 monitor=30000 on rhel7-2 ++ * Resource action: lxc2 monitor=30000 on rhel7-3 + * Pseudo action: all_stopped +- * Resource action: rsc_rhel7-1 monitor=5000 on rhel7-5 +- * Resource action: container1 start on rhel7-2 +- * Resource action: container2 start on rhel7-3 +- * Resource action: lxc1 start on rhel7-2 +- * Resource action: lxc2 start on rhel7-3 ++ * Resource action: Fencing start on rhel7-4 ++ * Resource action: Fencing monitor=120000 on rhel7-4 + * Resource action: lxc-ms start on lxc1 + * Resource action: lxc-ms start on lxc2 +- * Resource action: lxc-ms monitor=10000 on lxc2 + * Pseudo action: lxc-ms-master_running_0 +- * Resource action: lxc1 monitor=30000 on rhel7-2 +- * Resource action: lxc2 monitor=30000 on rhel7-3 ++ * Resource action: lxc-ms monitor=10000 on lxc2 + * Pseudo action: lxc-ms-master_promote_0 + * Resource action: lxc-ms promote on lxc1 + * Pseudo action: lxc-ms-master_promoted_0 +diff --git a/pengine/test10/whitebox-fail1.dot b/pengine/test10/whitebox-fail1.dot +index 0f0fe26..c6380ea 100644 +--- a/pengine/test10/whitebox-fail1.dot ++++ b/pengine/test10/whitebox-fail1.dot +@@ -25,11 +25,9 @@ digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "container1_start_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] + "container1_start_0 18node2" [ style=bold color="green" fontcolor="black"] +-"container1_stop_0 18node2" -> "B_stop_0 lxc1" [ style = bold] +-"container1_stop_0 18node2" -> "M-clone_stop_0" [ style = bold] +-"container1_stop_0 18node2" -> "M_stop_0 lxc1" [ style = bold] + "container1_stop_0 18node2" -> "all_stopped" [ style = bold] + "container1_stop_0 18node2" -> "container1_start_0 18node2" [ style = bold] ++"container1_stop_0 18node2" -> "stonith 'reboot' lxc1" [ style = bold] + "container1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] + "lxc1_monitor_30000 18node2" [ style=bold color="green" fontcolor="black"] + "lxc1_start_0 18node2" -> "B_monitor_10000 lxc1" [ style = bold] +@@ -42,4 +40,15 @@ digraph "g" { + "lxc1_stop_0 18node2" -> "container1_stop_0 18node2" [ style = bold] + "lxc1_stop_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] + "lxc1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' lxc1" -> "B_stop_0 lxc1" [ style = bold] ++"stonith 'reboot' lxc1" -> "M-clone_stop_0" [ style = bold] ++"stonith 'reboot' lxc1" -> "M_stop_0 lxc1" [ style = bold] ++"stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] ++"stonith_complete" -> "B_start_0 lxc1" [ style = bold] ++"stonith_complete" -> "M_start_0 lxc1" [ style = bold] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "container1_start_0 18node2" [ style = bold] ++"stonith_complete" -> "lxc1_start_0 18node2" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/whitebox-fail1.exp b/pengine/test10/whitebox-fail1.exp +index 9629a76..03e83c3 100644 +--- a/pengine/test10/whitebox-fail1.exp ++++ b/pengine/test10/whitebox-fail1.exp +@@ -10,6 +10,9 @@ + + + ++ ++ ++ + + + +@@ -58,6 +61,9 @@ + + + ++ ++ ++ + + + +@@ -68,10 +74,10 @@ + + + +- ++ + + +- ++ + + + +@@ -98,7 +104,7 @@ + + + +- ++ + + + +@@ -159,6 +165,9 @@ + + + ++ ++ ++ + + + +@@ -169,7 +178,7 @@ + + + +- ++ + + + +@@ -187,6 +196,9 @@ + + + ++ ++ ++ + + + +@@ -216,6 +228,33 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -233,6 +272,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/whitebox-fail1.summary b/pengine/test10/whitebox-fail1.summary +index 1586407..1872e9a 100644 +--- a/pengine/test10/whitebox-fail1.summary ++++ b/pengine/test10/whitebox-fail1.summary +@@ -22,20 +22,22 @@ Transition Summary: + Executing cluster transition: + * Resource action: lxc1 stop on 18node2 + * Resource action: container1 stop on 18node2 ++ * Pseudo action: stonith-lxc1-reboot on lxc1 ++ * Pseudo action: stonith_complete ++ * Resource action: container1 start on 18node2 + * Pseudo action: M-clone_stop_0 + * Pseudo action: B_stop_0 +- * Resource action: container1 start on 18node2 ++ * Resource action: lxc1 start on 18node2 ++ * Resource action: lxc1 monitor=30000 on 18node2 + * Pseudo action: M_stop_0 + * Pseudo action: M-clone_stopped_0 + * Pseudo action: M-clone_start_0 +- * Resource action: lxc1 start on 18node2 +- * Resource action: lxc1 monitor=30000 on 18node2 ++ * Resource action: B start on lxc1 + * Pseudo action: all_stopped + * Resource action: M start on lxc1 + * Pseudo action: M-clone_running_0 +- * Resource action: B start on lxc1 +- * Resource action: M monitor=10000 on lxc1 + * Resource action: B monitor=10000 on lxc1 ++ * Resource action: M monitor=10000 on lxc1 + + Revised cluster status: + Online: [ 18node1 18node2 18node3 ] +diff --git a/pengine/test10/whitebox-fail2.dot b/pengine/test10/whitebox-fail2.dot +index 0f0fe26..c6380ea 100644 +--- a/pengine/test10/whitebox-fail2.dot ++++ b/pengine/test10/whitebox-fail2.dot +@@ -25,11 +25,9 @@ digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "container1_start_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] + "container1_start_0 18node2" [ style=bold color="green" fontcolor="black"] +-"container1_stop_0 18node2" -> "B_stop_0 lxc1" [ style = bold] +-"container1_stop_0 18node2" -> "M-clone_stop_0" [ style = bold] +-"container1_stop_0 18node2" -> "M_stop_0 lxc1" [ style = bold] + "container1_stop_0 18node2" -> "all_stopped" [ style = bold] + "container1_stop_0 18node2" -> "container1_start_0 18node2" [ style = bold] ++"container1_stop_0 18node2" -> "stonith 'reboot' lxc1" [ style = bold] + "container1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] + "lxc1_monitor_30000 18node2" [ style=bold color="green" fontcolor="black"] + "lxc1_start_0 18node2" -> "B_monitor_10000 lxc1" [ style = bold] +@@ -42,4 +40,15 @@ digraph "g" { + "lxc1_stop_0 18node2" -> "container1_stop_0 18node2" [ style = bold] + "lxc1_stop_0 18node2" -> "lxc1_start_0 18node2" [ style = bold] + "lxc1_stop_0 18node2" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' lxc1" -> "B_stop_0 lxc1" [ style = bold] ++"stonith 'reboot' lxc1" -> "M-clone_stop_0" [ style = bold] ++"stonith 'reboot' lxc1" -> "M_stop_0 lxc1" [ style = bold] ++"stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] ++"stonith_complete" -> "B_start_0 lxc1" [ style = bold] ++"stonith_complete" -> "M_start_0 lxc1" [ style = bold] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "container1_start_0 18node2" [ style = bold] ++"stonith_complete" -> "lxc1_start_0 18node2" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/whitebox-fail2.exp b/pengine/test10/whitebox-fail2.exp +index 9629a76..03e83c3 100644 +--- a/pengine/test10/whitebox-fail2.exp ++++ b/pengine/test10/whitebox-fail2.exp +@@ -10,6 +10,9 @@ + + + ++ ++ ++ + + + +@@ -58,6 +61,9 @@ + + + ++ ++ ++ + + + +@@ -68,10 +74,10 @@ + + + +- ++ + + +- ++ + + + +@@ -98,7 +104,7 @@ + + + +- ++ + + + +@@ -159,6 +165,9 @@ + + + ++ ++ ++ + + + +@@ -169,7 +178,7 @@ + + + +- ++ + + + +@@ -187,6 +196,9 @@ + + + ++ ++ ++ + + + +@@ -216,6 +228,33 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -233,6 +272,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/whitebox-fail2.summary b/pengine/test10/whitebox-fail2.summary +index ab40d99..5db6588 100644 +--- a/pengine/test10/whitebox-fail2.summary ++++ b/pengine/test10/whitebox-fail2.summary +@@ -22,20 +22,22 @@ Transition Summary: + Executing cluster transition: + * Resource action: lxc1 stop on 18node2 + * Resource action: container1 stop on 18node2 ++ * Pseudo action: stonith-lxc1-reboot on lxc1 ++ * Pseudo action: stonith_complete ++ * Resource action: container1 start on 18node2 + * Pseudo action: M-clone_stop_0 + * Pseudo action: B_stop_0 +- * Resource action: container1 start on 18node2 ++ * Resource action: lxc1 start on 18node2 ++ * Resource action: lxc1 monitor=30000 on 18node2 + * Pseudo action: M_stop_0 + * Pseudo action: M-clone_stopped_0 + * Pseudo action: M-clone_start_0 +- * Resource action: lxc1 start on 18node2 +- * Resource action: lxc1 monitor=30000 on 18node2 ++ * Resource action: B start on lxc1 + * Pseudo action: all_stopped + * Resource action: M start on lxc1 + * Pseudo action: M-clone_running_0 +- * Resource action: B start on lxc1 +- * Resource action: M monitor=10000 on lxc1 + * Resource action: B monitor=10000 on lxc1 ++ * Resource action: M monitor=10000 on lxc1 + + Revised cluster status: + Online: [ 18node1 18node2 18node3 ] +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.dot b/pengine/test10/whitebox-imply-stop-on-fence.dot +index b3fd40b..0e17a16 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.dot ++++ b/pengine/test10/whitebox-imply-stop-on-fence.dot +@@ -6,6 +6,7 @@ + "R-lxc-01_kiff-01_stop_0 kiff-01" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] + "R-lxc-01_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "R-lxc-01_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] ++"R-lxc-01_kiff-01_stop_0 kiff-01" -> "stonith 'reboot' lxc-01_kiff-01" [ style = bold] + "R-lxc-01_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] + "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style=bold color="green" fontcolor="black"] + "R-lxc-02_kiff-01_start_0 kiff-02" -> "R-lxc-02_kiff-01_monitor_10000 kiff-02" [ style = bold] +@@ -14,7 +15,9 @@ + "R-lxc-02_kiff-01_stop_0 kiff-01" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] + "R-lxc-02_kiff-01_stop_0 kiff-01" -> "all_stopped" [ style = bold] + "R-lxc-02_kiff-01_stop_0 kiff-01" -> "shared0-clone_stop_0" [ style = bold] ++"R-lxc-02_kiff-01_stop_0 kiff-01" -> "stonith 'reboot' lxc-02_kiff-01" [ style = bold] + "R-lxc-02_kiff-01_stop_0 kiff-01" [ style=bold color="green" fontcolor="orange"] ++"all_stopped" -> "fence-kiff-02_start_0 kiff-02" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "clvmd-clone_stop_0" -> "clvmd-clone_stopped_0" [ style = bold] + "clvmd-clone_stop_0" -> "clvmd_stop_0 kiff-01" [ style = bold] +@@ -74,8 +77,12 @@ + "stonith 'reboot' kiff-01" -> "shared0-clone_stop_0" [ style = bold] + "stonith 'reboot' kiff-01" -> "shared0_stop_0 kiff-01" [ style = bold] + "stonith 'reboot' kiff-01" -> "stonith_complete" [ style = bold] +-"stonith 'reboot' kiff-01" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] + "stonith 'reboot' kiff-01" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' lxc-01_kiff-01" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' lxc-01_kiff-01" -> "vm-fs_stop_0 lxc-01_kiff-01" [ style = bold] ++"stonith 'reboot' lxc-01_kiff-01" [ style=bold color="green" fontcolor="orange"] ++"stonith 'reboot' lxc-02_kiff-01" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' lxc-02_kiff-01" [ style=bold color="green" fontcolor="orange"] + "stonith_complete" -> "R-lxc-01_kiff-01_start_0 kiff-02" [ style = bold] + "stonith_complete" -> "R-lxc-02_kiff-01_start_0 kiff-02" [ style = bold] + "stonith_complete" -> "all_stopped" [ style = bold] +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.exp b/pengine/test10/whitebox-imply-stop-on-fence.exp +index 0bd42b7..e38f4ea 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.exp ++++ b/pengine/test10/whitebox-imply-stop-on-fence.exp +@@ -1,33 +1,36 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ ++ ++ ++ + + + + + +- ++ + + + +@@ -35,16 +38,16 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -53,28 +56,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -83,16 +86,16 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -101,28 +104,28 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -131,13 +134,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -146,31 +149,31 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + +- ++ + + + +@@ -179,27 +182,27 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -208,7 +211,7 @@ + + + +- ++ + + + +@@ -223,27 +226,27 @@ + + + +- ++ + + + + + + +- ++ + + + + + +- ++ + + + + + + +- ++ + + + +@@ -252,7 +255,7 @@ + + + +- ++ + + + +@@ -267,48 +270,48 @@ + + + +- ++ + +- ++ + + + + +- ++ + + + + +- +- +- + + + + +- +- +- ++ ++ ++ ++ + + + +- ++ ++ ++ ++ ++ ++ ++ + + + + + +- +- +- +- ++ ++ ++ + + + +- +- +- +- ++ + + + +@@ -334,7 +337,7 @@ + + + +- ++ + + + +@@ -378,7 +381,7 @@ + + + +- ++ + + + +@@ -402,6 +405,36 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -413,7 +446,7 @@ + + + +- ++ + + + +@@ -423,35 +456,41 @@ + + + ++ ++ ++ ++ ++ ++ + + +- ++ + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-imply-stop-on-fence.summary b/pengine/test10/whitebox-imply-stop-on-fence.summary +index 3ee9570..50a3446 100644 +--- a/pengine/test10/whitebox-imply-stop-on-fence.summary ++++ b/pengine/test10/whitebox-imply-stop-on-fence.summary +@@ -2,7 +2,7 @@ + Current cluster status: + Node kiff-01 (1): UNCLEAN (offline) + Online: [ kiff-02 ] +-Containers: [ lxc-01_kiff-01:R-lxc-01_kiff-01 lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-01:R-lxc-02_kiff-01 lxc-02_kiff-02:R-lxc-02_kiff-02 ] ++Containers: [ lxc-01_kiff-02:R-lxc-01_kiff-02 lxc-02_kiff-02:R-lxc-02_kiff-02 ] + + fence-kiff-01 (stonith:fence_ipmilan): Started kiff-02 + fence-kiff-02 (stonith:fence_ipmilan): Started kiff-01 (UNCLEAN) +@@ -18,37 +18,37 @@ Containers: [ lxc-01_kiff-01:R-lxc-01_kiff-01 lxc-01_kiff-02:R-lxc-01_kiff-02 lx + shared0 (ocf::heartbeat:Filesystem): Started kiff-01 (UNCLEAN) + Started: [ kiff-02 ] + Stopped: [ lxc-01_kiff-01 lxc-01_kiff-02 lxc-02_kiff-01 lxc-02_kiff-02 ] +- R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-01 (UNCLEAN) ++ R-lxc-01_kiff-01 (ocf::heartbeat:VirtualDomain): FAILED kiff-01 (UNCLEAN) + R-lxc-02_kiff-01 (ocf::heartbeat:VirtualDomain): Started kiff-01 (UNCLEAN) + R-lxc-01_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 + R-lxc-02_kiff-02 (ocf::heartbeat:VirtualDomain): Started kiff-02 +- vm-fs (ocf::heartbeat:Filesystem): Started lxc-01_kiff-01 ++ vm-fs (ocf::heartbeat:Filesystem): FAILED lxc-01_kiff-01 + + Transition Summary: + * Move fence-kiff-02 (Started kiff-01 -> kiff-02) + * Stop dlm:0 (kiff-01) + * Stop clvmd:0 (kiff-01) + * Stop shared0:0 (kiff-01) +- * Move R-lxc-01_kiff-01 (Started kiff-01 -> kiff-02) ++ * Recover R-lxc-01_kiff-01 (Started kiff-01 -> kiff-02) + * Move R-lxc-02_kiff-01 (Started kiff-01 -> kiff-02) +- * Restart vm-fs (Started lxc-01_kiff-01) ++ * Recover vm-fs (Started lxc-01_kiff-01) + * Move lxc-01_kiff-01 (Started kiff-01 -> kiff-02) + * Move lxc-02_kiff-01 (Started kiff-01 -> kiff-02) + + Executing cluster transition: + * Pseudo action: fence-kiff-02_stop_0 + * Fencing kiff-01 (reboot) +- * Pseudo action: stonith_complete +- * Resource action: fence-kiff-02 start on kiff-02 +- * Pseudo action: vm-fs_stop_0 + * Pseudo action: lxc-01_kiff-01_stop_0 + * Pseudo action: lxc-02_kiff-01_stop_0 +- * Resource action: fence-kiff-02 monitor=60000 on kiff-02 + * Pseudo action: R-lxc-01_kiff-01_stop_0 + * Pseudo action: R-lxc-02_kiff-01_stop_0 ++ * Pseudo action: stonith-lxc-02_kiff-01-reboot on lxc-02_kiff-01 ++ * Pseudo action: stonith-lxc-01_kiff-01-reboot on lxc-01_kiff-01 ++ * Pseudo action: stonith_complete + * Pseudo action: shared0-clone_stop_0 + * Resource action: R-lxc-01_kiff-01 start on kiff-02 + * Resource action: R-lxc-02_kiff-01 start on kiff-02 ++ * Pseudo action: vm-fs_stop_0 + * Resource action: lxc-01_kiff-01 start on kiff-02 + * Resource action: lxc-02_kiff-01 start on kiff-02 + * Pseudo action: shared0_stop_0 +@@ -56,16 +56,18 @@ Executing cluster transition: + * Resource action: R-lxc-01_kiff-01 monitor=10000 on kiff-02 + * Resource action: R-lxc-02_kiff-01 monitor=10000 on kiff-02 + * Resource action: vm-fs start on lxc-01_kiff-01 +- * Resource action: vm-fs monitor=20000 on lxc-01_kiff-01 + * Resource action: lxc-01_kiff-01 monitor=30000 on kiff-02 + * Resource action: lxc-02_kiff-01 monitor=30000 on kiff-02 + * Pseudo action: clvmd-clone_stop_0 ++ * Resource action: vm-fs monitor=20000 on lxc-01_kiff-01 + * Pseudo action: clvmd_stop_0 + * Pseudo action: clvmd-clone_stopped_0 + * Pseudo action: dlm-clone_stop_0 + * Pseudo action: dlm_stop_0 + * Pseudo action: dlm-clone_stopped_0 + * Pseudo action: all_stopped ++ * Resource action: fence-kiff-02 start on kiff-02 ++ * Resource action: fence-kiff-02 monitor=60000 on kiff-02 + + Revised cluster status: + Online: [ kiff-02 ] +diff --git a/pengine/test10/whitebox-ms-ordering.dot b/pengine/test10/whitebox-ms-ordering.dot +index 7f03a65..bd77363 100644 +--- a/pengine/test10/whitebox-ms-ordering.dot ++++ b/pengine/test10/whitebox-ms-ordering.dot +@@ -79,4 +79,22 @@ + "lxc2_start_0 18node1" -> "lxc-ms_start_0 lxc2" [ style = bold] + "lxc2_start_0 18node1" -> "lxc2_monitor_30000 18node1" [ style = bold] + "lxc2_start_0 18node1" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' lxc1" -> "lxc-ms-master_stop_0" [ style = bold] ++"stonith 'reboot' lxc1" -> "lxc-ms_demote_0 lxc1" [ style = bold] ++"stonith 'reboot' lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] ++"stonith 'reboot' lxc1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' lxc1" [ style=bold color="green" fontcolor="orange"] ++"stonith 'reboot' lxc2" -> "lxc-ms-master_stop_0" [ style = bold] ++"stonith 'reboot' lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold] ++"stonith 'reboot' lxc2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' lxc2" [ style=bold color="green" fontcolor="orange"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "container1_start_0 18node1" [ style = bold] ++"stonith_complete" -> "container2_start_0 18node1" [ style = bold] ++"stonith_complete" -> "lxc-ms_promote_0 lxc1" [ style = bold] ++"stonith_complete" -> "lxc-ms_start_0 lxc1" [ style = bold] ++"stonith_complete" -> "lxc-ms_start_0 lxc2" [ style = bold] ++"stonith_complete" -> "lxc1_start_0 18node1" [ style = bold] ++"stonith_complete" -> "lxc2_start_0 18node1" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/whitebox-ms-ordering.exp b/pengine/test10/whitebox-ms-ordering.exp +index 0566f41..7aecfba 100644 +--- a/pengine/test10/whitebox-ms-ordering.exp ++++ b/pengine/test10/whitebox-ms-ordering.exp +@@ -16,6 +16,9 @@ + + + ++ ++ ++ + + + +@@ -62,6 +65,9 @@ + + + ++ ++ ++ + + + +@@ -111,6 +117,9 @@ + + + ++ ++ ++ + + + +@@ -130,6 +139,9 @@ + + + ++ ++ ++ + + + +@@ -154,6 +166,9 @@ + + + ++ ++ ++ + + + +@@ -166,6 +181,9 @@ + + + ++ ++ ++ + + + +@@ -228,6 +246,9 @@ + + + ++ ++ ++ + + + +@@ -240,6 +261,9 @@ + + + ++ ++ ++ + + + +@@ -323,6 +347,12 @@ + + + ++ ++ ++ ++ ++ ++ + + + +@@ -391,6 +421,9 @@ + + + ++ ++ ++ + + + +@@ -417,10 +450,50 @@ + + + ++ ++ ++ + + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -432,6 +505,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/whitebox-ms-ordering.summary b/pengine/test10/whitebox-ms-ordering.summary +index d6bbaaf..5d7c042 100644 +--- a/pengine/test10/whitebox-ms-ordering.summary ++++ b/pengine/test10/whitebox-ms-ordering.summary +@@ -27,6 +27,9 @@ Executing cluster transition: + * Resource action: lxc-ms monitor on 18node2 + * Resource action: lxc-ms monitor on 18node1 + * Pseudo action: lxc-ms-master_demote_0 ++ * Pseudo action: stonith-lxc2-reboot on lxc2 ++ * Pseudo action: stonith-lxc1-reboot on lxc1 ++ * Pseudo action: stonith_complete + * Resource action: container1 start on 18node1 + * Resource action: container2 start on 18node1 + * Pseudo action: lxc-ms_demote_0 +diff --git a/pengine/test10/whitebox-unexpectedly-running.dot b/pengine/test10/whitebox-unexpectedly-running.dot +index d87344a..2915d77 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.dot ++++ b/pengine/test10/whitebox-unexpectedly-running.dot +@@ -5,9 +5,14 @@ + "FAKE_start_0 18builder" [ style=bold color="green" fontcolor="black"] + "FAKE_stop_0 18builder" -> "FAKE_start_0 18builder" [ style = bold] + "FAKE_stop_0 18builder" -> "all_stopped" [ style = bold] ++"FAKE_stop_0 18builder" -> "stonith 'reboot' remote1" [ style = bold] + "FAKE_stop_0 18builder" [ style=bold color="green" fontcolor="black"] + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "remote1_monitor_30000 18builder" [ style=bold color="green" fontcolor="black"] + "remote1_start_0 18builder" -> "remote1_monitor_30000 18builder" [ style = bold] + "remote1_start_0 18builder" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' remote1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' remote1" [ style=bold color="green" fontcolor="orange"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/whitebox-unexpectedly-running.exp b/pengine/test10/whitebox-unexpectedly-running.exp +index 29cd66b..c5272f2 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.exp ++++ b/pengine/test10/whitebox-unexpectedly-running.exp +@@ -62,6 +62,33 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -70,6 +97,9 @@ + + + ++ ++ ++ + + + +diff --git a/pengine/test10/whitebox-unexpectedly-running.summary b/pengine/test10/whitebox-unexpectedly-running.summary +index f834e41..7d5b908 100644 +--- a/pengine/test10/whitebox-unexpectedly-running.summary ++++ b/pengine/test10/whitebox-unexpectedly-running.summary +@@ -10,6 +10,8 @@ Transition Summary: + + Executing cluster transition: + * Resource action: FAKE stop on 18builder ++ * Pseudo action: stonith-remote1-reboot on remote1 ++ * Pseudo action: stonith_complete + * Pseudo action: all_stopped + * Resource action: FAKE start on 18builder + * Resource action: remote1 start on 18builder +-- +1.8.3.1 + diff --git a/SOURCES/018-guest-fencing-cl5247-test.patch b/SOURCES/018-guest-fencing-cl5247-test.patch new file mode 100644 index 0000000..a50cd68 --- /dev/null +++ b/SOURCES/018-guest-fencing-cl5247-test.patch @@ -0,0 +1,348 @@ +From a9977a2dc1135d040088b90e1ea307b3ff71c0b5 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Tue, 22 Mar 2016 16:43:53 -0500 +Subject: [PATCH] Test: pengine: update regression tests for guest node + pseudo-fencing + +--- +diff --git a/pengine/test10/bug-cl-5247.exp b/pengine/test10/bug-cl-5247.exp +--- a/pengine/test10/bug-cl-5247.exp 2017-01-09 18:00:54.205719464 -0600 ++++ b/pengine/test10/bug-cl-5247.exp 2017-01-09 18:00:54.207719440 -0600 +@@ -20,7 +20,7 @@ + + + +- ++ + + + +@@ -64,9 +64,10 @@ + + + +- ++ ++ + +- ++ + + + +@@ -83,7 +84,10 @@ + + + +- ++ ++ ++ ++ + + + +@@ -175,6 +179,9 @@ + + + ++ ++ ++ + + + +@@ -221,10 +228,10 @@ + + + +- ++ + + +- ++ + + + +@@ -285,6 +292,9 @@ + + + ++ ++ ++ + + + +@@ -295,14 +305,14 @@ + + + +- +- +- + + + + + ++ ++ ++ + + + +@@ -335,6 +345,9 @@ + + + ++ ++ ++ + + + +@@ -345,37 +358,37 @@ + + + +- ++ + + +- ++ + + + + + +- +- ++ ++ + + + + +- ++ + + +- ++ + + + + + +- +- ++ ++ + + + + +- ++ + + + +@@ -387,14 +400,14 @@ + + + +- +- +- + + + + + ++ ++ ++ + + + +@@ -405,16 +418,16 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + +@@ -427,7 +440,7 @@ + + + +- ++ + + + +@@ -440,7 +453,7 @@ + + + +- ++ + + + +@@ -453,9 +466,9 @@ + + + +- ++ + +- ++ + + + +@@ -463,7 +476,7 @@ + + + +- ++ + + + +@@ -482,7 +495,7 @@ + + + +- ++ + + + +@@ -497,7 +510,7 @@ + + + +- ++ + + + +@@ -527,7 +540,7 @@ + + + +- ++ + + + +@@ -577,7 +590,7 @@ + + + +- ++ + + + +@@ -607,7 +620,7 @@ + + + +- ++ + + + +@@ -646,14 +659,14 @@ + + + +- +- +- + + + + + ++ ++ ++ + + + +@@ -670,6 +683,33 @@ + + + ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -679,7 +719,7 @@ + + + +- ++ + + + +@@ -700,7 +740,10 @@ + + + +- ++ ++ ++ ++ + + + + diff --git a/SOURCES/019-guest-fencing-status.patch b/SOURCES/019-guest-fencing-status.patch new file mode 100644 index 0000000..086e2af --- /dev/null +++ b/SOURCES/019-guest-fencing-status.patch @@ -0,0 +1,160 @@ +From 92dd7d7616dc16d345ef73d0685b12e06d09b36b Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Fri, 15 Apr 2016 15:04:03 -0500 +Subject: [PATCH] Fix: crmd: update cache status for guest node whose host is + fenced + +Normally, the remote RA's stop action handles setting the peer cache state +to down (along with other side effects of a stop). However, if a guest node's +host is fenced, the RA will not be called. Check for the fencing pseudo-action +created by the pengine in this case. +--- + crmd/crmd_lrm.h | 1 + + crmd/remote_lrmd_ra.c | 72 +++++++++++++++++++++++++++++++++++++++++++++++++-- + crmd/te_actions.c | 4 +++ + 3 files changed, 75 insertions(+), 2 deletions(-) + +diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h +index 412ce5b..08ba947 100644 +--- a/crmd/crmd_lrm.h ++++ b/crmd/crmd_lrm.h +@@ -160,5 +160,6 @@ int remote_ra_exec(lrm_state_t * lrm_state, const char *rsc_id, const char *acti + lrmd_key_value_t * params); + void remote_ra_cleanup(lrm_state_t * lrm_state); + void remote_ra_fail(const char *node_name); ++void remote_ra_process_pseudo(xmlNode *xml); + + gboolean process_lrm_event(lrm_state_t * lrm_state, lrmd_event_data_t * op, struct recurring_op_s *pending); +diff --git a/crmd/remote_lrmd_ra.c b/crmd/remote_lrmd_ra.c +index b9c5068..eb995ea 100644 +--- a/crmd/remote_lrmd_ra.c ++++ b/crmd/remote_lrmd_ra.c +@@ -226,14 +226,20 @@ remote_node_up(const char *node_name) + free_xml(update); + } + ++enum down_opts { ++ DOWN_KEEP_LRM, ++ DOWN_ERASE_LRM ++}; ++ + /*! + * \internal + * \brief Handle cluster communication related to pacemaker_remote node leaving + * + * \param[in] node_name Name of lost node ++ * \param[in] opts Whether to keep or erase LRM history + */ + static void +-remote_node_down(const char *node_name) ++remote_node_down(const char *node_name, const enum down_opts opts) + { + xmlNode *update; + int call_id = 0; +@@ -246,6 +252,14 @@ remote_node_down(const char *node_name) + /* Purge node's transient attributes */ + erase_status_tag(node_name, XML_TAG_TRANSIENT_NODEATTRS, call_opt); + ++ /* Normally, the LRM operation history should be kept until the node comes ++ * back up. However, after a successful fence, we want to clear it, so we ++ * don't think resources are still running on the node. ++ */ ++ if (opts == DOWN_ERASE_LRM) { ++ erase_status_tag(node_name, XML_CIB_TAG_LRM, call_opt); ++ } ++ + /* Ensure node is in the remote peer cache with lost state */ + node = crm_remote_peer_get(node_name); + CRM_CHECK(node != NULL, return); +@@ -301,7 +315,7 @@ check_remote_node_state(remote_ra_cmd_t *cmd) + if (ra_data) { + if (ra_data->migrate_status != takeover_complete) { + /* Stop means down if we didn't successfully migrate elsewhere */ +- remote_node_down(cmd->rsc_id); ++ remote_node_down(cmd->rsc_id, DOWN_KEEP_LRM); + } else if (AM_I_DC == FALSE) { + /* Only the connection host and DC track node state, + * so if the connection migrated elsewhere and we aren't DC, +@@ -1072,3 +1086,57 @@ remote_ra_fail(const char *node_name) + } + } + ++/* A guest node fencing implied by host fencing looks like: ++ * ++ * ++ * ++ * ++ * ++ * ++ * ++ */ ++#define XPATH_PSEUDO_FENCE "//" XML_GRAPH_TAG_PSEUDO_EVENT \ ++ "[@" XML_LRM_ATTR_TASK "='stonith']/" XML_GRAPH_TAG_DOWNED \ ++ "/" XML_CIB_TAG_NODE ++ ++/*! ++ * \internal ++ * \brief Check a pseudo-action for Pacemaker Remote node side effects ++ * ++ * \param[in] xml XML of pseudo-action to check ++ */ ++void ++remote_ra_process_pseudo(xmlNode *xml) ++{ ++ xmlXPathObjectPtr search = xpath_search(xml, XPATH_PSEUDO_FENCE); ++ ++ if (numXpathResults(search) == 1) { ++ xmlNode *result = getXpathResult(search, 0); ++ ++ /* Normally, we handle the necessary side effects of a guest node stop ++ * action when reporting the remote agent's result. However, if the stop ++ * is implied due to fencing, it will be a fencing pseudo-event, and ++ * there won't be a result to report. Handle that case here. ++ * ++ * This will result in a duplicate call to remote_node_down() if the ++ * guest stop was real instead of implied, but that shouldn't hurt. ++ * ++ * There is still one corner case that isn't handled: if a guest node ++ * isn't running any resources when its host is fenced, it will appear ++ * to be cleanly stopped, so there will be no pseudo-fence, and our ++ * peer cache state will be incorrect unless and until the guest is ++ * recovered. ++ */ ++ if (result) { ++ const char *remote = ID(result); ++ ++ if (remote) { ++ remote_node_down(remote, DOWN_ERASE_LRM); ++ } ++ } ++ } ++ freeXpathObject(search); ++} +diff --git a/crmd/te_actions.c b/crmd/te_actions.c +index c971273..01538af 100644 +--- a/crmd/te_actions.c ++++ b/crmd/te_actions.c +@@ -27,6 +27,7 @@ + #include + + #include ++#include + #include + #include + #include +@@ -52,6 +53,9 @@ te_start_action_timer(crm_graph_t * graph, crm_action_t * action) + static gboolean + te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo) + { ++ /* Check action for Pacemaker Remote node side effects */ ++ remote_ra_process_pseudo(pseudo->xml); ++ + crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id, + crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY)); + te_action_confirmed(pseudo); +-- +1.8.3.1 + diff --git a/SOURCES/020-crm_node-remote.patch b/SOURCES/020-crm_node-remote.patch new file mode 100644 index 0000000..27284cc --- /dev/null +++ b/SOURCES/020-crm_node-remote.patch @@ -0,0 +1,69 @@ +From e0eb9e766dc3dd296e7a5e623bf7b10ebcdb33f7 Mon Sep 17 00:00:00 2001 +From: Ken Gaillot +Date: Mon, 6 Feb 2017 13:54:50 -0600 +Subject: [PATCH] Fix: libcrmcommon,pengine,tools: pass local node name to + resource agents + +Preivously, crm_node -n would show the local uname on remote nodes, even when +the node name known to the cluster was different. Now, the policy engine adds +environment variables with the local node name and UUID before calling the +resource agent. If crm_node detects these variables, it will use them instead +of uname. + +This only affects crm_node behavior when called via the cluster, not +directly from the command line. + +Based on patch originally provided by Andrew Beekhof +--- + lib/common/utils.c | 2 ++ + pengine/graph.c | 3 +++ + tools/crm_node.c | 6 +++++- + 3 files changed, 10 insertions(+), 1 deletion(-) + +diff --git a/lib/common/utils.c b/lib/common/utils.c +index 83072c5..3e3abd3 100644 +--- a/lib/common/utils.c ++++ b/lib/common/utils.c +@@ -894,6 +894,8 @@ filter_action_parameters(xmlNode * param_set, const char *version) + XML_ATTR_ID, + XML_ATTR_CRM_VERSION, + XML_LRM_ATTR_OP_DIGEST, ++ XML_LRM_ATTR_TARGET, ++ XML_LRM_ATTR_TARGET_UUID, + }; + + gboolean do_delete = FALSE; +diff --git a/pengine/graph.c b/pengine/graph.c +index 569cf6e..81d8355 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -948,6 +948,9 @@ action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set) + if (router_node) { + crm_xml_add(action_xml, XML_LRM_ATTR_ROUTER_NODE, router_node->details->uname); + } ++ ++ g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET), strdup(action->node->details->uname)); ++ g_hash_table_insert(action->meta, strdup(XML_LRM_ATTR_TARGET_UUID), strdup(action->node->details->id)); + } + + /* No details if this action is only being listed in the inputs section */ +diff --git a/tools/crm_node.c b/tools/crm_node.c +index d927f31..7092db4 100644 +--- a/tools/crm_node.c ++++ b/tools/crm_node.c +@@ -951,7 +951,11 @@ main(int argc, char **argv) + } + + if (command == 'n') { +- fprintf(stdout, "%s\n", get_local_node_name()); ++ const char *name = getenv("OCF_RESKEY_" CRM_META "_" XML_LRM_ATTR_TARGET); ++ if(name == NULL) { ++ name = get_local_node_name(); ++ } ++ fprintf(stdout, "%s\n", name); + crm_exit(pcmk_ok); + + } else if (command == 'N') { +-- +1.8.3.1 + diff --git a/SOURCES/crm_node-regression-tests.patch b/SOURCES/crm_node-regression-tests.patch new file mode 100644 index 0000000..5032fe0 --- /dev/null +++ b/SOURCES/crm_node-regression-tests.patch @@ -0,0 +1,61478 @@ +From 76dfa6454899055a5abb38ef74d1474308230d7d Mon Sep 17 00:00:00 2001 +From: rpm-build +Date: Wed, 8 Feb 2017 13:46:49 -0600 +Subject: [PATCH] Test: pengine: update regression tests for new + meta-attributes + +--- + pengine/test10/1-a-then-bm-move-b.exp | 8 +- + pengine/test10/10-a-then-bm-b-move-a-clone.exp | 8 +- + .../11-a-then-bm-b-move-a-clone-starting.exp | 10 +- + pengine/test10/1360.exp | 4 +- + pengine/test10/1484.exp | 2 +- + pengine/test10/1494.exp | 2 +- + pengine/test10/2-am-then-b-move-a.exp | 8 +- + pengine/test10/3-am-then-bm-both-migrate.exp | 16 +- + pengine/test10/4-am-then-bm-b-not-migratable.exp | 14 +- + pengine/test10/5-am-then-bm-a-not-migratable.exp | 12 +- + pengine/test10/594.exp | 28 +- + pengine/test10/6-migrate-group.exp | 24 +- + pengine/test10/662.exp | 56 +-- + pengine/test10/696.exp | 54 +-- + .../test10/7-migrate-group-one-unmigratable.exp | 20 +- + pengine/test10/726.exp | 94 ++--- + pengine/test10/735.exp | 26 +- + pengine/test10/764.exp | 30 +- + pengine/test10/797.exp | 50 +-- + .../test10/8-am-then-bm-a-migrating-b-stopping.exp | 10 +- + pengine/test10/829.exp | 38 +- + .../test10/9-am-then-bm-b-migrating-a-stopping.exp | 4 +- + pengine/test10/994-2.exp | 10 +- + pengine/test10/994.exp | 6 +- + pengine/test10/a-demote-then-b-migrate.exp | 36 +- + pengine/test10/a-promote-then-b-migrate.exp | 22 +- + pengine/test10/anti-colocation-master.exp | 8 +- + pengine/test10/anti-colocation-order.exp | 12 +- + pengine/test10/anti-colocation-slave.exp | 8 +- + pengine/test10/asymmetric.exp | 8 +- + pengine/test10/asymmetrical-order-move.exp | 2 +- + pengine/test10/attrs1.exp | 6 +- + pengine/test10/attrs2.exp | 6 +- + pengine/test10/attrs3.exp | 6 +- + pengine/test10/attrs4.exp | 6 +- + pengine/test10/attrs5.exp | 4 +- + pengine/test10/attrs6.exp | 6 +- + pengine/test10/attrs7.exp | 6 +- + pengine/test10/attrs8.exp | 6 +- + pengine/test10/balanced.exp | 12 +- + pengine/test10/base-score.exp | 10 +- + pengine/test10/bnc-515172.exp | 4 +- + pengine/test10/bug-1572-1.exp | 46 +- + pengine/test10/bug-1572-2.exp | 26 +- + pengine/test10/bug-1573.exp | 4 +- + pengine/test10/bug-1685.exp | 16 +- + pengine/test10/bug-1718.exp | 14 +- + pengine/test10/bug-1765.exp | 8 +- + pengine/test10/bug-1820-1.exp | 24 +- + pengine/test10/bug-1820.exp | 18 +- + pengine/test10/bug-1822.exp | 10 +- + pengine/test10/bug-5007-masterslave_colocation.exp | 8 +- + pengine/test10/bug-5014-A-start-B-start.exp | 12 +- + pengine/test10/bug-5014-A-stop-B-started.exp | 2 +- + pengine/test10/bug-5014-A-stopped-B-stopped.exp | 4 +- + pengine/test10/bug-5014-CLONE-A-start-B-start.exp | 12 +- + pengine/test10/bug-5014-CLONE-A-stop-B-started.exp | 2 +- + pengine/test10/bug-5014-CthenAthenB-C-stopped.exp | 6 +- + pengine/test10/bug-5014-GROUP-A-start-B-start.exp | 8 +- + .../test10/bug-5014-GROUP-A-stopped-B-started.exp | 2 +- + .../bug-5014-ordered-set-symmetrical-false.exp | 2 +- + .../bug-5014-ordered-set-symmetrical-true.exp | 4 +- + pengine/test10/bug-5025-1.exp | 6 +- + pengine/test10/bug-5025-3.exp | 8 +- + pengine/test10/bug-5025-4.exp | 8 +- + pengine/test10/bug-5028-bottom.exp | 2 +- + pengine/test10/bug-5028-detach.exp | 2 +- + pengine/test10/bug-5059.exp | 44 +- + pengine/test10/bug-5069-op-disabled.exp | 4 +- + pengine/test10/bug-5140-require-all-false.exp | 6 +- + pengine/test10/bug-5143-ms-shuffle.exp | 14 +- + pengine/test10/bug-5186-partial-migrate.exp | 16 +- + pengine/test10/bug-cl-5168.exp | 14 +- + pengine/test10/bug-cl-5170.exp | 4 +- + pengine/test10/bug-cl-5212.exp | 4 +- + pengine/test10/bug-cl-5213.exp | 2 +- + pengine/test10/bug-cl-5219.exp | 2 +- + pengine/test10/bug-cl-5247.exp | 34 +- + pengine/test10/bug-lf-1852.exp | 8 +- + pengine/test10/bug-lf-1920.exp | 2 +- + pengine/test10/bug-lf-2106.exp | 16 +- + pengine/test10/bug-lf-2153.exp | 10 +- + pengine/test10/bug-lf-2160.exp | 4 +- + pengine/test10/bug-lf-2171.exp | 8 +- + pengine/test10/bug-lf-2213.exp | 4 +- + pengine/test10/bug-lf-2317.exp | 14 +- + pengine/test10/bug-lf-2358.exp | 8 +- + pengine/test10/bug-lf-2361.exp | 18 +- + pengine/test10/bug-lf-2422.exp | 26 +- + pengine/test10/bug-lf-2435.exp | 12 +- + pengine/test10/bug-lf-2445.exp | 4 +- + pengine/test10/bug-lf-2453.exp | 10 +- + pengine/test10/bug-lf-2474.exp | 4 +- + pengine/test10/bug-lf-2493.exp | 68 +-- + pengine/test10/bug-lf-2508.exp | 14 +- + pengine/test10/bug-lf-2544.exp | 2 +- + pengine/test10/bug-lf-2551.exp | 10 +- + pengine/test10/bug-lf-2574.exp | 8 +- + pengine/test10/bug-lf-2581.exp | 26 +- + pengine/test10/bug-lf-2606.exp | 4 +- + pengine/test10/bug-lf-2613.exp | 36 +- + pengine/test10/bug-lf-2619.exp | 38 +- + pengine/test10/bug-n-385265-2.exp | 12 +- + pengine/test10/bug-n-385265.exp | 2 +- + pengine/test10/bug-n-387749.exp | 30 +- + pengine/test10/bug-pm-11.exp | 14 +- + pengine/test10/bug-pm-12.exp | 12 +- + pengine/test10/bug-rh-1097457.exp | 30 +- + pengine/test10/bug-rh-880249.exp | 14 +- + pengine/test10/bug-suse-707150.exp | 16 +- + pengine/test10/clone-anon-dup.exp | 10 +- + pengine/test10/clone-anon-failcount.exp | 38 +- + pengine/test10/clone-anon-probe-1.exp | 8 +- + pengine/test10/clone-anon-probe-2.exp | 2 +- + pengine/test10/clone-colocate-instance-1.exp | 20 +- + pengine/test10/clone-colocate-instance-2.exp | 20 +- + pengine/test10/clone-fail-block-colocation.exp | 24 +- + pengine/test10/clone-interleave-1.exp | 16 +- + pengine/test10/clone-interleave-2.exp | 8 +- + pengine/test10/clone-interleave-3.exp | 12 +- + pengine/test10/clone-max-zero.exp | 12 +- + pengine/test10/clone-no-shuffle.exp | 28 +- + pengine/test10/clone-order-16instances.exp | 56 +-- + pengine/test10/clone-order-instance.exp | 14 +- + pengine/test10/clone-order-primitive.exp | 6 +- + pengine/test10/clone-require-all-1.exp | 8 +- + pengine/test10/clone-require-all-2.exp | 10 +- + pengine/test10/clone-require-all-3.exp | 14 +- + pengine/test10/clone-require-all-4.exp | 8 +- + pengine/test10/clone-require-all-5.exp | 16 +- + pengine/test10/clone-require-all-6.exp | 4 +- + pengine/test10/clone-require-all-7.exp | 24 +- + .../test10/clone-require-all-no-interleave-1.exp | 20 +- + .../test10/clone-require-all-no-interleave-2.exp | 20 +- + .../test10/clone-require-all-no-interleave-3.exp | 18 +- + pengine/test10/clone_min_interleave_start_one.exp | 4 +- + pengine/test10/clone_min_interleave_start_two.exp | 32 +- + pengine/test10/clone_min_interleave_stop_one.exp | 2 +- + pengine/test10/clone_min_interleave_stop_two.exp | 16 +- + pengine/test10/clone_min_start_one.exp | 10 +- + pengine/test10/clone_min_start_two.exp | 12 +- + pengine/test10/clone_min_stop_all.exp | 14 +- + pengine/test10/clone_min_stop_one.exp | 2 +- + pengine/test10/clone_min_stop_two.exp | 12 +- + pengine/test10/cloned-group-stop.exp | 32 +- + pengine/test10/cloned-group.exp | 16 +- + pengine/test10/cloned_start_one.exp | 8 +- + pengine/test10/cloned_start_two.exp | 10 +- + pengine/test10/cloned_stop_one.exp | 4 +- + pengine/test10/cloned_stop_two.exp | 8 +- + pengine/test10/cluster-specific-params.exp | 8 +- + pengine/test10/colo_master_w_native.exp | 24 +- + pengine/test10/colo_slave_w_native.exp | 30 +- + pengine/test10/coloc-attr.exp | 4 +- + pengine/test10/coloc-clone-stays-active.exp | 8 +- + pengine/test10/coloc-group.exp | 22 +- + pengine/test10/coloc-intra-set.exp | 22 +- + pengine/test10/coloc-list.exp | 28 +- + pengine/test10/coloc-loop.exp | 24 +- + pengine/test10/coloc-many-one.exp | 24 +- + pengine/test10/coloc-negative-group.exp | 4 +- + pengine/test10/coloc-slave-anti.exp | 12 +- + pengine/test10/coloc_fp_logic.exp | 6 +- + pengine/test10/colocate-primitive-with-clone.exp | 16 +- + pengine/test10/colocated-utilization-clone.exp | 46 +- + pengine/test10/colocated-utilization-group.exp | 30 +- + .../test10/colocated-utilization-primitive-1.exp | 18 +- + .../test10/colocated-utilization-primitive-2.exp | 16 +- + .../test10/colocation_constraint_stops_master.exp | 12 +- + .../test10/colocation_constraint_stops_slave.exp | 6 +- + pengine/test10/comments.exp | 12 +- + pengine/test10/complex_enforce_colo.exp | 186 ++++----- + pengine/test10/container-1.exp | 16 +- + pengine/test10/container-2.exp | 18 +- + pengine/test10/container-3.exp | 16 +- + pengine/test10/container-4.exp | 18 +- + pengine/test10/container-group-1.exp | 16 +- + pengine/test10/container-group-2.exp | 18 +- + pengine/test10/container-group-3.exp | 14 +- + pengine/test10/container-group-4.exp | 18 +- + pengine/test10/date-1.exp | 6 +- + pengine/test10/enforce-colo1.exp | 10 +- + pengine/test10/expire-non-blocked-failure.exp | 2 +- + pengine/test10/failcount-block.exp | 16 +- + pengine/test10/failcount.exp | 8 +- + pengine/test10/group-dependents.exp | 146 +++---- + pengine/test10/group-fail.exp | 12 +- + pengine/test10/group-unmanaged-stopped.exp | 2 +- + pengine/test10/group1.exp | 18 +- + pengine/test10/group10.exp | 36 +- + pengine/test10/group11.exp | 4 +- + pengine/test10/group13.exp | 2 +- + pengine/test10/group14.exp | 10 +- + pengine/test10/group15.exp | 30 +- + pengine/test10/group2.exp | 30 +- + pengine/test10/group3.exp | 36 +- + pengine/test10/group4.exp | 10 +- + pengine/test10/group5.exp | 30 +- + pengine/test10/group6.exp | 36 +- + pengine/test10/group7.exp | 56 +-- + pengine/test10/group8.exp | 22 +- + pengine/test10/group9.exp | 36 +- + pengine/test10/guest-node-host-dies.exp | 30 +- + pengine/test10/honor_stonith_rsc_order1.exp | 12 +- + pengine/test10/honor_stonith_rsc_order2.exp | 20 +- + pengine/test10/honor_stonith_rsc_order3.exp | 20 +- + pengine/test10/honor_stonith_rsc_order4.exp | 12 +- + pengine/test10/ignore_stonith_rsc_order1.exp | 8 +- + pengine/test10/ignore_stonith_rsc_order2.exp | 12 +- + pengine/test10/ignore_stonith_rsc_order3.exp | 12 +- + pengine/test10/ignore_stonith_rsc_order4.exp | 12 +- + pengine/test10/inc0.exp | 28 +- + pengine/test10/inc1.exp | 40 +- + pengine/test10/inc10.exp | 18 +- + pengine/test10/inc11.exp | 26 +- + pengine/test10/inc12.exp | 62 +-- + pengine/test10/inc2.exp | 20 +- + pengine/test10/inc3.exp | 38 +- + pengine/test10/inc4.exp | 38 +- + pengine/test10/inc5.exp | 80 ++-- + pengine/test10/inc6.exp | 24 +- + pengine/test10/inc7.exp | 88 ++-- + pengine/test10/inc8.exp | 50 +-- + pengine/test10/inc9.exp | 6 +- + pengine/test10/interleave-0.exp | 324 +++++++------- + pengine/test10/interleave-1.exp | 324 +++++++------- + pengine/test10/interleave-2.exp | 324 +++++++------- + pengine/test10/interleave-3.exp | 324 +++++++------- + pengine/test10/interleave-pseudo-stop.exp | 12 +- + pengine/test10/interleave-restart.exp | 52 +-- + pengine/test10/interleave-stop.exp | 26 +- + pengine/test10/isolation-clone.exp | 16 +- + pengine/test10/isolation-restart-all.exp | 66 +-- + pengine/test10/isolation-start-all.exp | 44 +- + pengine/test10/load-stopped-loop-2.exp | 68 +-- + pengine/test10/load-stopped-loop.exp | 56 +-- + pengine/test10/location-sets-templates.exp | 36 +- + pengine/test10/managed-0.exp | 162 +++---- + pengine/test10/managed-1.exp | 162 +++---- + pengine/test10/managed-2.exp | 230 +++++----- + pengine/test10/master-0.exp | 28 +- + pengine/test10/master-1.exp | 30 +- + pengine/test10/master-10.exp | 62 +-- + pengine/test10/master-11.exp | 20 +- + pengine/test10/master-12.exp | 2 +- + pengine/test10/master-13.exp | 34 +- + pengine/test10/master-2.exp | 54 +-- + pengine/test10/master-3.exp | 30 +- + pengine/test10/master-4.exp | 60 +-- + pengine/test10/master-5.exp | 54 +-- + pengine/test10/master-6.exp | 48 +-- + pengine/test10/master-7.exp | 58 +-- + pengine/test10/master-8.exp | 64 +-- + pengine/test10/master-9.exp | 38 +- + pengine/test10/master-asymmetrical-order.exp | 6 +- + pengine/test10/master-colocation.exp | 6 +- + pengine/test10/master-demote-2.exp | 28 +- + pengine/test10/master-demote-block.exp | 2 +- + pengine/test10/master-demote.exp | 16 +- + pengine/test10/master-depend.exp | 22 +- + pengine/test10/master-dependent-ban.exp | 16 +- + pengine/test10/master-failed-demote-2.exp | 14 +- + pengine/test10/master-failed-demote.exp | 32 +- + pengine/test10/master-group.exp | 6 +- + pengine/test10/master-move.exp | 44 +- + pengine/test10/master-notify.exp | 14 +- + pengine/test10/master-ordering.exp | 50 +-- + pengine/test10/master-partially-demoted-group.exp | 58 +-- + pengine/test10/master-probed-score.exp | 282 ++++++------- + pengine/test10/master-promotion-constraint.exp | 6 +- + pengine/test10/master-pseudo.exp | 18 +- + pengine/test10/master-reattach.exp | 10 +- + pengine/test10/master-role.exp | 2 +- + pengine/test10/master-stop.exp | 2 +- + pengine/test10/master-unmanaged-monitor.exp | 20 +- + pengine/test10/master_monitor_restart.exp | 2 +- + pengine/test10/migrate-1.exp | 8 +- + pengine/test10/migrate-3.exp | 6 +- + pengine/test10/migrate-4.exp | 4 +- + pengine/test10/migrate-5.exp | 8 +- + pengine/test10/migrate-begin.exp | 6 +- + pengine/test10/migrate-both-vms.exp | 36 +- + pengine/test10/migrate-fail-2.exp | 6 +- + pengine/test10/migrate-fail-3.exp | 4 +- + pengine/test10/migrate-fail-4.exp | 6 +- + pengine/test10/migrate-fail-5.exp | 2 +- + pengine/test10/migrate-fail-6.exp | 6 +- + pengine/test10/migrate-fail-7.exp | 2 +- + pengine/test10/migrate-fail-8.exp | 4 +- + pengine/test10/migrate-fail-9.exp | 4 +- + pengine/test10/migrate-fencing.exp | 36 +- + pengine/test10/migrate-partial-1.exp | 2 +- + pengine/test10/migrate-partial-2.exp | 4 +- + pengine/test10/migrate-partial-3.exp | 4 +- + pengine/test10/migrate-partial-4.exp | 28 +- + pengine/test10/migrate-shutdown.exp | 42 +- + pengine/test10/migrate-start-complex.exp | 20 +- + pengine/test10/migrate-start.exp | 8 +- + pengine/test10/migrate-stop-complex.exp | 14 +- + pengine/test10/migrate-stop-start-complex.exp | 12 +- + pengine/test10/migrate-stop.exp | 8 +- + pengine/test10/migrate-stop_start.exp | 10 +- + pengine/test10/minimal.exp | 12 +- + pengine/test10/mon-rsc-1.exp | 8 +- + pengine/test10/mon-rsc-2.exp | 8 +- + pengine/test10/mon-rsc-3.exp | 6 +- + pengine/test10/mon-rsc-4.exp | 8 +- + pengine/test10/monitor-onfail-restart.exp | 6 +- + pengine/test10/monitor-onfail-stop.exp | 2 +- + pengine/test10/monitor-recovery.exp | 2 +- + pengine/test10/multi1.exp | 6 +- + pengine/test10/multiple-monitor-one-failed.exp | 8 +- + pengine/test10/node-maintenance-1.exp | 4 +- + pengine/test10/node-maintenance-2.exp | 6 +- + pengine/test10/not-installed-agent.exp | 12 +- + pengine/test10/not-installed-tools.exp | 6 +- + pengine/test10/not-reschedule-unneeded-monitor.exp | 6 +- + pengine/test10/notify-0.exp | 8 +- + pengine/test10/notify-1.exp | 16 +- + pengine/test10/notify-2.exp | 16 +- + pengine/test10/notify-3.exp | 30 +- + pengine/test10/novell-239079.exp | 8 +- + pengine/test10/novell-239082.exp | 30 +- + pengine/test10/novell-251689.exp | 2 +- + pengine/test10/novell-252693-2.exp | 56 +-- + pengine/test10/novell-252693-3.exp | 62 +-- + pengine/test10/novell-252693.exp | 38 +- + pengine/test10/nvpair-id-ref.exp | 16 +- + pengine/test10/obsolete-lrm-resource.exp | 6 +- + pengine/test10/one-or-more-0.exp | 16 +- + pengine/test10/one-or-more-1.exp | 8 +- + pengine/test10/one-or-more-2.exp | 14 +- + pengine/test10/one-or-more-3.exp | 10 +- + pengine/test10/one-or-more-4.exp | 14 +- + pengine/test10/one-or-more-5.exp | 20 +- + pengine/test10/one-or-more-6.exp | 2 +- + pengine/test10/one-or-more-7.exp | 2 +- + .../test10/one-or-more-unrunnnable-instances.exp | 464 ++++++++++----------- + pengine/test10/order-clone.exp | 2 +- + pengine/test10/order-mandatory.exp | 8 +- + pengine/test10/order-optional-keyword.exp | 2 +- + pengine/test10/order-optional.exp | 2 +- + pengine/test10/order-required.exp | 8 +- + pengine/test10/order-serialize-set.exp | 48 +-- + pengine/test10/order-serialize.exp | 48 +-- + pengine/test10/order-sets.exp | 24 +- + pengine/test10/order-wrong-kind.exp | 6 +- + pengine/test10/order1.exp | 18 +- + pengine/test10/order2.exp | 24 +- + pengine/test10/order3.exp | 24 +- + pengine/test10/order4.exp | 18 +- + pengine/test10/order5.exp | 32 +- + pengine/test10/order6.exp | 32 +- + pengine/test10/order7.exp | 14 +- + pengine/test10/order_constraint_stops_master.exp | 14 +- + pengine/test10/order_constraint_stops_slave.exp | 6 +- + pengine/test10/ordered-set-basic-startup.exp | 14 +- + pengine/test10/origin.exp | 2 +- + pengine/test10/orphan-0.exp | 26 +- + pengine/test10/orphan-1.exp | 34 +- + pengine/test10/orphan-2.exp | 38 +- + pengine/test10/params-0.exp | 30 +- + pengine/test10/params-1.exp | 42 +- + pengine/test10/params-2.exp | 26 +- + pengine/test10/params-4.exp | 40 +- + pengine/test10/params-5.exp | 42 +- + pengine/test10/params-6.exp | 8 +- + pengine/test10/per-node-attrs.exp | 8 +- + pengine/test10/placement-capacity.exp | 2 +- + pengine/test10/placement-location.exp | 2 +- + pengine/test10/placement-priority.exp | 2 +- + pengine/test10/placement-stickiness.exp | 2 +- + pengine/test10/probe-0.exp | 12 +- + pengine/test10/probe-1.exp | 6 +- + pengine/test10/probe-2.exp | 70 ++-- + pengine/test10/probe-timeout.exp | 20 +- + pengine/test10/quorum-1.exp | 14 +- + pengine/test10/quorum-2.exp | 12 +- + pengine/test10/quorum-3.exp | 12 +- + pengine/test10/quorum-4.exp | 6 +- + pengine/test10/quorum-5.exp | 12 +- + pengine/test10/quorum-6.exp | 20 +- + pengine/test10/rec-node-1.exp | 8 +- + pengine/test10/rec-node-10.exp | 6 +- + pengine/test10/rec-node-11.exp | 16 +- + pengine/test10/rec-node-12.exp | 86 ++-- + pengine/test10/rec-node-15.exp | 44 +- + pengine/test10/rec-node-2.exp | 28 +- + pengine/test10/rec-node-3.exp | 8 +- + pengine/test10/rec-node-4.exp | 12 +- + pengine/test10/rec-node-5.exp | 8 +- + pengine/test10/rec-node-6.exp | 12 +- + pengine/test10/rec-node-7.exp | 12 +- + pengine/test10/rec-node-8.exp | 8 +- + pengine/test10/rec-node-9.exp | 4 +- + pengine/test10/rec-rsc-0.exp | 4 +- + pengine/test10/rec-rsc-1.exp | 6 +- + pengine/test10/rec-rsc-2.exp | 8 +- + pengine/test10/rec-rsc-3.exp | 4 +- + pengine/test10/rec-rsc-4.exp | 2 +- + pengine/test10/rec-rsc-5.exp | 12 +- + pengine/test10/rec-rsc-6.exp | 6 +- + pengine/test10/rec-rsc-7.exp | 4 +- + pengine/test10/rec-rsc-9.exp | 12 +- + pengine/test10/remote-disable.exp | 4 +- + pengine/test10/remote-fence-before-reconnect.exp | 6 +- + pengine/test10/remote-fence-unclean.exp | 22 +- + pengine/test10/remote-move.exp | 14 +- + pengine/test10/remote-orphaned.exp | 14 +- + pengine/test10/remote-partial-migrate.exp | 82 ++-- + pengine/test10/remote-partial-migrate2.exp | 108 ++--- + pengine/test10/remote-recover-fail.exp | 22 +- + pengine/test10/remote-recover.exp | 14 +- + pengine/test10/remote-stale-node-entry.exp | 106 ++--- + pengine/test10/remote-start-fail.exp | 6 +- + pengine/test10/remote-startup-probes.exp | 24 +- + pengine/test10/remote-startup.exp | 26 +- + pengine/test10/remote-unclean2.exp | 6 +- + pengine/test10/reprobe-target_rc.exp | 4 +- + pengine/test10/resource-discovery.exp | 120 +++--- + pengine/test10/rsc-discovery-per-node.exp | 138 +++--- + pengine/test10/rsc-maintenance.exp | 4 +- + pengine/test10/rsc-sets-clone-1.exp | 64 +-- + pengine/test10/rsc-sets-clone.exp | 10 +- + pengine/test10/rsc-sets-master.exp | 18 +- + pengine/test10/rsc-sets-seq-false.exp | 24 +- + pengine/test10/rsc-sets-seq-true.exp | 24 +- + pengine/test10/rsc_dep1.exp | 12 +- + pengine/test10/rsc_dep10.exp | 10 +- + pengine/test10/rsc_dep2.exp | 16 +- + pengine/test10/rsc_dep3.exp | 12 +- + pengine/test10/rsc_dep4.exp | 20 +- + pengine/test10/rsc_dep5.exp | 16 +- + pengine/test10/rsc_dep7.exp | 18 +- + pengine/test10/rsc_dep8.exp | 16 +- + pengine/test10/simple11.exp | 12 +- + pengine/test10/simple12.exp | 12 +- + pengine/test10/simple2.exp | 6 +- + pengine/test10/simple3.exp | 2 +- + pengine/test10/simple4.exp | 2 +- + pengine/test10/simple6.exp | 6 +- + pengine/test10/simple7.exp | 4 +- + pengine/test10/simple8.exp | 8 +- + pengine/test10/site-specific-params.exp | 10 +- + pengine/test10/standby.exp | 48 +-- + pengine/test10/start-then-stop-with-unfence.exp | 18 +- + pengine/test10/stonith-0.exp | 22 +- + pengine/test10/stonith-1.exp | 48 +-- + pengine/test10/stonith-2.exp | 4 +- + pengine/test10/stonith-3.exp | 10 +- + pengine/test10/stop-failure-with-fencing.exp | 2 +- + pengine/test10/stopped-monitor-00.exp | 10 +- + pengine/test10/stopped-monitor-01.exp | 6 +- + pengine/test10/stopped-monitor-02.exp | 10 +- + pengine/test10/stopped-monitor-03.exp | 4 +- + pengine/test10/stopped-monitor-08.exp | 10 +- + pengine/test10/stopped-monitor-20.exp | 8 +- + pengine/test10/stopped-monitor-21.exp | 4 +- + pengine/test10/stopped-monitor-22.exp | 8 +- + pengine/test10/stopped-monitor-23.exp | 6 +- + pengine/test10/stopped-monitor-25.exp | 4 +- + pengine/test10/stopped-monitor-27.exp | 4 +- + pengine/test10/stopped-monitor-30.exp | 4 +- + pengine/test10/stopped-monitor-31.exp | 4 +- + pengine/test10/systemhealth2.exp | 16 +- + pengine/test10/systemhealth3.exp | 16 +- + pengine/test10/systemhealthm2.exp | 16 +- + pengine/test10/systemhealthm3.exp | 6 +- + pengine/test10/systemhealthn2.exp | 16 +- + pengine/test10/systemhealthn3.exp | 16 +- + pengine/test10/systemhealtho2.exp | 6 +- + pengine/test10/systemhealtho3.exp | 6 +- + pengine/test10/systemhealthp2.exp | 14 +- + pengine/test10/systemhealthp3.exp | 6 +- + pengine/test10/tags-coloc-order-1.exp | 24 +- + pengine/test10/tags-coloc-order-2.exp | 72 ++-- + pengine/test10/tags-location.exp | 36 +- + pengine/test10/tags-ticket.exp | 24 +- + pengine/test10/target-0.exp | 30 +- + pengine/test10/target-1.exp | 26 +- + pengine/test10/target-2.exp | 32 +- + pengine/test10/template-1.exp | 12 +- + pengine/test10/template-2.exp | 12 +- + pengine/test10/template-3.exp | 20 +- + pengine/test10/template-clone-group.exp | 16 +- + pengine/test10/template-clone-primitive.exp | 8 +- + pengine/test10/template-coloc-1.exp | 24 +- + pengine/test10/template-coloc-2.exp | 24 +- + pengine/test10/template-coloc-3.exp | 36 +- + pengine/test10/template-order-1.exp | 24 +- + pengine/test10/template-order-2.exp | 24 +- + pengine/test10/template-order-3.exp | 36 +- + pengine/test10/template-rsc-sets-1.exp | 30 +- + pengine/test10/template-rsc-sets-2.exp | 30 +- + pengine/test10/template-rsc-sets-3.exp | 30 +- + pengine/test10/template-rsc-sets-4.exp | 12 +- + pengine/test10/template-ticket.exp | 12 +- + pengine/test10/ticket-clone-1.exp | 4 +- + pengine/test10/ticket-clone-10.exp | 4 +- + pengine/test10/ticket-clone-11.exp | 8 +- + pengine/test10/ticket-clone-14.exp | 4 +- + pengine/test10/ticket-clone-15.exp | 4 +- + pengine/test10/ticket-clone-17.exp | 4 +- + pengine/test10/ticket-clone-18.exp | 4 +- + pengine/test10/ticket-clone-2.exp | 8 +- + pengine/test10/ticket-clone-20.exp | 4 +- + pengine/test10/ticket-clone-23.exp | 4 +- + pengine/test10/ticket-clone-3.exp | 4 +- + pengine/test10/ticket-clone-4.exp | 4 +- + pengine/test10/ticket-clone-5.exp | 8 +- + pengine/test10/ticket-clone-6.exp | 4 +- + pengine/test10/ticket-clone-7.exp | 4 +- + pengine/test10/ticket-clone-8.exp | 8 +- + pengine/test10/ticket-group-1.exp | 8 +- + pengine/test10/ticket-group-10.exp | 8 +- + pengine/test10/ticket-group-11.exp | 8 +- + pengine/test10/ticket-group-14.exp | 4 +- + pengine/test10/ticket-group-15.exp | 4 +- + pengine/test10/ticket-group-17.exp | 4 +- + pengine/test10/ticket-group-18.exp | 4 +- + pengine/test10/ticket-group-2.exp | 8 +- + pengine/test10/ticket-group-20.exp | 4 +- + pengine/test10/ticket-group-23.exp | 4 +- + pengine/test10/ticket-group-3.exp | 4 +- + pengine/test10/ticket-group-4.exp | 8 +- + pengine/test10/ticket-group-5.exp | 8 +- + pengine/test10/ticket-group-6.exp | 4 +- + pengine/test10/ticket-group-7.exp | 8 +- + pengine/test10/ticket-group-8.exp | 8 +- + pengine/test10/ticket-master-1.exp | 4 +- + pengine/test10/ticket-master-10.exp | 8 +- + pengine/test10/ticket-master-11.exp | 2 +- + pengine/test10/ticket-master-14.exp | 6 +- + pengine/test10/ticket-master-15.exp | 6 +- + pengine/test10/ticket-master-17.exp | 2 +- + pengine/test10/ticket-master-18.exp | 2 +- + pengine/test10/ticket-master-2.exp | 6 +- + pengine/test10/ticket-master-20.exp | 2 +- + pengine/test10/ticket-master-21.exp | 2 +- + pengine/test10/ticket-master-23.exp | 2 +- + pengine/test10/ticket-master-3.exp | 6 +- + pengine/test10/ticket-master-4.exp | 8 +- + pengine/test10/ticket-master-5.exp | 2 +- + pengine/test10/ticket-master-6.exp | 2 +- + pengine/test10/ticket-master-7.exp | 8 +- + pengine/test10/ticket-master-8.exp | 2 +- + pengine/test10/ticket-master-9.exp | 2 +- + pengine/test10/ticket-primitive-1.exp | 4 +- + pengine/test10/ticket-primitive-10.exp | 4 +- + pengine/test10/ticket-primitive-11.exp | 4 +- + pengine/test10/ticket-primitive-14.exp | 2 +- + pengine/test10/ticket-primitive-15.exp | 2 +- + pengine/test10/ticket-primitive-17.exp | 2 +- + pengine/test10/ticket-primitive-18.exp | 2 +- + pengine/test10/ticket-primitive-2.exp | 4 +- + pengine/test10/ticket-primitive-20.exp | 2 +- + pengine/test10/ticket-primitive-23.exp | 2 +- + pengine/test10/ticket-primitive-3.exp | 2 +- + pengine/test10/ticket-primitive-4.exp | 4 +- + pengine/test10/ticket-primitive-5.exp | 4 +- + pengine/test10/ticket-primitive-6.exp | 2 +- + pengine/test10/ticket-primitive-7.exp | 4 +- + pengine/test10/ticket-primitive-8.exp | 4 +- + pengine/test10/ticket-rsc-sets-1.exp | 24 +- + pengine/test10/ticket-rsc-sets-10.exp | 12 +- + pengine/test10/ticket-rsc-sets-12.exp | 6 +- + pengine/test10/ticket-rsc-sets-13.exp | 12 +- + pengine/test10/ticket-rsc-sets-14.exp | 12 +- + pengine/test10/ticket-rsc-sets-2.exp | 22 +- + pengine/test10/ticket-rsc-sets-3.exp | 12 +- + pengine/test10/ticket-rsc-sets-4.exp | 24 +- + pengine/test10/ticket-rsc-sets-5.exp | 12 +- + pengine/test10/ticket-rsc-sets-6.exp | 10 +- + pengine/test10/ticket-rsc-sets-7.exp | 12 +- + pengine/test10/ticket-rsc-sets-9.exp | 12 +- + pengine/test10/unfence-definition.exp | 28 +- + pengine/test10/unfence-parameters.exp | 30 +- + pengine/test10/unfence-startup.exp | 14 +- + pengine/test10/unmanaged-block-restart.exp | 4 +- + pengine/test10/unmanaged-master.exp | 4 +- + pengine/test10/unrunnable-1.exp | 22 +- + pengine/test10/use-after-free-merge.exp | 22 +- + pengine/test10/utilization-check-allowed-nodes.exp | 10 +- + pengine/test10/utilization-order1.exp | 4 +- + pengine/test10/utilization-order2.exp | 10 +- + pengine/test10/utilization-order3.exp | 8 +- + pengine/test10/utilization-order4.exp | 18 +- + pengine/test10/utilization-shuffle.exp | 24 +- + pengine/test10/utilization.exp | 10 +- + pengine/test10/whitebox-asymmetric.exp | 8 +- + pengine/test10/whitebox-fail1.exp | 18 +- + pengine/test10/whitebox-fail2.exp | 18 +- + pengine/test10/whitebox-fail3.exp | 18 +- + pengine/test10/whitebox-imply-stop-on-fence.exp | 24 +- + pengine/test10/whitebox-migrate1.exp | 28 +- + pengine/test10/whitebox-move.exp | 22 +- + pengine/test10/whitebox-ms-ordering-move.exp | 18 +- + pengine/test10/whitebox-ms-ordering.exp | 38 +- + pengine/test10/whitebox-nested-group.exp | 90 ++-- + pengine/test10/whitebox-orphan-ms.exp | 50 +-- + pengine/test10/whitebox-orphaned.exp | 28 +- + pengine/test10/whitebox-start.exp | 22 +- + pengine/test10/whitebox-stop.exp | 12 +- + pengine/test10/whitebox-unexpectedly-running.exp | 10 +- + 604 files changed, 6494 insertions(+), 6494 deletions(-) + +diff --git a/pengine/test10/1-a-then-bm-move-b.exp b/pengine/test10/1-a-then-bm-move-b.exp +index 542ddfa..b29c4ff 100644 +--- a/pengine/test10/1-a-then-bm-move-b.exp ++++ b/pengine/test10/1-a-then-bm-move-b.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -53,7 +53,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/10-a-then-bm-b-move-a-clone.exp b/pengine/test10/10-a-then-bm-b-move-a-clone.exp +index aa13464..c7ea7dc 100644 +--- a/pengine/test10/10-a-then-bm-b-move-a-clone.exp ++++ b/pengine/test10/10-a-then-bm-b-move-a-clone.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -80,7 +80,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp +index 6126486..76652ab 100644 +--- a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp ++++ b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -35,7 +35,7 @@ + + + +- ++ + + + +@@ -101,7 +101,7 @@ + + + +- ++ + + + +@@ -117,7 +117,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/1360.exp b/pengine/test10/1360.exp +index 17858ff..4db2681 100644 +--- a/pengine/test10/1360.exp ++++ b/pengine/test10/1360.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/1484.exp b/pengine/test10/1484.exp +index 8921198..91807b8 100644 +--- a/pengine/test10/1484.exp ++++ b/pengine/test10/1484.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/1494.exp b/pengine/test10/1494.exp +index 8145b57..b5e03e9 100644 +--- a/pengine/test10/1494.exp ++++ b/pengine/test10/1494.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/2-am-then-b-move-a.exp b/pengine/test10/2-am-then-b-move-a.exp +index a12bd6d..ed7d3fa 100644 +--- a/pengine/test10/2-am-then-b-move-a.exp ++++ b/pengine/test10/2-am-then-b-move-a.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -53,7 +53,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/3-am-then-bm-both-migrate.exp b/pengine/test10/3-am-then-bm-both-migrate.exp +index deecc5d..aac63ba 100644 +--- a/pengine/test10/3-am-then-bm-both-migrate.exp ++++ b/pengine/test10/3-am-then-bm-both-migrate.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -69,7 +69,7 @@ + + + +- ++ + + + +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -95,7 +95,7 @@ + + + +- ++ + + + +@@ -126,7 +126,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/4-am-then-bm-b-not-migratable.exp b/pengine/test10/4-am-then-bm-b-not-migratable.exp +index 7684002..66aa4ad 100644 +--- a/pengine/test10/4-am-then-bm-b-not-migratable.exp ++++ b/pengine/test10/4-am-then-bm-b-not-migratable.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -57,7 +57,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -86,7 +86,7 @@ + + + +- ++ + + + +@@ -102,7 +102,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/5-am-then-bm-a-not-migratable.exp b/pengine/test10/5-am-then-bm-a-not-migratable.exp +index 199ef39..c3bb278 100644 +--- a/pengine/test10/5-am-then-bm-a-not-migratable.exp ++++ b/pengine/test10/5-am-then-bm-a-not-migratable.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -71,7 +71,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/594.exp b/pengine/test10/594.exp +index bb15a36..5d58ab4 100644 +--- a/pengine/test10/594.exp ++++ b/pengine/test10/594.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -35,7 +35,7 @@ + + + +- ++ + + + +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -57,7 +57,7 @@ + + + +- ++ + + + +@@ -66,7 +66,7 @@ + + + +- ++ + + + +@@ -79,7 +79,7 @@ + + + +- ++ + + + +@@ -98,7 +98,7 @@ + + + +- ++ + + + +@@ -111,7 +111,7 @@ + + + +- ++ + + + +@@ -120,7 +120,7 @@ + + + +- ++ + + + +@@ -136,7 +136,7 @@ + + + +- ++ + + + +@@ -145,7 +145,7 @@ + + + +- ++ + + + +@@ -158,7 +158,7 @@ + + + +- ++ + + + +@@ -203,7 +203,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/6-migrate-group.exp b/pengine/test10/6-migrate-group.exp +index 0041c40..23c5fae 100644 +--- a/pengine/test10/6-migrate-group.exp ++++ b/pengine/test10/6-migrate-group.exp +@@ -65,7 +65,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -87,7 +87,7 @@ + + + +- ++ + + + +@@ -118,7 +118,7 @@ + + + +- ++ + + + +@@ -137,7 +137,7 @@ + + + +- ++ + + + +@@ -150,7 +150,7 @@ + + + +- ++ + + + +@@ -163,7 +163,7 @@ + + + +- ++ + + + +@@ -197,7 +197,7 @@ + + + +- ++ + + + +@@ -216,7 +216,7 @@ + + + +- ++ + + + +@@ -229,7 +229,7 @@ + + + +- ++ + + + +@@ -242,7 +242,7 @@ + + + +- ++ + + + +@@ -276,7 +276,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/662.exp b/pengine/test10/662.exp +index b18a5ea..d05f4d3 100644 +--- a/pengine/test10/662.exp ++++ b/pengine/test10/662.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -21,7 +21,7 @@ + + + +- ++ + + + +@@ -30,7 +30,7 @@ + + + +- ++ + + + +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -57,7 +57,7 @@ + + + +- ++ + + + +@@ -70,7 +70,7 @@ + + + +- ++ + + + +@@ -92,7 +92,7 @@ + + + +- ++ + + + +@@ -111,7 +111,7 @@ + + + +- ++ + + + +@@ -120,7 +120,7 @@ + + + +- ++ + + + +@@ -129,7 +129,7 @@ + + + +- ++ + + + +@@ -138,7 +138,7 @@ + + + +- ++ + + + +@@ -147,7 +147,7 @@ + + + +- ++ + + + +@@ -156,7 +156,7 @@ + + + +- ++ + + + +@@ -165,7 +165,7 @@ + + + +- ++ + + + +@@ -174,7 +174,7 @@ + + + +- ++ + + + +@@ -183,7 +183,7 @@ + + + +- ++ + + + +@@ -202,7 +202,7 @@ + + + +- ++ + + + +@@ -211,7 +211,7 @@ + + + +- ++ + + + +@@ -220,7 +220,7 @@ + + + +- ++ + + + +@@ -229,7 +229,7 @@ + + + +- ++ + + + +@@ -238,7 +238,7 @@ + + + +- ++ + + + +@@ -247,7 +247,7 @@ + + + +- ++ + + + +@@ -256,7 +256,7 @@ + + + +- ++ + + + +@@ -265,7 +265,7 @@ + + + +- ++ + + + +@@ -274,7 +274,7 @@ + + + +- ++ + + + +@@ -305,7 +305,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/696.exp b/pengine/test10/696.exp +index 42477ab..71befb4 100644 +--- a/pengine/test10/696.exp ++++ b/pengine/test10/696.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -41,7 +41,7 @@ + + + +- ++ + + + +@@ -50,7 +50,7 @@ + + + +- ++ + + + +@@ -63,7 +63,7 @@ + + + +- ++ + + + +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -98,7 +98,7 @@ + + + +- ++ + + + +@@ -107,7 +107,7 @@ + + + +- ++ + + + +@@ -116,7 +116,7 @@ + + + +- ++ + + + +@@ -129,7 +129,7 @@ + + + +- ++ + + + +@@ -145,7 +145,7 @@ + + + +- ++ + + + +@@ -154,7 +154,7 @@ + + + +- ++ + + + +@@ -163,7 +163,7 @@ + + + +- ++ + + + +@@ -172,7 +172,7 @@ + + + +- ++ + + + +@@ -181,7 +181,7 @@ + + + +- ++ + + + +@@ -190,7 +190,7 @@ + + + +- ++ + + + +@@ -199,7 +199,7 @@ + + + +- ++ + + + +@@ -208,7 +208,7 @@ + + + +- ++ + + + +@@ -217,7 +217,7 @@ + + + +- ++ + + + +@@ -226,7 +226,7 @@ + + + +- ++ + + + +@@ -235,7 +235,7 @@ + + + +- ++ + + + +@@ -244,7 +244,7 @@ + + + +- ++ + + + +@@ -257,7 +257,7 @@ + + + +- ++ + + + +@@ -270,7 +270,7 @@ + + + +- ++ + + + +@@ -279,7 +279,7 @@ + + + +- ++ + + + +@@ -288,7 +288,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/7-migrate-group-one-unmigratable.exp b/pengine/test10/7-migrate-group-one-unmigratable.exp +index 3988eb3..8005321 100644 +--- a/pengine/test10/7-migrate-group-one-unmigratable.exp ++++ b/pengine/test10/7-migrate-group-one-unmigratable.exp +@@ -65,7 +65,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -91,7 +91,7 @@ + + + +- ++ + + + +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -141,7 +141,7 @@ + + + +- ++ + + + +@@ -154,7 +154,7 @@ + + + +- ++ + + + +@@ -173,7 +173,7 @@ + + + +- ++ + + + +@@ -189,7 +189,7 @@ + + + +- ++ + + + +@@ -202,7 +202,7 @@ + + + +- ++ + + + +@@ -221,7 +221,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/726.exp b/pengine/test10/726.exp +index bb9345e..a1a0be4 100644 +--- a/pengine/test10/726.exp ++++ b/pengine/test10/726.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -21,7 +21,7 @@ + + + +- ++ + + + +@@ -30,7 +30,7 @@ + + + +- ++ + + + +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -74,7 +74,7 @@ + + + +- ++ + + + +@@ -83,7 +83,7 @@ + + + +- ++ + + + +@@ -92,7 +92,7 @@ + + + +- ++ + + + +@@ -101,7 +101,7 @@ + + + +- ++ + + + +@@ -110,7 +110,7 @@ + + + +- ++ + + + +@@ -123,7 +123,7 @@ + + + +- ++ + + + +@@ -145,7 +145,7 @@ + + + +- ++ + + + +@@ -164,7 +164,7 @@ + + + +- ++ + + + +@@ -173,7 +173,7 @@ + + + +- ++ + + + +@@ -182,7 +182,7 @@ + + + +- ++ + + + +@@ -191,7 +191,7 @@ + + + +- ++ + + + +@@ -204,7 +204,7 @@ + + + +- ++ + + + +@@ -226,7 +226,7 @@ + + + +- ++ + + + +@@ -235,7 +235,7 @@ + + + +- ++ + + + +@@ -244,7 +244,7 @@ + + + +- ++ + + + +@@ -253,7 +253,7 @@ + + + +- ++ + + + +@@ -262,7 +262,7 @@ + + + +- ++ + + + +@@ -271,7 +271,7 @@ + + + +- ++ + + + +@@ -280,7 +280,7 @@ + + + +- ++ + + + +@@ -289,7 +289,7 @@ + + + +- ++ + + + +@@ -298,7 +298,7 @@ + + + +- ++ + + + +@@ -311,7 +311,7 @@ + + + +- ++ + + + +@@ -324,7 +324,7 @@ + + + +- ++ + + + +@@ -333,7 +333,7 @@ + + + +- ++ + + + +@@ -342,7 +342,7 @@ + + + +- ++ + + + +@@ -355,7 +355,7 @@ + + + +- ++ + + + +@@ -368,7 +368,7 @@ + + + +- ++ + + + +@@ -377,7 +377,7 @@ + + + +- ++ + + + +@@ -386,7 +386,7 @@ + + + +- ++ + + + +@@ -395,7 +395,7 @@ + + + +- ++ + + + +@@ -408,7 +408,7 @@ + + + +- ++ + + + +@@ -421,7 +421,7 @@ + + + +- ++ + + + +@@ -430,7 +430,7 @@ + + + +- ++ + + + +@@ -439,7 +439,7 @@ + + + +- ++ + + + +@@ -448,7 +448,7 @@ + + + +- ++ + + + +@@ -457,7 +457,7 @@ + + + +- ++ + + + +@@ -470,7 +470,7 @@ + + + +- ++ + + + +@@ -483,7 +483,7 @@ + + + +- ++ + + + +@@ -492,7 +492,7 @@ + + + +- ++ + + + +@@ -501,7 +501,7 @@ + + + +- ++ + + + +@@ -510,7 +510,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/735.exp b/pengine/test10/735.exp +index 10bc0d9..b5f89bf 100644 +--- a/pengine/test10/735.exp ++++ b/pengine/test10/735.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -34,7 +34,7 @@ + + + +- ++ + + + +@@ -47,7 +47,7 @@ + + + +- ++ + + + +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -69,7 +69,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -91,7 +91,7 @@ + + + +- ++ + + + +@@ -104,7 +104,7 @@ + + + +- ++ + + + +@@ -113,7 +113,7 @@ + + + +- ++ + + + +@@ -126,7 +126,7 @@ + + + +- ++ + + + +@@ -139,7 +139,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/764.exp b/pengine/test10/764.exp +index 4f9aa06..5101644 100644 +--- a/pengine/test10/764.exp ++++ b/pengine/test10/764.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -38,7 +38,7 @@ + + + +- ++ + + + +@@ -47,7 +47,7 @@ + + + +- ++ + + + +@@ -60,7 +60,7 @@ + + + +- ++ + + + +@@ -69,7 +69,7 @@ + + + +- ++ + + + +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -91,7 +91,7 @@ + + + +- ++ + + + +@@ -104,7 +104,7 @@ + + + +- ++ + + + +@@ -113,7 +113,7 @@ + + + +- ++ + + + +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -131,7 +131,7 @@ + + + +- ++ + + + +@@ -140,7 +140,7 @@ + + + +- ++ + + + +@@ -149,7 +149,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/797.exp b/pengine/test10/797.exp +index 6126a4c..51f64cb 100644 +--- a/pengine/test10/797.exp ++++ b/pengine/test10/797.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -21,7 +21,7 @@ + + + +- ++ + + + +@@ -34,7 +34,7 @@ + + + +- ++ + + + +@@ -47,7 +47,7 @@ + + + +- ++ + + + +@@ -63,7 +63,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -97,7 +97,7 @@ + + + +- ++ + + + +@@ -106,7 +106,7 @@ + + + +- ++ + + + +@@ -115,7 +115,7 @@ + + + +- ++ + + + +@@ -128,7 +128,7 @@ + + + +- ++ + + + +@@ -137,7 +137,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +@@ -159,7 +159,7 @@ + + + +- ++ + + + +@@ -168,7 +168,7 @@ + + + +- ++ + + + +@@ -187,7 +187,7 @@ + + + +- ++ + + + +@@ -200,7 +200,7 @@ + + + +- ++ + + + +@@ -213,7 +213,7 @@ + + + +- ++ + + + +@@ -226,7 +226,7 @@ + + + +- ++ + + + +@@ -239,7 +239,7 @@ + + + +- ++ + + + +@@ -248,7 +248,7 @@ + + + +- ++ + + + +@@ -257,7 +257,7 @@ + + + +- ++ + + + +@@ -266,7 +266,7 @@ + + + +- ++ + + + +@@ -342,7 +342,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/8-am-then-bm-a-migrating-b-stopping.exp b/pengine/test10/8-am-then-bm-a-migrating-b-stopping.exp +index 2c64ae6..6fb28d5 100644 +--- a/pengine/test10/8-am-then-bm-a-migrating-b-stopping.exp ++++ b/pengine/test10/8-am-then-bm-a-migrating-b-stopping.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -57,7 +57,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/829.exp b/pengine/test10/829.exp +index 5975a6b..115a39f 100644 +--- a/pengine/test10/829.exp ++++ b/pengine/test10/829.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -21,7 +21,7 @@ + + + +- ++ + + + +@@ -30,7 +30,7 @@ + + + +- ++ + + + +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -98,7 +98,7 @@ + + + +- ++ + + + +@@ -107,7 +107,7 @@ + + + +- ++ + + + +@@ -116,7 +116,7 @@ + + + +- ++ + + + +@@ -125,7 +125,7 @@ + + + +- ++ + + + +@@ -134,7 +134,7 @@ + + + +- ++ + + + +@@ -143,7 +143,7 @@ + + + +- ++ + + + +@@ -152,7 +152,7 @@ + + + +- ++ + + + +@@ -179,7 +179,7 @@ + + + +- ++ + + + +@@ -188,7 +188,7 @@ + + + +- ++ + + + +@@ -197,7 +197,7 @@ + + + +- ++ + + + +@@ -206,7 +206,7 @@ + + + +- ++ + + + +@@ -215,7 +215,7 @@ + + + +- ++ + + + +@@ -224,7 +224,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/9-am-then-bm-b-migrating-a-stopping.exp b/pengine/test10/9-am-then-bm-b-migrating-a-stopping.exp +index 759a28f..10f0f8c 100644 +--- a/pengine/test10/9-am-then-bm-b-migrating-a-stopping.exp ++++ b/pengine/test10/9-am-then-bm-b-migrating-a-stopping.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/994-2.exp b/pengine/test10/994-2.exp +index 1d8def6..c00f2f2 100644 +--- a/pengine/test10/994-2.exp ++++ b/pengine/test10/994-2.exp +@@ -57,7 +57,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -86,7 +86,7 @@ + + + +- ++ + + + +@@ -99,7 +99,7 @@ + + + +- ++ + + + +@@ -115,7 +115,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/994.exp b/pengine/test10/994.exp +index ddc7551..b0111ff 100644 +--- a/pengine/test10/994.exp ++++ b/pengine/test10/994.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -69,7 +69,7 @@ + + + +- ++ + + + +@@ -82,7 +82,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/a-demote-then-b-migrate.exp b/pengine/test10/a-demote-then-b-migrate.exp +index f75ca52..32ee3f9 100644 +--- a/pengine/test10/a-demote-then-b-migrate.exp ++++ b/pengine/test10/a-demote-then-b-migrate.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -74,7 +74,7 @@ + + + +- ++ + + + +@@ -90,7 +90,7 @@ + + + +- ++ + + + +@@ -99,7 +99,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +@@ -125,7 +125,7 @@ + + + +- ++ + + + +@@ -138,7 +138,7 @@ + + + +- ++ + + + +@@ -151,7 +151,7 @@ + + + +- ++ + + + +@@ -170,7 +170,7 @@ + + + +- ++ + + + +@@ -186,7 +186,7 @@ + + + +- ++ + + + +@@ -374,7 +374,7 @@ + + + +- ++ + + + +@@ -387,7 +387,7 @@ + + + +- ++ + + + +@@ -400,7 +400,7 @@ + + + +- ++ + + + +@@ -428,7 +428,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/a-promote-then-b-migrate.exp b/pengine/test10/a-promote-then-b-migrate.exp +index e527de6..727036a 100644 +--- a/pengine/test10/a-promote-then-b-migrate.exp ++++ b/pengine/test10/a-promote-then-b-migrate.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -71,7 +71,7 @@ + + + +- ++ + + + +@@ -87,7 +87,7 @@ + + + +- ++ + + + +@@ -179,7 +179,7 @@ + + + +- ++ + + + +@@ -192,7 +192,7 @@ + + + +- ++ + + + +@@ -205,7 +205,7 @@ + + + +- ++ + + + +@@ -236,7 +236,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/anti-colocation-master.exp b/pengine/test10/anti-colocation-master.exp +index 00e20b1..011bfdd 100644 +--- a/pengine/test10/anti-colocation-master.exp ++++ b/pengine/test10/anti-colocation-master.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -41,7 +41,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/anti-colocation-order.exp b/pengine/test10/anti-colocation-order.exp +index 24de4d2..fa5304f 100644 +--- a/pengine/test10/anti-colocation-order.exp ++++ b/pengine/test10/anti-colocation-order.exp +@@ -62,7 +62,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -94,7 +94,7 @@ + + + +- ++ + + + +@@ -113,7 +113,7 @@ + + + +- ++ + + + +@@ -152,7 +152,7 @@ + + + +- ++ + + + +@@ -168,7 +168,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/anti-colocation-slave.exp b/pengine/test10/anti-colocation-slave.exp +index afb8983..dc3eb08 100644 +--- a/pengine/test10/anti-colocation-slave.exp ++++ b/pengine/test10/anti-colocation-slave.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -80,7 +80,7 @@ + + + +- ++ + + + +@@ -96,7 +96,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/asymmetric.exp b/pengine/test10/asymmetric.exp +index a18ba0a..6284925 100644 +--- a/pengine/test10/asymmetric.exp ++++ b/pengine/test10/asymmetric.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -21,7 +21,7 @@ + + + +- ++ + + + +@@ -30,7 +30,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/asymmetrical-order-move.exp b/pengine/test10/asymmetrical-order-move.exp +index 458094f..ac48583 100644 +--- a/pengine/test10/asymmetrical-order-move.exp ++++ b/pengine/test10/asymmetrical-order-move.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/attrs1.exp b/pengine/test10/attrs1.exp +index 69f93df..7df2479 100644 +--- a/pengine/test10/attrs1.exp ++++ b/pengine/test10/attrs1.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/attrs2.exp b/pengine/test10/attrs2.exp +index 69f93df..7df2479 100644 +--- a/pengine/test10/attrs2.exp ++++ b/pengine/test10/attrs2.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/attrs3.exp b/pengine/test10/attrs3.exp +index 52309b8..d86378f 100644 +--- a/pengine/test10/attrs3.exp ++++ b/pengine/test10/attrs3.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/attrs4.exp b/pengine/test10/attrs4.exp +index 52309b8..d86378f 100644 +--- a/pengine/test10/attrs4.exp ++++ b/pengine/test10/attrs4.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/attrs5.exp b/pengine/test10/attrs5.exp +index 7d7b6c4..f74da82 100644 +--- a/pengine/test10/attrs5.exp ++++ b/pengine/test10/attrs5.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/attrs6.exp b/pengine/test10/attrs6.exp +index 52309b8..d86378f 100644 +--- a/pengine/test10/attrs6.exp ++++ b/pengine/test10/attrs6.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/attrs7.exp b/pengine/test10/attrs7.exp +index 69f93df..7df2479 100644 +--- a/pengine/test10/attrs7.exp ++++ b/pengine/test10/attrs7.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/attrs8.exp b/pengine/test10/attrs8.exp +index 69f93df..7df2479 100644 +--- a/pengine/test10/attrs8.exp ++++ b/pengine/test10/attrs8.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/balanced.exp b/pengine/test10/balanced.exp +index db8d65b..cfe66f8 100644 +--- a/pengine/test10/balanced.exp ++++ b/pengine/test10/balanced.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -22,7 +22,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -40,7 +40,7 @@ + + + +- ++ + + + +@@ -59,7 +59,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/base-score.exp b/pengine/test10/base-score.exp +index baa4f77..a13d02f 100644 +--- a/pengine/test10/base-score.exp ++++ b/pengine/test10/base-score.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -34,7 +34,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bnc-515172.exp b/pengine/test10/bnc-515172.exp +index 45fc9b8..4086cff 100644 +--- a/pengine/test10/bnc-515172.exp ++++ b/pengine/test10/bnc-515172.exp +@@ -26,7 +26,7 @@ + + + +- ++ + + + +@@ -39,7 +39,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1572-1.exp b/pengine/test10/bug-1572-1.exp +index 27e773c..e5355d8 100644 +--- a/pengine/test10/bug-1572-1.exp ++++ b/pengine/test10/bug-1572-1.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -94,7 +94,7 @@ + + + +- ++ + + + +@@ -107,7 +107,7 @@ + + + +- ++ + + + +@@ -120,7 +120,7 @@ + + + +- ++ + + + +@@ -133,7 +133,7 @@ + + + +- ++ + + + +@@ -152,7 +152,7 @@ + + + +- ++ + + + +@@ -168,7 +168,7 @@ + + + +- ++ + + + +@@ -181,7 +181,7 @@ + + + +- ++ + + + +@@ -627,7 +627,7 @@ + + + +- ++ + + + +@@ -643,7 +643,7 @@ + + + +- ++ + + + +@@ -659,7 +659,7 @@ + + + +- ++ + + + +@@ -678,7 +678,7 @@ + + + +- ++ + + + +@@ -694,7 +694,7 @@ + + + +- ++ + + + +@@ -707,7 +707,7 @@ + + + +- ++ + + + +@@ -726,7 +726,7 @@ + + + +- ++ + + + +@@ -739,7 +739,7 @@ + + + +- ++ + + + +@@ -751,7 +751,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1572-2.exp b/pengine/test10/bug-1572-2.exp +index 6193c86..e346e2b 100644 +--- a/pengine/test10/bug-1572-2.exp ++++ b/pengine/test10/bug-1572-2.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -94,7 +94,7 @@ + + + +- ++ + + + +@@ -107,7 +107,7 @@ + + + +- ++ + + + +@@ -328,7 +328,7 @@ + + + +- ++ + + + +@@ -344,7 +344,7 @@ + + + +- ++ + + + +@@ -360,7 +360,7 @@ + + + +- ++ + + + +@@ -372,7 +372,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1573.exp b/pengine/test10/bug-1573.exp +index 651d263..79700fc 100644 +--- a/pengine/test10/bug-1573.exp ++++ b/pengine/test10/bug-1573.exp +@@ -26,7 +26,7 @@ + + + +- ++ + + + +@@ -38,7 +38,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1685.exp b/pengine/test10/bug-1685.exp +index f16040b..16ff43b 100644 +--- a/pengine/test10/bug-1685.exp ++++ b/pengine/test10/bug-1685.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +@@ -164,7 +164,7 @@ + + + +- ++ + + + +@@ -177,7 +177,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1718.exp b/pengine/test10/bug-1718.exp +index 13e6bda..7c5db02 100644 +--- a/pengine/test10/bug-1718.exp ++++ b/pengine/test10/bug-1718.exp +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -58,7 +58,7 @@ + + + +- ++ + + + +@@ -67,7 +67,7 @@ + + + +- ++ + + + +@@ -76,7 +76,7 @@ + + + +- ++ + + + +@@ -85,7 +85,7 @@ + + + +- ++ + + + +@@ -94,7 +94,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1765.exp b/pengine/test10/bug-1765.exp +index 4b8d04b..2c01402 100644 +--- a/pengine/test10/bug-1765.exp ++++ b/pengine/test10/bug-1765.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1820-1.exp b/pengine/test10/bug-1820-1.exp +index 03e5a08..7a41069 100644 +--- a/pengine/test10/bug-1820-1.exp ++++ b/pengine/test10/bug-1820-1.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -93,7 +93,7 @@ + + + +- ++ + + + +@@ -106,7 +106,7 @@ + + + +- ++ + + + +@@ -115,7 +115,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +@@ -165,7 +165,7 @@ + + + +- ++ + + + +@@ -178,7 +178,7 @@ + + + +- ++ + + + +@@ -191,7 +191,7 @@ + + + +- ++ + + + +@@ -225,7 +225,7 @@ + + + +- ++ + + + +@@ -240,7 +240,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1820.exp b/pengine/test10/bug-1820.exp +index 4ec4a4d..35c0e65 100644 +--- a/pengine/test10/bug-1820.exp ++++ b/pengine/test10/bug-1820.exp +@@ -59,7 +59,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +@@ -131,7 +131,7 @@ + + + +- ++ + + + +@@ -144,7 +144,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -191,7 +191,7 @@ + + + +- ++ + + + +@@ -206,7 +206,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1822.exp b/pengine/test10/bug-1822.exp +index 974d74d..8ce6d70 100644 +--- a/pengine/test10/bug-1822.exp ++++ b/pengine/test10/bug-1822.exp +@@ -60,7 +60,7 @@ + + + +- ++ + + + +@@ -76,7 +76,7 @@ + + + +- ++ + + + +@@ -89,7 +89,7 @@ + + + +- ++ + + + +@@ -105,7 +105,7 @@ + + + +- ++ + + + +@@ -167,7 +167,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5007-masterslave_colocation.exp b/pengine/test10/bug-5007-masterslave_colocation.exp +index 3c33624..bf83867 100644 +--- a/pengine/test10/bug-5007-masterslave_colocation.exp ++++ b/pengine/test10/bug-5007-masterslave_colocation.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -38,7 +38,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-A-start-B-start.exp b/pengine/test10/bug-5014-A-start-B-start.exp +index 154b23b..c89ed4b 100644 +--- a/pengine/test10/bug-5014-A-start-B-start.exp ++++ b/pengine/test10/bug-5014-A-start-B-start.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -38,7 +38,7 @@ + + + +- ++ + + + +@@ -51,7 +51,7 @@ + + + +- ++ + + + +@@ -67,7 +67,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-A-stop-B-started.exp b/pengine/test10/bug-5014-A-stop-B-started.exp +index 745ef4b..dc289f0 100644 +--- a/pengine/test10/bug-5014-A-stop-B-started.exp ++++ b/pengine/test10/bug-5014-A-stop-B-started.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-A-stopped-B-stopped.exp b/pengine/test10/bug-5014-A-stopped-B-stopped.exp +index 4ef72cd..0d3b9ea 100644 +--- a/pengine/test10/bug-5014-A-stopped-B-stopped.exp ++++ b/pengine/test10/bug-5014-A-stopped-B-stopped.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-CLONE-A-start-B-start.exp b/pengine/test10/bug-5014-CLONE-A-start-B-start.exp +index 38b1705..1e9013e 100644 +--- a/pengine/test10/bug-5014-CLONE-A-start-B-start.exp ++++ b/pengine/test10/bug-5014-CLONE-A-start-B-start.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -65,7 +65,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -91,7 +91,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-CLONE-A-stop-B-started.exp b/pengine/test10/bug-5014-CLONE-A-stop-B-started.exp +index 5da5b83..aa99923 100644 +--- a/pengine/test10/bug-5014-CLONE-A-stop-B-started.exp ++++ b/pengine/test10/bug-5014-CLONE-A-stop-B-started.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-CthenAthenB-C-stopped.exp b/pengine/test10/bug-5014-CthenAthenB-C-stopped.exp +index 9e9265f..0896ce2 100644 +--- a/pengine/test10/bug-5014-CthenAthenB-C-stopped.exp ++++ b/pengine/test10/bug-5014-CthenAthenB-C-stopped.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -21,7 +21,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-GROUP-A-start-B-start.exp b/pengine/test10/bug-5014-GROUP-A-start-B-start.exp +index 407f04b..598bdce 100644 +--- a/pengine/test10/bug-5014-GROUP-A-start-B-start.exp ++++ b/pengine/test10/bug-5014-GROUP-A-start-B-start.exp +@@ -26,7 +26,7 @@ + + + +- ++ + + + +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -79,7 +79,7 @@ + + + +- ++ + + + +@@ -92,7 +92,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-GROUP-A-stopped-B-started.exp b/pengine/test10/bug-5014-GROUP-A-stopped-B-started.exp +index 06749e9..0a0cb5c 100644 +--- a/pengine/test10/bug-5014-GROUP-A-stopped-B-started.exp ++++ b/pengine/test10/bug-5014-GROUP-A-stopped-B-started.exp +@@ -26,7 +26,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-ordered-set-symmetrical-false.exp b/pengine/test10/bug-5014-ordered-set-symmetrical-false.exp +index 445a77c..4525661 100644 +--- a/pengine/test10/bug-5014-ordered-set-symmetrical-false.exp ++++ b/pengine/test10/bug-5014-ordered-set-symmetrical-false.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-ordered-set-symmetrical-true.exp b/pengine/test10/bug-5014-ordered-set-symmetrical-true.exp +index e6fe6d0..d0c9593 100644 +--- a/pengine/test10/bug-5014-ordered-set-symmetrical-true.exp ++++ b/pengine/test10/bug-5014-ordered-set-symmetrical-true.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5025-1.exp b/pengine/test10/bug-5025-1.exp +index 8465dfb..053ece4 100644 +--- a/pengine/test10/bug-5025-1.exp ++++ b/pengine/test10/bug-5025-1.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -21,7 +21,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5025-3.exp b/pengine/test10/bug-5025-3.exp +index 3b01eae..eb2e2e6 100644 +--- a/pengine/test10/bug-5025-3.exp ++++ b/pengine/test10/bug-5025-3.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -21,7 +21,7 @@ + + + +- ++ + + + +@@ -34,7 +34,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5025-4.exp b/pengine/test10/bug-5025-4.exp +index 16d9fc1..a51c888 100644 +--- a/pengine/test10/bug-5025-4.exp ++++ b/pengine/test10/bug-5025-4.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -38,7 +38,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5028-bottom.exp b/pengine/test10/bug-5028-bottom.exp +index 2853f2d..f7e7050 100644 +--- a/pengine/test10/bug-5028-bottom.exp ++++ b/pengine/test10/bug-5028-bottom.exp +@@ -11,7 +11,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5028-detach.exp b/pengine/test10/bug-5028-detach.exp +index 3f7cdaf..8c3cae3 100644 +--- a/pengine/test10/bug-5028-detach.exp ++++ b/pengine/test10/bug-5028-detach.exp +@@ -2,7 +2,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5059.exp b/pengine/test10/bug-5059.exp +index 3f1fe64..b4a2e07 100644 +--- a/pengine/test10/bug-5059.exp ++++ b/pengine/test10/bug-5059.exp +@@ -57,7 +57,7 @@ + + + +- ++ + + + +@@ -70,7 +70,7 @@ + + + +- ++ + + + +@@ -83,7 +83,7 @@ + + + +- ++ + + + +@@ -96,7 +96,7 @@ + + + +- ++ + + + +@@ -109,7 +109,7 @@ + + + +- ++ + + + +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -135,7 +135,7 @@ + + + +- ++ + + + +@@ -148,7 +148,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +@@ -180,7 +180,7 @@ + + + +- ++ + + + +@@ -220,7 +220,7 @@ + + + +- ++ + + + +@@ -233,7 +233,7 @@ + + + +- ++ + + + +@@ -246,7 +246,7 @@ + + + +- ++ + + + +@@ -259,7 +259,7 @@ + + + +- ++ + + + +@@ -272,7 +272,7 @@ + + + +- ++ + + + +@@ -288,7 +288,7 @@ + + + +- ++ + + + +@@ -301,7 +301,7 @@ + + + +- ++ + + + +@@ -314,7 +314,7 @@ + + + +- ++ + + + +@@ -327,7 +327,7 @@ + + + +- ++ + + + +@@ -346,7 +346,7 @@ + + + +- ++ + + + +@@ -556,7 +556,7 @@ + + + +- ++ + + + +@@ -565,7 +565,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5069-op-disabled.exp b/pengine/test10/bug-5069-op-disabled.exp +index ca7f56c..9653af1 100644 +--- a/pengine/test10/bug-5069-op-disabled.exp ++++ b/pengine/test10/bug-5069-op-disabled.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5140-require-all-false.exp b/pengine/test10/bug-5140-require-all-false.exp +index bce90d0..9751b9b 100644 +--- a/pengine/test10/bug-5140-require-all-false.exp ++++ b/pengine/test10/bug-5140-require-all-false.exp +@@ -36,7 +36,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5143-ms-shuffle.exp b/pengine/test10/bug-5143-ms-shuffle.exp +index 5fffd20..c591421 100644 +--- a/pengine/test10/bug-5143-ms-shuffle.exp ++++ b/pengine/test10/bug-5143-ms-shuffle.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +@@ -84,7 +84,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5186-partial-migrate.exp b/pengine/test10/bug-5186-partial-migrate.exp +index eddabe1..bba083e 100644 +--- a/pengine/test10/bug-5186-partial-migrate.exp ++++ b/pengine/test10/bug-5186-partial-migrate.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -75,7 +75,7 @@ + + + +- ++ + + + +@@ -140,7 +140,7 @@ + + + +- ++ + + + +@@ -153,7 +153,7 @@ + + + +- ++ + + + +@@ -184,7 +184,7 @@ + + + +- ++ + + + +@@ -197,7 +197,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-cl-5168.exp b/pengine/test10/bug-cl-5168.exp +index 82089f9..0b6617e 100644 +--- a/pengine/test10/bug-cl-5168.exp ++++ b/pengine/test10/bug-cl-5168.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +@@ -84,7 +84,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-cl-5170.exp b/pengine/test10/bug-cl-5170.exp +index c6b8720..d8143c0 100644 +--- a/pengine/test10/bug-cl-5170.exp ++++ b/pengine/test10/bug-cl-5170.exp +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-cl-5212.exp b/pengine/test10/bug-cl-5212.exp +index 080ba5a..1c35061 100644 +--- a/pengine/test10/bug-cl-5212.exp ++++ b/pengine/test10/bug-cl-5212.exp +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -56,7 +56,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-cl-5213.exp b/pengine/test10/bug-cl-5213.exp +index 154384d..790b12a 100644 +--- a/pengine/test10/bug-cl-5213.exp ++++ b/pengine/test10/bug-cl-5213.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-cl-5219.exp b/pengine/test10/bug-cl-5219.exp +index 182b279..608c849 100644 +--- a/pengine/test10/bug-cl-5219.exp ++++ b/pengine/test10/bug-cl-5219.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-cl-5247.exp b/pengine/test10/bug-cl-5247.exp +index 521c124..ae3ad11 100644 +--- a/pengine/test10/bug-cl-5247.exp ++++ b/pengine/test10/bug-cl-5247.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -66,7 +66,7 @@ + + + +- ++ + + + +@@ -79,7 +79,7 @@ + + + +- ++ + + + +@@ -98,7 +98,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +@@ -174,7 +174,7 @@ + + + +- ++ + + + +@@ -193,7 +193,7 @@ + + + +- ++ + + + +@@ -269,7 +269,7 @@ + + + +- ++ + + + +@@ -282,7 +282,7 @@ + + + +- ++ + + + +@@ -319,7 +319,7 @@ + + + +- ++ + + + +@@ -332,7 +332,7 @@ + + + +- ++ + + + +@@ -429,7 +429,7 @@ + + + +- ++ + + + +@@ -442,7 +442,7 @@ + + + +- ++ + + + +@@ -455,7 +455,7 @@ + + + +- ++ + + + +@@ -468,7 +468,7 @@ + + + +- ++ + + + +@@ -484,7 +484,7 @@ + + + +- ++ + + + +@@ -673,7 +673,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-1852.exp b/pengine/test10/bug-lf-1852.exp +index f5a2fef..2b79617 100644 +--- a/pengine/test10/bug-lf-1852.exp ++++ b/pengine/test10/bug-lf-1852.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-1920.exp b/pengine/test10/bug-lf-1920.exp +index da2d03c..27965ed 100644 +--- a/pengine/test10/bug-lf-1920.exp ++++ b/pengine/test10/bug-lf-1920.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2106.exp b/pengine/test10/bug-lf-2106.exp +index fc89c3f..f7d4900 100644 +--- a/pengine/test10/bug-lf-2106.exp ++++ b/pengine/test10/bug-lf-2106.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -54,7 +54,7 @@ + + + +- ++ + + + +@@ -67,7 +67,7 @@ + + + +- ++ + + + +@@ -83,7 +83,7 @@ + + + +- ++ + + + +@@ -96,7 +96,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2153.exp b/pengine/test10/bug-lf-2153.exp +index f222477..54f211f 100644 +--- a/pengine/test10/bug-lf-2153.exp ++++ b/pengine/test10/bug-lf-2153.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -138,7 +138,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2160.exp b/pengine/test10/bug-lf-2160.exp +index 61fd792..1a69082 100644 +--- a/pengine/test10/bug-lf-2160.exp ++++ b/pengine/test10/bug-lf-2160.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2171.exp b/pengine/test10/bug-lf-2171.exp +index c6c3379..8679088 100644 +--- a/pengine/test10/bug-lf-2171.exp ++++ b/pengine/test10/bug-lf-2171.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -85,7 +85,7 @@ + + + +- ++ + + + +@@ -98,7 +98,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2213.exp b/pengine/test10/bug-lf-2213.exp +index c2f7052..5a6a311 100644 +--- a/pengine/test10/bug-lf-2213.exp ++++ b/pengine/test10/bug-lf-2213.exp +@@ -30,7 +30,7 @@ + + + +- ++ + + + +@@ -70,7 +70,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2317.exp b/pengine/test10/bug-lf-2317.exp +index 3059408..ecf808d 100644 +--- a/pengine/test10/bug-lf-2317.exp ++++ b/pengine/test10/bug-lf-2317.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -71,7 +71,7 @@ + + + +- ++ + + + +@@ -87,7 +87,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2358.exp b/pengine/test10/bug-lf-2358.exp +index 2abb622..437ffec 100644 +--- a/pengine/test10/bug-lf-2358.exp ++++ b/pengine/test10/bug-lf-2358.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2361.exp b/pengine/test10/bug-lf-2361.exp +index ccae9c2..dd909bd 100644 +--- a/pengine/test10/bug-lf-2361.exp ++++ b/pengine/test10/bug-lf-2361.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -138,7 +138,7 @@ + + + +- ++ + + + +@@ -147,7 +147,7 @@ + + + +- ++ + + + +@@ -156,7 +156,7 @@ + + + +- ++ + + + +@@ -165,7 +165,7 @@ + + + +- ++ + + + +@@ -174,7 +174,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2422.exp b/pengine/test10/bug-lf-2422.exp +index 724cb23..14eba30 100644 +--- a/pengine/test10/bug-lf-2422.exp ++++ b/pengine/test10/bug-lf-2422.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +@@ -107,7 +107,7 @@ + + + +- ++ + + + +@@ -123,7 +123,7 @@ + + + +- ++ + + + +@@ -169,7 +169,7 @@ + + + +- ++ + + + +@@ -185,7 +185,7 @@ + + + +- ++ + + + +@@ -231,7 +231,7 @@ + + + +- ++ + + + +@@ -247,7 +247,7 @@ + + + +- ++ + + + +@@ -296,7 +296,7 @@ + + + +- ++ + + + +@@ -312,7 +312,7 @@ + + + +- ++ + + + +@@ -328,7 +328,7 @@ + + + +- ++ + + + +@@ -344,7 +344,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2435.exp b/pengine/test10/bug-lf-2435.exp +index 22fda7a..4349312 100644 +--- a/pengine/test10/bug-lf-2435.exp ++++ b/pengine/test10/bug-lf-2435.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -46,7 +46,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2445.exp b/pengine/test10/bug-lf-2445.exp +index 5f89edb..953b32d 100644 +--- a/pengine/test10/bug-lf-2445.exp ++++ b/pengine/test10/bug-lf-2445.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2453.exp b/pengine/test10/bug-lf-2453.exp +index 2aabeab..c4dd2b2 100644 +--- a/pengine/test10/bug-lf-2453.exp ++++ b/pengine/test10/bug-lf-2453.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2474.exp b/pengine/test10/bug-lf-2474.exp +index 78287be..8eca0af 100644 +--- a/pengine/test10/bug-lf-2474.exp ++++ b/pengine/test10/bug-lf-2474.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2493.exp b/pengine/test10/bug-lf-2493.exp +index b47cda4..19c162a 100644 +--- a/pengine/test10/bug-lf-2493.exp ++++ b/pengine/test10/bug-lf-2493.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -12,7 +12,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -46,7 +46,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -91,7 +91,7 @@ + + + +- ++ + + + +@@ -100,7 +100,7 @@ + + + +- ++ + + + +@@ -109,7 +109,7 @@ + + + +- ++ + + + +@@ -118,7 +118,7 @@ + + + +- ++ + + + +@@ -127,7 +127,7 @@ + + + +- ++ + + + +@@ -136,7 +136,7 @@ + + + +- ++ + + + +@@ -145,7 +145,7 @@ + + + +- ++ + + + +@@ -154,7 +154,7 @@ + + + +- ++ + + + +@@ -163,7 +163,7 @@ + + + +- ++ + + + +@@ -172,7 +172,7 @@ + + + +- ++ + + + +@@ -181,7 +181,7 @@ + + + +- ++ + + + +@@ -190,7 +190,7 @@ + + + +- ++ + + + +@@ -199,7 +199,7 @@ + + + +- ++ + + + +@@ -208,7 +208,7 @@ + + + +- ++ + + + +@@ -217,7 +217,7 @@ + + + +- ++ + + + +@@ -226,7 +226,7 @@ + + + +- ++ + + + +@@ -235,7 +235,7 @@ + + + +- ++ + + + +@@ -244,7 +244,7 @@ + + + +- ++ + + + +@@ -253,7 +253,7 @@ + + + +- ++ + + + +@@ -262,7 +262,7 @@ + + + +- ++ + + + +@@ -271,7 +271,7 @@ + + + +- ++ + + + +@@ -280,7 +280,7 @@ + + + +- ++ + + + +@@ -289,7 +289,7 @@ + + + +- ++ + + + +@@ -298,7 +298,7 @@ + + + +- ++ + + + +@@ -307,7 +307,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2508.exp b/pengine/test10/bug-lf-2508.exp +index 363154d..bf70378 100644 +--- a/pengine/test10/bug-lf-2508.exp ++++ b/pengine/test10/bug-lf-2508.exp +@@ -26,7 +26,7 @@ + + + +- ++ + + + +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -109,7 +109,7 @@ + + + +- ++ + + + +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -336,7 +336,7 @@ + + + +- ++ + + + +@@ -345,7 +345,7 @@ + + + +- ++ + + + +@@ -358,7 +358,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2544.exp b/pengine/test10/bug-lf-2544.exp +index 1755130..64cd62a 100644 +--- a/pengine/test10/bug-lf-2544.exp ++++ b/pengine/test10/bug-lf-2544.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2551.exp b/pengine/test10/bug-lf-2551.exp +index 44bdfca..f5b8236 100644 +--- a/pengine/test10/bug-lf-2551.exp ++++ b/pengine/test10/bug-lf-2551.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -24,7 +24,7 @@ + + + +- ++ + + + +@@ -33,7 +33,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +@@ -55,7 +55,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2574.exp b/pengine/test10/bug-lf-2574.exp +index 5d0a58a..2b35f48 100644 +--- a/pengine/test10/bug-lf-2574.exp ++++ b/pengine/test10/bug-lf-2574.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -38,7 +38,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2581.exp b/pengine/test10/bug-lf-2581.exp +index c04e478..421e75a 100644 +--- a/pengine/test10/bug-lf-2581.exp ++++ b/pengine/test10/bug-lf-2581.exp +@@ -33,7 +33,7 @@ + + + +- ++ + + + +@@ -46,7 +46,7 @@ + + + +- ++ + + + +@@ -59,7 +59,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -97,7 +97,7 @@ + + + +- ++ + + + +@@ -136,7 +136,7 @@ + + + +- ++ + + + +@@ -145,7 +145,7 @@ + + + +- ++ + + + +@@ -154,7 +154,7 @@ + + + +- ++ + + + +@@ -163,7 +163,7 @@ + + + +- ++ + + + +@@ -172,7 +172,7 @@ + + + +- ++ + + + +@@ -185,7 +185,7 @@ + + + +- ++ + + + +@@ -198,7 +198,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2606.exp b/pengine/test10/bug-lf-2606.exp +index 467e233..55f138b 100644 +--- a/pengine/test10/bug-lf-2606.exp ++++ b/pengine/test10/bug-lf-2606.exp +@@ -15,7 +15,7 @@ + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2613.exp b/pengine/test10/bug-lf-2613.exp +index e4dd227..47d7fc8 100644 +--- a/pengine/test10/bug-lf-2613.exp ++++ b/pengine/test10/bug-lf-2613.exp +@@ -83,7 +83,7 @@ + + + +- ++ + + + +@@ -96,7 +96,7 @@ + + + +- ++ + + + +@@ -115,7 +115,7 @@ + + + +- ++ + + + +@@ -131,7 +131,7 @@ + + + +- ++ + + + +@@ -144,7 +144,7 @@ + + + +- ++ + + + +@@ -163,7 +163,7 @@ + + + +- ++ + + + +@@ -179,7 +179,7 @@ + + + +- ++ + + + +@@ -192,7 +192,7 @@ + + + +- ++ + + + +@@ -211,7 +211,7 @@ + + + +- ++ + + + +@@ -227,7 +227,7 @@ + + + +- ++ + + + +@@ -240,7 +240,7 @@ + + + +- ++ + + + +@@ -259,7 +259,7 @@ + + + +- ++ + + + +@@ -275,7 +275,7 @@ + + + +- ++ + + + +@@ -288,7 +288,7 @@ + + + +- ++ + + + +@@ -307,7 +307,7 @@ + + + +- ++ + + + +@@ -323,7 +323,7 @@ + + + +- ++ + + + +@@ -336,7 +336,7 @@ + + + +- ++ + + + +@@ -355,7 +355,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2619.exp b/pengine/test10/bug-lf-2619.exp +index 4240db1..57aff38 100644 +--- a/pengine/test10/bug-lf-2619.exp ++++ b/pengine/test10/bug-lf-2619.exp +@@ -83,7 +83,7 @@ + + + +- ++ + + + +@@ -96,7 +96,7 @@ + + + +- ++ + + + +@@ -115,7 +115,7 @@ + + + +- ++ + + + +@@ -131,7 +131,7 @@ + + + +- ++ + + + +@@ -144,7 +144,7 @@ + + + +- ++ + + + +@@ -163,7 +163,7 @@ + + + +- ++ + + + +@@ -179,7 +179,7 @@ + + + +- ++ + + + +@@ -192,7 +192,7 @@ + + + +- ++ + + + +@@ -211,7 +211,7 @@ + + + +- ++ + + + +@@ -227,7 +227,7 @@ + + + +- ++ + + + +@@ -240,7 +240,7 @@ + + + +- ++ + + + +@@ -259,7 +259,7 @@ + + + +- ++ + + + +@@ -275,7 +275,7 @@ + + + +- ++ + + + +@@ -288,7 +288,7 @@ + + + +- ++ + + + +@@ -307,7 +307,7 @@ + + + +- ++ + + + +@@ -323,7 +323,7 @@ + + + +- ++ + + + +@@ -336,7 +336,7 @@ + + + +- ++ + + + +@@ -355,7 +355,7 @@ + + + +- ++ + + + +@@ -368,7 +368,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-n-385265-2.exp b/pengine/test10/bug-n-385265-2.exp +index 4c38b55..2f536dc 100644 +--- a/pengine/test10/bug-n-385265-2.exp ++++ b/pengine/test10/bug-n-385265-2.exp +@@ -59,7 +59,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +@@ -88,7 +88,7 @@ + + + +- ++ + + + +@@ -104,7 +104,7 @@ + + + +- ++ + + + +@@ -117,7 +117,7 @@ + + + +- ++ + + + +@@ -136,7 +136,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-n-385265.exp b/pengine/test10/bug-n-385265.exp +index 82624fa..c8bad25 100644 +--- a/pengine/test10/bug-n-385265.exp ++++ b/pengine/test10/bug-n-385265.exp +@@ -26,7 +26,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-n-387749.exp b/pengine/test10/bug-n-387749.exp +index 3ca8c2a..bba3012 100644 +--- a/pengine/test10/bug-n-387749.exp ++++ b/pengine/test10/bug-n-387749.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,7 +29,7 @@ + + + +- ++ + + + +@@ -38,7 +38,7 @@ + + + +- ++ + + + +@@ -51,7 +51,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -233,7 +233,7 @@ + + + +- ++ + + + +@@ -246,7 +246,7 @@ + + + +- ++ + + + +@@ -265,7 +265,7 @@ + + + +- ++ + + + +@@ -284,7 +284,7 @@ + + + +- ++ + + + +@@ -293,7 +293,7 @@ + + + +- ++ + + + +@@ -306,7 +306,7 @@ + + + +- ++ + + + +@@ -328,7 +328,7 @@ + + + +- ++ + + + +@@ -344,7 +344,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-pm-11.exp b/pengine/test10/bug-pm-11.exp +index 4576a78..505e51e 100644 +--- a/pengine/test10/bug-pm-11.exp ++++ b/pengine/test10/bug-pm-11.exp +@@ -30,7 +30,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +@@ -128,7 +128,7 @@ + + + +- ++ + + + +@@ -141,7 +141,7 @@ + + + +- ++ + + + +@@ -150,7 +150,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-pm-12.exp b/pengine/test10/bug-pm-12.exp +index bea17dc..ced0661 100644 +--- a/pengine/test10/bug-pm-12.exp ++++ b/pengine/test10/bug-pm-12.exp +@@ -60,7 +60,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -197,7 +197,7 @@ + + + +- ++ + + + +@@ -213,7 +213,7 @@ + + + +- ++ + + + +@@ -229,7 +229,7 @@ + + + +- ++ + + + +@@ -242,7 +242,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-rh-1097457.exp b/pengine/test10/bug-rh-1097457.exp +index 0c3430c..16b6b77 100644 +--- a/pengine/test10/bug-rh-1097457.exp ++++ b/pengine/test10/bug-rh-1097457.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -58,7 +58,7 @@ + + + +- ++ + + + +@@ -92,7 +92,7 @@ + + + +- ++ + + + +@@ -168,7 +168,7 @@ + + + +- ++ + + + +@@ -208,7 +208,7 @@ + + + +- ++ + + + +@@ -224,7 +224,7 @@ + + + +- ++ + + + +@@ -264,7 +264,7 @@ + + + +- ++ + + + +@@ -280,7 +280,7 @@ + + + +- ++ + + + +@@ -317,7 +317,7 @@ + + + +- ++ + + + +@@ -387,7 +387,7 @@ + + + +- ++ + + + +@@ -406,7 +406,7 @@ + + + +- ++ + + + +@@ -418,7 +418,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-rh-880249.exp b/pengine/test10/bug-rh-880249.exp +index 97749e6..c66bd8c 100644 +--- a/pengine/test10/bug-rh-880249.exp ++++ b/pengine/test10/bug-rh-880249.exp +@@ -3,7 +3,7 @@ + + + +- ++ + + + +@@ -22,7 +22,7 @@ + + + +- ++ + + + +@@ -41,7 +41,7 @@ + + + +- ++ + + + +@@ -60,7 +60,7 @@ + + + +- ++ + + + +@@ -79,7 +79,7 @@ + +