diff --git a/SOURCES/bz1078078-pcmk-crm_report_suppress_logging_errors_after_the_target_directory_has_been_compressed.patch b/SOURCES/bz1078078-pcmk-crm_report_suppress_logging_errors_after_the_target_directory_has_been_compressed.patch new file mode 100644 index 0000000..4b96e99 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-crm_report_suppress_logging_errors_after_the_target_directory_has_been_compressed.patch @@ -0,0 +1,21 @@ +commit 3de41452c059cb44028433bea1ab6425e239a38c +Author: Andrew Beekhof +Date: Wed Dec 18 08:12:29 2013 +1100 + + Fix: crm_report: Suppress logging errors after the target directory has been compressed + + (cherry picked from commit 571c7b0e66b174442604883fe105dbe8d10f1bc4) + +diff --git a/tools/report.common.in b/tools/report.common.in +index 9742f1c..bba7c63 100644 +--- a/tools/report.common.in ++++ b/tools/report.common.in +@@ -76,7 +76,7 @@ record() { + if [ x != x"$REPORT_HOME" -a -d "${REPORT_HOME}/$shorthost" ]; then + rec="${REPORT_HOME}/$shorthost/report.out" + +- elif [ x != x"${l_base}" ]; then ++ elif [ x != x"${l_base}" -a -d "${l_base}" ]; then + rec="${l_base}/report.summary" + + else diff --git a/SOURCES/bz1078078-pcmk-crm_resource_wait_for_the_correct_number_of_replies_when_cleaning_up_resources.patch b/SOURCES/bz1078078-pcmk-crm_resource_wait_for_the_correct_number_of_replies_when_cleaning_up_resources.patch new file mode 100644 index 0000000..9b7a4e5 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-crm_resource_wait_for_the_correct_number_of_replies_when_cleaning_up_resources.patch @@ -0,0 +1,29 @@ +commit ec35b157495c79cf79572fdca4b369f37d42e2bf +Author: Andrew Beekhof +Date: Thu Oct 31 14:04:21 2013 +1100 + + Fix: crm_resource: Wait for the correct number of replies when cleaning up resources + + (cherry picked from commit 145c782e432d8108ca865f994640cf5a62406363) + +diff --git a/tools/crm_resource.c b/tools/crm_resource.c +index 1fcb2a2..c344997 100644 +--- a/tools/crm_resource.c ++++ b/tools/crm_resource.c +@@ -61,7 +61,7 @@ char *our_pid = NULL; + crm_ipc_t *crmd_channel = NULL; + char *xml_file = NULL; + int cib_options = cib_sync_call; +-int crmd_replies_needed = 0; ++int crmd_replies_needed = 1; /* The welcome message */ + GMainLoop *mainloop = NULL; + + extern void cleanup_alloc_calculations(pe_working_set_t * data_set); +@@ -93,7 +93,6 @@ static void + start_mainloop(void) + { + mainloop = g_main_new(FALSE); +- crmd_replies_needed++; /* The welcome message */ + fprintf(stderr, "Waiting for %d replies from the CRMd", crmd_replies_needed); + crm_debug("Waiting for %d replies from the CRMd", crmd_replies_needed); + diff --git a/SOURCES/bz1078078-pcmk-crmd_correctly_react_to_successful_unfencing_operations.patch b/SOURCES/bz1078078-pcmk-crmd_correctly_react_to_successful_unfencing_operations.patch new file mode 100644 index 0000000..e3c79b3 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-crmd_correctly_react_to_successful_unfencing_operations.patch @@ -0,0 +1,27 @@ +commit 110cdd2056407a3a448a0e0cf3ef4e126fb26753 +Author: Andrew Beekhof +Date: Fri Apr 4 12:13:04 2014 +1100 + + Fix: crmd: Correctly react to successful unfencing operations + + Hint: We don't want to shut down + (cherry picked from commit 69c8d227c605d0b85038a536be948f5a56cfe4bd) + +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index 239af63..60dd21c 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -123,7 +123,12 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) + return; + } + +- if (st_event->result == pcmk_ok && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) { ++ if (st_event->result == pcmk_ok && safe_str_eq("on", st_event->action)) { ++ crm_notice("%s was successfully unfenced by %s for %s with %s", ++ st_event->executioner, st_event->origin, st_event->device); ++ return; ++ ++ } else if (st_event->result == pcmk_ok && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) { + crm_crit("We were alegedly just fenced by %s for %s with %s!", st_event->executioner, + st_event->origin, st_event->device); /* Dumps blackbox if enabled */ + diff --git a/SOURCES/bz1078078-pcmk-crmd_device_isn_t_hooked_up_yet_don_t_bother_logging_it.patch b/SOURCES/bz1078078-pcmk-crmd_device_isn_t_hooked_up_yet_don_t_bother_logging_it.patch new file mode 100644 index 0000000..3ddd9d2 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-crmd_device_isn_t_hooked_up_yet_don_t_bother_logging_it.patch @@ -0,0 +1,40 @@ +commit 4cf4c0dee7128ae0896bf5fc6cbaeffd21cfb984 +Author: Andrew Beekhof +Date: Thu Apr 10 13:20:20 2014 +1000 + + Log: crmd: device isn't hooked up yet, don't bother logging it + + (cherry picked from commit 97705f05e96dcc0f8b232f32b842ee36844a6567) + +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index b2003ba..c7260fd 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -124,20 +124,20 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) + } + + if (st_event->result == pcmk_ok && safe_str_eq("on", st_event->action)) { +- crm_notice("%s was successfully unfenced by %s (at the request of %s) with %s", +- st_event->target, st_event->executioner, st_event->origin, +- st_event->device); /* TODO: Why is device never set? */ ++ crm_notice("%s was successfully unfenced by %s (at the request of %s)", ++ st_event->target, st_event->executioner, st_event->origin); ++ /* TODO: Hook up st_event->device */ + return; + + } else if (safe_str_eq("on", st_event->action)) { +- crm_err("Unfencing of %s by %s with %s failed: %s (%d)", +- st_event->target, st_event->executioner, st_event->device, ++ crm_err("Unfencing of %s by %s failed: %s (%d)", ++ st_event->target, st_event->executioner, + pcmk_strerror(st_event->result), st_event->result); + return; + + } else if (st_event->result == pcmk_ok && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) { +- crm_crit("We were alegedly just fenced by %s for %s with %s!", st_event->executioner, +- st_event->origin, st_event->device); /* Dumps blackbox if enabled */ ++ crm_crit("We were alegedly just fenced by %s for %s!", ++ st_event->executioner, st_event->origin); /* Dumps blackbox if enabled */ + + qb_log_fini(); /* Try to get the above log message to disk - somehow */ + diff --git a/SOURCES/bz1078078-pcmk-crmd_do_not_erase_the_status_section_for_unfenced_nodes.patch b/SOURCES/bz1078078-pcmk-crmd_do_not_erase_the_status_section_for_unfenced_nodes.patch new file mode 100644 index 0000000..6b23f8b --- /dev/null +++ b/SOURCES/bz1078078-pcmk-crmd_do_not_erase_the_status_section_for_unfenced_nodes.patch @@ -0,0 +1,29 @@ +commit de876a58b6cd80389f7f345784a15080056faf9d +Author: Andrew Beekhof +Date: Thu Apr 10 08:59:04 2014 +1000 + + Fix: crmd: Do not erase the status section for unfenced nodes + + (cherry picked from commit ac788bf0eed36eeb489e7c6e9bcf5c5ded91fd44) + + Conflicts: + crmd/te_callbacks.c + +diff --git a/crmd/te_callbacks.c b/crmd/te_callbacks.c +index 4b8aac5..ce8ae9c 100644 +--- a/crmd/te_callbacks.c ++++ b/crmd/te_callbacks.c +@@ -442,11 +442,12 @@ tengine_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data) + if (rc == pcmk_ok) { + const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET); + const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID); ++ const char *op = crm_meta_value(action->params, "stonith_action"); + + crm_debug("Stonith operation %d for %s passed", call_id, target); + if (action->confirmed == FALSE) { + action->confirmed = TRUE; +- if (action->sent_update == FALSE) { ++ if (action->sent_update == FALSE && safe_str_neq("on", op)) { + send_stonith_update(action, target, uuid); + } + } diff --git a/SOURCES/bz1078078-pcmk-crmd_do_not_overwrite_existing_node_state_when_fencing_completes.patch b/SOURCES/bz1078078-pcmk-crmd_do_not_overwrite_existing_node_state_when_fencing_completes.patch new file mode 100644 index 0000000..7129505 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-crmd_do_not_overwrite_existing_node_state_when_fencing_completes.patch @@ -0,0 +1,155 @@ +commit e0bcac68df1f684c1879ddb76387c47b177ece6f +Author: Andrew Beekhof +Date: Tue Feb 18 10:37:07 2014 +1100 + + Fix: crmd: Do not overwrite existing node state when fencing completes + + Let the normal membership callbacks do that. This allows us to function + correctly when a node returns before we get the fencing notification. + + (cherry picked from commit e777b1759786d0a917299399e8a1fee1242a1eef) + + Conflicts: + crmd/callbacks.c + crmd/te_utils.c + crmd/utils.c + +diff --git a/crmd/crmd_utils.h b/crmd/crmd_utils.h +index 92d14b1..ca1e177 100644 +--- a/crmd/crmd_utils.h ++++ b/crmd/crmd_utils.h +@@ -99,6 +99,7 @@ void crmd_join_phase_log(int level); + const char *get_timer_desc(fsa_timer_t * timer); + gboolean too_many_st_failures(void); + void st_fail_count_reset(const char * target); ++void crmd_peer_down(crm_node_t *peer, bool full); + + # define fsa_register_cib_callback(id, flag, data, fn) do { \ + fsa_cib_conn->cmds->register_callback( \ +diff --git a/crmd/te_actions.c b/crmd/te_actions.c +index 4e3bc22..ee66af6 100644 +--- a/crmd/te_actions.c ++++ b/crmd/te_actions.c +@@ -75,11 +75,8 @@ send_stonith_update(crm_action_t * action, const char *target, const char *uuid) + crm_info("Recording uuid '%s' for node '%s'", uuid, target); + peer->uuid = strdup(uuid); + } +- crm_update_peer_proc(__FUNCTION__, peer, crm_proc_none, NULL); +- crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_LOST, 0); +- crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN); +- crm_update_peer_join(__FUNCTION__, peer, crm_join_none); + ++ crmd_peer_down(peer, TRUE); + node_state = + do_update_node_cib(peer, + node_update_cluster | node_update_peer | node_update_join | +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index c7260fd..9edb128 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -234,11 +234,7 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) + + } + +- /* Everyone records them as safely down */ +- crm_update_peer_proc(__FUNCTION__, peer, crm_proc_none, NULL); +- crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_LOST, 0); +- crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN); +- crm_update_peer_join(__FUNCTION__, peer, crm_join_none); ++ crmd_peer_down(peer, TRUE); + } + } + +diff --git a/crmd/utils.c b/crmd/utils.c +index 07c71b2..a8516c4 100644 +--- a/crmd/utils.c ++++ b/crmd/utils.c +@@ -1091,3 +1091,13 @@ update_attrd(const char *host, const char *name, const char *value, const char * + } + } + } ++ ++void crmd_peer_down(crm_node_t *peer, bool full) ++{ ++ if(full && peer->state == NULL) { ++ crm_update_peer_state(__FUNCTION__, peer, CRM_NODE_LOST, 0); ++ crm_update_peer_proc(__FUNCTION__, peer, crm_proc_none, NULL); ++ } ++ crm_update_peer_join(__FUNCTION__, peer, crm_join_none); ++ crm_update_peer_expected(__FUNCTION__, peer, CRMD_JOINSTATE_DOWN); ++} +diff --git a/pengine/test10/whitebox-orphan-ms.pe.dot b/pengine/test10/whitebox-orphan-ms.pe.dot +deleted file mode 100644 +index 81c37eb..0000000 +--- a/pengine/test10/whitebox-orphan-ms.pe.dot ++++ /dev/null +@@ -1,69 +0,0 @@ +-digraph "g" { +-"FencingFail_start_0 18node1" [ style=bold color="green" fontcolor="black"] +-"FencingFail_stop_0 18node3" -> "FencingFail_start_0 18node1" [ style = bold] +-"FencingFail_stop_0 18node3" -> "all_stopped" [ style = bold] +-"FencingFail_stop_0 18node3" [ style=bold color="green" fontcolor="black"] +-"all_stopped" [ style=bold color="green" fontcolor="orange"] +-"container1_delete_0 18node1" -> "container1_start_0 " [ style = dashed] +-"container1_delete_0 18node1" [ style=bold color="green" fontcolor="black"] +-"container1_delete_0 18node2" -> "container1_start_0 " [ style = dashed] +-"container1_delete_0 18node2" [ style=bold color="green" fontcolor="black"] +-"container1_delete_0 18node3" -> "container1_start_0 " [ style = dashed] +-"container1_delete_0 18node3" [ style=bold color="green" fontcolor="black"] +-"container1_start_0 " [ style=dashed color="red" fontcolor="black"] +-"container1_stop_0 18node1" -> "all_stopped" [ style = bold] +-"container1_stop_0 18node1" -> "container1_delete_0 18node1" [ style = bold] +-"container1_stop_0 18node1" -> "container1_delete_0 18node2" [ style = bold] +-"container1_stop_0 18node1" -> "container1_delete_0 18node3" [ style = bold] +-"container1_stop_0 18node1" -> "container1_start_0 " [ style = dashed] +-"container1_stop_0 18node1" [ style=bold color="green" fontcolor="black"] +-"container2_delete_0 18node1" [ style=bold color="green" fontcolor="black"] +-"container2_delete_0 18node2" [ style=bold color="green" fontcolor="black"] +-"container2_delete_0 18node3" [ style=bold color="green" fontcolor="black"] +-"container2_stop_0 18node1" -> "all_stopped" [ style = bold] +-"container2_stop_0 18node1" -> "container2_delete_0 18node1" [ style = bold] +-"container2_stop_0 18node1" -> "container2_delete_0 18node2" [ style = bold] +-"container2_stop_0 18node1" -> "container2_delete_0 18node3" [ style = bold] +-"container2_stop_0 18node1" [ style=bold color="green" fontcolor="black"] +-"lxc-ms_delete_0 18node1" [ style=bold color="green" fontcolor="black"] +-"lxc-ms_delete_0 18node2" [ style=bold color="green" fontcolor="black"] +-"lxc-ms_delete_0 18node3" [ style=bold color="green" fontcolor="black"] +-"lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold] +-"lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc2" [ style = bold] +-"lxc-ms_demote_0 lxc1" -> "lxc1_stop_0 18node1" [ style = bold] +-"lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="black"] +-"lxc-ms_demote_0 lxc2" -> "lxc-ms_stop_0 lxc1" [ style = bold] +-"lxc-ms_demote_0 lxc2" -> "lxc-ms_stop_0 lxc2" [ style = bold] +-"lxc-ms_demote_0 lxc2" -> "lxc2_stop_0 18node1" [ style = bold] +-"lxc-ms_demote_0 lxc2" [ style=bold color="green" fontcolor="black"] +-"lxc-ms_stop_0 lxc1" -> "all_stopped" [ style = bold] +-"lxc-ms_stop_0 lxc1" -> "lxc-ms_delete_0 18node1" [ style = bold] +-"lxc-ms_stop_0 lxc1" -> "lxc-ms_delete_0 18node2" [ style = bold] +-"lxc-ms_stop_0 lxc1" -> "lxc-ms_delete_0 18node3" [ style = bold] +-"lxc-ms_stop_0 lxc1" -> "lxc1_stop_0 18node1" [ style = bold] +-"lxc-ms_stop_0 lxc1" [ style=bold color="green" fontcolor="black"] +-"lxc-ms_stop_0 lxc2" -> "all_stopped" [ style = bold] +-"lxc-ms_stop_0 lxc2" -> "lxc-ms_delete_0 18node1" [ style = bold] +-"lxc-ms_stop_0 lxc2" -> "lxc-ms_delete_0 18node2" [ style = bold] +-"lxc-ms_stop_0 lxc2" -> "lxc-ms_delete_0 18node3" [ style = bold] +-"lxc-ms_stop_0 lxc2" -> "lxc2_stop_0 18node1" [ style = bold] +-"lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="black"] +-"lxc1_delete_0 18node1" [ style=bold color="green" fontcolor="black"] +-"lxc1_delete_0 18node2" [ style=bold color="green" fontcolor="black"] +-"lxc1_delete_0 18node3" [ style=bold color="green" fontcolor="black"] +-"lxc1_stop_0 18node1" -> "all_stopped" [ style = bold] +-"lxc1_stop_0 18node1" -> "container1_stop_0 18node1" [ style = bold] +-"lxc1_stop_0 18node1" -> "lxc1_delete_0 18node1" [ style = bold] +-"lxc1_stop_0 18node1" -> "lxc1_delete_0 18node2" [ style = bold] +-"lxc1_stop_0 18node1" -> "lxc1_delete_0 18node3" [ style = bold] +-"lxc1_stop_0 18node1" [ style=bold color="green" fontcolor="black"] +-"lxc2_delete_0 18node1" [ style=bold color="green" fontcolor="black"] +-"lxc2_delete_0 18node2" [ style=bold color="green" fontcolor="black"] +-"lxc2_delete_0 18node3" [ style=bold color="green" fontcolor="black"] +-"lxc2_stop_0 18node1" -> "all_stopped" [ style = bold] +-"lxc2_stop_0 18node1" -> "container2_stop_0 18node1" [ style = bold] +-"lxc2_stop_0 18node1" -> "lxc2_delete_0 18node1" [ style = bold] +-"lxc2_stop_0 18node1" -> "lxc2_delete_0 18node2" [ style = bold] +-"lxc2_stop_0 18node1" -> "lxc2_delete_0 18node3" [ style = bold] +-"lxc2_stop_0 18node1" [ style=bold color="green" fontcolor="black"] +-} diff --git a/SOURCES/bz1078078-pcmk-crmd_report_unsuccessful_unfencing_operations.patch b/SOURCES/bz1078078-pcmk-crmd_report_unsuccessful_unfencing_operations.patch new file mode 100644 index 0000000..cddd885 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-crmd_report_unsuccessful_unfencing_operations.patch @@ -0,0 +1,30 @@ +commit 2a8e1620b6a224905d6cb4e9bc444c957a85f946 +Author: Andrew Beekhof +Date: Wed Apr 9 12:17:25 2014 +1000 + + Fix: crmd: Report unsuccessful unfencing operations + + (cherry picked from commit 32f6349100e85b6a48dac01a80c967bc0134d165) + +diff --git a/crmd/te_utils.c b/crmd/te_utils.c +index 60dd21c..b2003ba 100644 +--- a/crmd/te_utils.c ++++ b/crmd/te_utils.c +@@ -124,8 +124,15 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event) + } + + if (st_event->result == pcmk_ok && safe_str_eq("on", st_event->action)) { +- crm_notice("%s was successfully unfenced by %s for %s with %s", +- st_event->executioner, st_event->origin, st_event->device); ++ crm_notice("%s was successfully unfenced by %s (at the request of %s) with %s", ++ st_event->target, st_event->executioner, st_event->origin, ++ st_event->device); /* TODO: Why is device never set? */ ++ return; ++ ++ } else if (safe_str_eq("on", st_event->action)) { ++ crm_err("Unfencing of %s by %s with %s failed: %s (%d)", ++ st_event->target, st_event->executioner, st_event->device, ++ pcmk_strerror(st_event->result), st_event->result); + return; + + } else if (st_event->result == pcmk_ok && crm_str_eq(st_event->target, fsa_our_uname, TRUE)) { diff --git a/SOURCES/bz1078078-pcmk-default_to_off_when_agent_does_not_advertise_reboot_in_metadata.patch b/SOURCES/bz1078078-pcmk-default_to_off_when_agent_does_not_advertise_reboot_in_metadata.patch new file mode 100644 index 0000000..82fbf8d --- /dev/null +++ b/SOURCES/bz1078078-pcmk-default_to_off_when_agent_does_not_advertise_reboot_in_metadata.patch @@ -0,0 +1,57 @@ +commit ad224d149d03351fd5ff63011697b117ad661aaf +Author: David Vossel +Date: Thu Apr 17 11:37:50 2014 -0500 + + High: fencing: default to 'off' when agent does not advertise 'reboot' in metadata + + (cherry picked from commit 8383a38a478ed6473ff2179596335ed4de583cfa) + +diff --git a/fencing/commands.c b/fencing/commands.c +index b4cd862..fbbee1a 100644 +--- a/fencing/commands.c ++++ b/fencing/commands.c +@@ -206,6 +206,7 @@ static gboolean + stonith_device_execute(stonith_device_t * device) + { + int exec_rc = 0; ++ const char *action_str = NULL; + async_command_t *cmd = NULL; + stonith_action_t *action = NULL; + +@@ -246,8 +247,14 @@ stonith_device_execute(stonith_device_t * device) + } + #endif + ++ action_str = cmd->action; ++ if (safe_str_eq(cmd->action, "reboot") && is_not_set(device->flags, st_device_supports_reboot)) { ++ crm_warn("Agent '%s' does not advertise support for 'reboot', performing 'off' action instead", device->agent); ++ action_str = "off"; ++ } ++ + action = stonith_action_create(device->agent, +- cmd->action, ++ action_str, + cmd->victim, + cmd->victim_nodeid, + cmd->timeout, device->params, device->aliases); +@@ -592,6 +599,8 @@ read_action_metadata(stonith_device_t *device) + set_bit(device->flags, st_device_supports_list); + } else if(safe_str_eq(action, "status")) { + set_bit(device->flags, st_device_supports_status); ++ } else if(safe_str_eq(action, "reboot")) { ++ set_bit(device->flags, st_device_supports_reboot); + } else if(safe_str_eq(action, "on") && (crm_is_true(automatic))) { + /* this setting implies required=true for unfencing */ + required = "true"; +diff --git a/fencing/internal.h b/fencing/internal.h +index 3fcad20..d94e2d6 100644 +--- a/fencing/internal.h ++++ b/fencing/internal.h +@@ -16,6 +16,7 @@ enum st_device_flags + { + st_device_supports_list = 0x0001, + st_device_supports_status = 0x0002, ++ st_device_supports_reboot = 0x0004, + }; + + typedef struct stonith_device_s { diff --git a/SOURCES/bz1078078-pcmk-execute_all_required_fencing_devices_regardless_of_what_topology_level_they_are_at.patch b/SOURCES/bz1078078-pcmk-execute_all_required_fencing_devices_regardless_of_what_topology_level_they_are_at.patch new file mode 100644 index 0000000..6dfeeeb --- /dev/null +++ b/SOURCES/bz1078078-pcmk-execute_all_required_fencing_devices_regardless_of_what_topology_level_they_are_at.patch @@ -0,0 +1,180 @@ +commit 7aa42a14d8b06638f3e3e175fc59e16f730979a8 +Author: David Vossel +Date: Thu Apr 17 10:52:45 2014 -0500 + + High: fencing: Execute all required fencing devices regardless of what topology level they are at + + (cherry picked from commit c922fd50f7c8deb337aad932cb0b8d1c26cbcd99) + +diff --git a/fencing/commands.c b/fencing/commands.c +index 8efb156..b4cd862 100644 +--- a/fencing/commands.c ++++ b/fencing/commands.c +@@ -1297,6 +1297,10 @@ stonith_query_capable_device_cb(GList * devices, void *user_data) + crm_xml_add(dev, "namespace", device->namespace); + crm_xml_add(dev, "agent", device->agent); + crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified); ++ if (is_action_required(query->action, device)) { ++ crm_xml_add_int(dev, F_STONITH_DEVICE_REQUIRED, 1); ++ } ++ crm_xml_add_int(dev, F_STONITH_DEVICE_VERIFIED, device->verified); + if (action_specific_timeout) { + crm_xml_add_int(dev, F_STONITH_ACTION_TIMEOUT, action_specific_timeout); + } +diff --git a/fencing/internal.h b/fencing/internal.h +index 971a32f..3fcad20 100644 +--- a/fencing/internal.h ++++ b/fencing/internal.h +@@ -113,6 +113,10 @@ typedef struct remote_fencing_op_s { + + /*! The current topology level being executed */ + guint level; ++ ++ /*! List of required devices the topology must execute regardless of what ++ * topology level they exist at. */ ++ GListPtr required_list; + /*! The device list of all the devices at the current executing topology level. */ + GListPtr devices_list; + /*! Current entry in the topology device list */ +diff --git a/fencing/remote.c b/fencing/remote.c +index fd25025..10447e7 100644 +--- a/fencing/remote.c ++++ b/fencing/remote.c +@@ -68,6 +68,12 @@ static void report_timeout_period(remote_fencing_op_t * op, int op_timeout); + static int get_op_total_timeout(remote_fencing_op_t * op, st_query_result_t * chosen_peer, + int default_timeout); + ++static gint ++sort_strings(gconstpointer a, gconstpointer b) ++{ ++ return strcmp(a, b); ++} ++ + static void + free_remote_query(gpointer data) + { +@@ -128,6 +134,10 @@ free_remote_op(gpointer data) + g_list_free_full(op->devices_list, free); + op->devices_list = NULL; + } ++ if (op->required_list) { ++ g_list_free_full(op->required_list, free); ++ op->required_list = NULL; ++ } + free(op); + } + +@@ -400,6 +410,25 @@ topology_is_empty(stonith_topology_t *tp) + return TRUE; + } + ++static void ++add_required_device(remote_fencing_op_t * op, const char *device) ++{ ++ GListPtr match = g_list_find_custom(op->required_list, device, sort_strings); ++ if (match) { ++ /* device already marked required */ ++ return; ++ } ++ op->required_list = g_list_prepend(op->required_list, strdup(device)); ++ ++ /* make sure the required devices is in the current list of devices to be executed */ ++ if (op->devices_list) { ++ GListPtr match = g_list_find_custom(op->devices_list, device, sort_strings); ++ if (match == NULL) { ++ op->devices_list = g_list_append(op->devices_list, strdup(device)); ++ } ++ } ++} ++ + /* deep copy the device list */ + static void + set_op_device_list(remote_fencing_op_t * op, GListPtr devices) +@@ -413,6 +442,18 @@ set_op_device_list(remote_fencing_op_t * op, GListPtr devices) + for (lpc = devices; lpc != NULL; lpc = lpc->next) { + op->devices_list = g_list_append(op->devices_list, strdup(lpc->data)); + } ++ ++ /* tack on whatever required devices have not been executed ++ * to the end of the current devices list. This ensures that ++ * the required devices will get executed regardless of what topology ++ * level they exist at. */ ++ for (lpc = op->required_list; lpc != NULL; lpc = lpc->next) { ++ GListPtr match = g_list_find_custom(op->devices_list, lpc->data, sort_strings); ++ if (match == NULL) { ++ op->devices_list = g_list_append(op->devices_list, strdup(lpc->data)); ++ } ++ } ++ + op->devices = op->devices_list; + } + +@@ -714,12 +755,6 @@ initiate_remote_stonith_op(crm_client_t * client, xmlNode * request, gboolean ma + return op; + } + +-static gint +-sort_strings(gconstpointer a, gconstpointer b) +-{ +- return strcmp(a, b); +-} +- + enum find_best_peer_options { + /*! Skip checking the target peer for capable fencing devices */ + FIND_PEER_SKIP_TARGET = 0x0001, +@@ -1165,11 +1200,13 @@ process_remote_stonith_query(xmlNode * msg) + const char *device = ID(child); + int action_timeout = 0; + int verified = 0; ++ int required = 0; + + if (device) { + result->device_list = g_list_prepend(result->device_list, strdup(device)); + crm_element_value_int(child, F_STONITH_ACTION_TIMEOUT, &action_timeout); + crm_element_value_int(child, F_STONITH_DEVICE_VERIFIED, &verified); ++ crm_element_value_int(child, F_STONITH_DEVICE_REQUIRED, &required); + if (action_timeout) { + crm_trace("Peer %s with device %s returned action timeout %d", + result->host, device, action_timeout); +@@ -1181,6 +1218,13 @@ process_remote_stonith_query(xmlNode * msg) + g_hash_table_insert(result->verified_devices, + strdup(device), GINT_TO_POINTER(verified)); + } ++ if (required) { ++ crm_trace("Peer %s requires device %s to execute for action %s", ++ result->host, device, op->action); ++ /* This matters when executing a topology. Required devices will get ++ * executed regardless of their topology level. We use this for unfencing. */ ++ add_required_device(op, device); ++ } + } + } + +@@ -1312,10 +1356,14 @@ process_remote_stonith_exec(xmlNode * msg) + * Continue the topology if more devices exist at the current level, otherwise + * mark as done. */ + if (rc == pcmk_ok) { ++ GListPtr required_match = g_list_find_custom(op->required_list, device, sort_strings); + if (op->devices) { + /* Success, are there any more? */ + op->devices = op->devices->next; + } ++ if (required_match) { ++ op->required_list = g_list_remove(op->required_list, required_match->data); ++ } + /* if no more devices at this fencing level, we are done, + * else we need to contine with executing the next device in the list */ + if (op->devices == NULL) { +diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h +index 2822e9a..3625cf9 100644 +--- a/include/crm/fencing/internal.h ++++ b/include/crm/fencing/internal.h +@@ -66,6 +66,8 @@ xmlNode *create_device_registration_xml(const char *id, const char *namespace, c + /*! Has this device been verified using a monitor type + * operation (monitor, list, status) */ + # define F_STONITH_DEVICE_VERIFIED "st_monitor_verified" ++/*! device is required for this action */ ++# define F_STONITH_DEVICE_REQUIRED "st_required" + # define F_STONITH_CALLBACK_TOKEN "st_async_id" + # define F_STONITH_CLIENTNAME "st_clientname" + # define F_STONITH_CLIENTNODE "st_clientnode" diff --git a/SOURCES/bz1078078-pcmk-fence_using_all_required_devices.patch b/SOURCES/bz1078078-pcmk-fence_using_all_required_devices.patch new file mode 100644 index 0000000..6a30090 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-fence_using_all_required_devices.patch @@ -0,0 +1,289 @@ +commit 1928af7d8d2fc7d225df22caeb7ea93964f48bde +Author: David Vossel +Date: Wed Apr 16 10:28:46 2014 -0500 + + High: fencing: Fence using all required devices + + (cherry picked from commit 1f39dfe13d8cbfb5dd22f8324735319f81230fdb) + + Conflicts: + fencing/main.c + +diff --git a/fencing/commands.c b/fencing/commands.c +index 41690f7..8efb156 100644 +--- a/fencing/commands.c ++++ b/fencing/commands.c +@@ -111,6 +111,20 @@ typedef struct async_command_s { + static xmlNode *stonith_construct_async_reply(async_command_t * cmd, const char *output, + xmlNode * data, int rc); + ++static gboolean ++is_action_required(const char *action, stonith_device_t *device) ++{ ++ if (device->required_actions == NULL) { ++ return FALSE; ++ } ++ ++ if (strstr(device->required_actions, action)) { ++ return TRUE; ++ } ++ ++ return FALSE; ++} ++ + static int + get_action_timeout(stonith_device_t * device, const char *action, int default_timeout) + { +@@ -318,6 +332,7 @@ free_device(gpointer data) + free_xml(device->agent_metadata); + free(device->namespace); + free(device->on_target_actions); ++ free(device->required_actions); + free(device->agent); + free(device->id); + free(device); +@@ -525,10 +540,24 @@ is_nodeid_required(xmlNode * xml) + return TRUE; + } + ++static char * ++add_action(char *actions, const char *action) ++{ ++ static size_t len = 256; ++ if (actions == NULL) { ++ actions = calloc(1, len); ++ } ++ if (strlen(actions)) { ++ g_strlcat(actions, " ", len); ++ } ++ g_strlcat(actions, action, len); ++ ++ return actions; ++} ++ + static void +-get_on_target_actions(stonith_device_t *device) ++read_action_metadata(stonith_device_t *device) + { +- char *actions = NULL; + xmlXPathObjectPtr xpath = NULL; + int max = 0; + int lpc = 0; +@@ -545,40 +574,38 @@ get_on_target_actions(stonith_device_t *device) + return; + } + +- actions = calloc(1, 512); +- + for (lpc = 0; lpc < max; lpc++) { + const char *on_target = NULL; + const char *action = NULL; ++ const char *automatic = NULL; ++ const char *required = NULL; + xmlNode *match = getXpathResult(xpath, lpc); + + CRM_CHECK(match != NULL, continue); + + on_target = crm_element_value(match, "on_target"); + action = crm_element_value(match, "name"); ++ automatic = crm_element_value(match, "automatic"); ++ required = crm_element_value(match, "required"); + + if(safe_str_eq(action, "list")) { + set_bit(device->flags, st_device_supports_list); + } else if(safe_str_eq(action, "status")) { + set_bit(device->flags, st_device_supports_status); ++ } else if(safe_str_eq(action, "on") && (crm_is_true(automatic))) { ++ /* this setting implies required=true for unfencing */ ++ required = "true"; + } + + if (action && crm_is_true(on_target)) { +- if (strlen(actions)) { +- g_strlcat(actions, " ", 512); +- } +- g_strlcat(actions, action, 512); ++ device->on_target_actions = add_action(device->on_target_actions, action); ++ } ++ if (action && crm_is_true(required)) { ++ device->required_actions = add_action(device->required_actions, action); + } + } + + freeXpathObject(xpath); +- +- if (!strlen(actions)) { +- free(actions); +- actions = NULL; +- } +- +- device->on_target_actions = actions; + } + + static stonith_device_t * +@@ -603,13 +630,23 @@ build_device_from_xml(xmlNode * msg) + device->aliases = build_port_aliases(value, &(device->targets)); + + device->agent_metadata = get_agent_metadata(device->agent); +- get_on_target_actions(device); ++ read_action_metadata(device); + + value = g_hash_table_lookup(device->params, "nodeid"); + if (!value) { + device->include_nodeid = is_nodeid_required(device->agent_metadata); + } + ++ value = crm_element_value(dev, "rsc_provides"); ++ if (safe_str_eq(value, "unfencing")) { ++ /* if this agent requires unfencing, 'on' is considered a required action */ ++ device->required_actions = add_action(device->required_actions, "on"); ++ } ++ ++ if (is_action_required("on", device)) { ++ crm_info("The fencing device '%s' requires unfencing", device->id); ++ } ++ + if (device->on_target_actions) { + crm_info("The fencing device '%s' requires actions (%s) to be executed on the target node", + device->id, device->on_target_actions); +@@ -1460,6 +1497,7 @@ static void + st_child_done(GPid pid, int rc, const char *output, gpointer user_data) + { + stonith_device_t *device = NULL; ++ stonith_device_t *next_device = NULL; + async_command_t *cmd = user_data; + + GListPtr gIter = NULL; +@@ -1483,21 +1521,37 @@ st_child_done(GPid pid, int rc, const char *output, gpointer user_data) + mainloop_set_trigger(device->work); + } + +- crm_trace("Operation %s on %s completed with rc=%d (%d remaining)", ++ crm_debug("Operation '%s' on '%s' completed with rc=%d (%d remaining)", + cmd->action, cmd->device, rc, g_list_length(cmd->device_next)); + +- if (rc != 0 && cmd->device_next) { +- stonith_device_t *dev = g_hash_table_lookup(device_list, cmd->device_next->data); +- +- if (dev) { +- log_operation(cmd, rc, pid, dev->id, output); ++ if (rc == 0) { ++ GListPtr iter; ++ /* see if there are any required devices left to execute for this op */ ++ for (iter = cmd->device_next; iter != NULL; iter = iter->next) { ++ next_device = g_hash_table_lookup(device_list, iter->data); + +- cmd->device_next = cmd->device_next->next; +- schedule_stonith_command(cmd, dev); +- /* Prevent cmd from being freed */ +- cmd = NULL; +- goto done; ++ if (next_device != NULL && is_action_required(cmd->action, next_device)) { ++ cmd->device_next = iter->next; ++ break; ++ } ++ next_device = NULL; + } ++ ++ } else if (rc != 0 && cmd->device_next && (is_action_required(cmd->action, device) == FALSE)) { ++ /* if this device didn't work out, see if there are any others we can try. ++ * if the failed device was 'required', we can't pick another device. */ ++ next_device = g_hash_table_lookup(device_list, cmd->device_next->data); ++ cmd->device_next = cmd->device_next->next; ++ } ++ ++ /* this operation requires more fencing, hooray! */ ++ if (next_device) { ++ log_operation(cmd, rc, pid, device->id, output); ++ ++ schedule_stonith_command(cmd, next_device); ++ /* Prevent cmd from being freed */ ++ cmd = NULL; ++ goto done; + } + + if (rc > 0) { +diff --git a/fencing/internal.h b/fencing/internal.h +index 4e0525c..971a32f 100644 +--- a/fencing/internal.h ++++ b/fencing/internal.h +@@ -25,6 +25,7 @@ typedef struct stonith_device_s { + + /*! list of actions that must execute on the target node. Used for unfencing */ + char *on_target_actions; ++ char *required_actions; + GListPtr targets; + time_t targets_age; + gboolean has_attr_map; +diff --git a/fencing/main.c b/fencing/main.c +index e002125..16713fe 100644 +--- a/fencing/main.c ++++ b/fencing/main.c +@@ -645,9 +645,13 @@ static void cib_device_update(resource_t *rsc, pe_working_set_t *data_set) + const char *name = NULL; + const char *agent = crm_element_value(rsc->xml, XML_EXPR_ATTR_TYPE); + const char *provider = crm_element_value(rsc->xml, XML_AGENT_ATTR_PROVIDER); ++ const char *rsc_provides = NULL; + + crm_info("Device %s is allowed on %s: score=%d", rsc->id, stonith_our_uname, node->weight); + get_rsc_attributes(rsc->parameters, rsc, node, data_set); ++ get_meta_attributes(rsc->meta, rsc, node, data_set); ++ ++ rsc_provides = g_hash_table_lookup(rsc->meta, XML_RSC_ATTR_PROVIDES); + + g_hash_table_iter_init(&gIter, rsc->parameters); + while (g_hash_table_iter_next(&gIter, (gpointer *) & name, (gpointer *) & value)) { +@@ -658,7 +662,7 @@ static void cib_device_update(resource_t *rsc, pe_working_set_t *data_set) + crm_trace(" %s=%s", name, value); + } + +- data = create_device_registration_xml(rsc_name(rsc), provider, agent, params); ++ data = create_device_registration_xml(rsc_name(rsc), provider, agent, params, rsc_provides); + stonith_device_register(data, NULL, TRUE); + + stonith_key_value_freeall(params, 1, 1); +diff --git a/include/crm/fencing/internal.h b/include/crm/fencing/internal.h +index 4f30fed..2822e9a 100644 +--- a/include/crm/fencing/internal.h ++++ b/include/crm/fencing/internal.h +@@ -46,7 +46,7 @@ xmlNode *create_level_registration_xml(const char *node, int level, + stonith_key_value_t * device_list); + + xmlNode *create_device_registration_xml(const char *id, const char *namespace, const char *agent, +- stonith_key_value_t * params); ++ stonith_key_value_t * params, const char *rsc_provides); + + # define ST_LEVEL_MAX 10 + +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index 2be854f..06e0726 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -178,7 +178,7 @@ stonith_connection_destroy(gpointer user_data) + + xmlNode * + create_device_registration_xml(const char *id, const char *namespace, const char *agent, +- stonith_key_value_t * params) ++ stonith_key_value_t * params, const char *rsc_provides) + { + xmlNode *data = create_xml_node(NULL, F_STONITH_DEVICE); + xmlNode *args = create_xml_node(data, XML_TAG_ATTRS); +@@ -195,6 +195,9 @@ create_device_registration_xml(const char *id, const char *namespace, const char + crm_xml_add(data, "origin", __FUNCTION__); + crm_xml_add(data, "agent", agent); + crm_xml_add(data, "namespace", namespace); ++ if (rsc_provides) { ++ crm_xml_add(data, "rsc_provides", rsc_provides); ++ } + + for (; params; params = params->next) { + hash2field((gpointer) params->key, (gpointer) params->value, args); +@@ -211,7 +214,7 @@ stonith_api_register_device(stonith_t * st, int call_options, + int rc = 0; + xmlNode *data = NULL; + +- data = create_device_registration_xml(id, namespace, agent, params); ++ data = create_device_registration_xml(id, namespace, agent, params, NULL); + + rc = stonith_send_command(st, STONITH_OP_DEVICE_ADD, data, NULL, call_options, 0); + free_xml(data); diff --git a/SOURCES/bz1078078-pcmk-fencing_automatically_switch_from_list_to_status_to_static_list_if_those_actions_are_not_advertised_in_the_metadata.patch b/SOURCES/bz1078078-pcmk-fencing_automatically_switch_from_list_to_status_to_static_list_if_those_actions_are_not_advertised_in_the_metadata.patch new file mode 100644 index 0000000..247d9d7 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-fencing_automatically_switch_from_list_to_status_to_static_list_if_those_actions_are_not_advertised_in_the_metadata.patch @@ -0,0 +1,114 @@ +commit 0fb13527630fd308d533110ab91b7cf00538e8da +Author: Andrew Beekhof +Date: Wed Apr 9 09:05:36 2014 +1000 + + Fix: fencing: Automatically switch from 'list' to 'status' to 'static-list' if those actions are not advertised in the metadata + + (cherry picked from commit b87c249c7b862f6bac424e5d0bb5528e8f7b9c94) + +diff --git a/fencing/commands.c b/fencing/commands.c +index 909dc24..14933a5 100644 +--- a/fencing/commands.c ++++ b/fencing/commands.c +@@ -524,24 +524,24 @@ is_nodeid_required(xmlNode * xml) + return TRUE; + } + +-static char * +-get_on_target_actions(xmlNode * xml) ++static void ++get_on_target_actions(stonith_device_t *device) + { + char *actions = NULL; + xmlXPathObjectPtr xpath = NULL; + int max = 0; + int lpc = 0; + +- if (!xml) { +- return NULL; ++ if (device->agent_metadata == NULL) { ++ return; + } + +- xpath = xpath_search(xml, "//action"); ++ xpath = xpath_search(device->agent_metadata, "//action"); + max = numXpathResults(xpath); + + if (max <= 0) { + freeXpathObject(xpath); +- return NULL; ++ return; + } + + actions = calloc(1, 512); +@@ -556,6 +556,12 @@ get_on_target_actions(xmlNode * xml) + on_target = crm_element_value(match, "on_target"); + action = crm_element_value(match, "name"); + ++ if(safe_str_eq(action, "list")) { ++ set_bit(device->flags, st_device_supports_list); ++ } else if(safe_str_eq(action, "status")) { ++ set_bit(device->flags, st_device_supports_status); ++ } ++ + if (action && crm_is_true(on_target)) { + if (strlen(actions)) { + g_strlcat(actions, " ", 512); +@@ -571,7 +577,7 @@ get_on_target_actions(xmlNode * xml) + actions = NULL; + } + +- return actions; ++ device->on_target_actions = actions; + } + + static stonith_device_t * +@@ -596,7 +602,7 @@ build_device_from_xml(xmlNode * msg) + device->aliases = build_port_aliases(value, &(device->targets)); + + device->agent_metadata = get_agent_metadata(device->agent); +- device->on_target_actions = get_on_target_actions(device->agent_metadata); ++ get_on_target_actions(device); + + value = g_hash_table_lookup(device->params, "nodeid"); + if (!value) { +@@ -627,8 +633,12 @@ target_list_type(stonith_device_t * dev) + check_type = "static-list"; + } else if (g_hash_table_lookup(dev->params, STONITH_ATTR_HOSTMAP)) { + check_type = "static-list"; +- } else { ++ } else if(is_set(dev->flags, st_device_supports_list)){ + check_type = "dynamic-list"; ++ } else if(is_set(dev->flags, st_device_supports_status)){ ++ check_type = "status"; ++ } else { ++ check_type = "none"; + } + } + +diff --git a/fencing/internal.h b/fencing/internal.h +index da510c3..4e0525c 100644 +--- a/fencing/internal.h ++++ b/fencing/internal.h +@@ -12,6 +12,12 @@ + */ + gboolean stonith_check_fence_tolerance(int tolerance, const char *target, const char *action); + ++enum st_device_flags ++{ ++ st_device_supports_list = 0x0001, ++ st_device_supports_status = 0x0002, ++}; ++ + typedef struct stonith_device_s { + char *id; + char *agent; +@@ -27,6 +33,8 @@ typedef struct stonith_device_s { + guint priority; + guint active_pid; + ++ enum st_device_flags flags; ++ + GHashTable *params; + GHashTable *aliases; + GList *pending_ops; diff --git a/SOURCES/bz1078078-pcmk-fencing_correctly_record_which_peer_performed_the_fencing_operation.patch b/SOURCES/bz1078078-pcmk-fencing_correctly_record_which_peer_performed_the_fencing_operation.patch new file mode 100644 index 0000000..b7dc219 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-fencing_correctly_record_which_peer_performed_the_fencing_operation.patch @@ -0,0 +1,53 @@ +commit e40eaf4a1ba1787696a2fea3043e2620a5d98e3c +Author: Andrew Beekhof +Date: Thu Apr 10 11:50:46 2014 +1000 + + Fix: Fencing: Correctly record which peer performed the fencing operation + + (cherry picked from commit a932361589964545f76376009d7ff7a61f03cab3) + +diff --git a/fencing/commands.c b/fencing/commands.c +index abbb1ad..41690f7 100644 +--- a/fencing/commands.c ++++ b/fencing/commands.c +@@ -1405,7 +1405,8 @@ stonith_send_async_reply(async_command_t * cmd, const char *output, int rc, GPid + crm_xml_add_int(notify_data, F_STONITH_RC, rc); + crm_xml_add(notify_data, F_STONITH_TARGET, cmd->victim); + crm_xml_add(notify_data, F_STONITH_OPERATION, cmd->op); +- crm_xml_add(notify_data, F_STONITH_DELEGATE, cmd->device); ++ crm_xml_add(notify_data, F_STONITH_DELEGATE, "localhost"); ++ crm_xml_add(notify_data, F_STONITH_DEVICE, cmd->device); + crm_xml_add(notify_data, F_STONITH_REMOTE_OP_ID, cmd->remote_op_id); + crm_xml_add(notify_data, F_STONITH_ORIGIN, cmd->client); + +diff --git a/fencing/remote.c b/fencing/remote.c +index 399dce5..fd25025 100644 +--- a/fencing/remote.c ++++ b/fencing/remote.c +@@ -267,7 +267,12 @@ remote_op_done(remote_fencing_op_t * op, xmlNode * data, int rc, int dup) + } + + if (!op->delegate && data) { +- op->delegate = crm_element_value_copy(data, F_ORIG); ++ xmlNode *ndata = get_xpath_object("//@" F_STONITH_DELEGATE, data, LOG_WARNING); ++ if(ndata) { ++ op->delegate = crm_element_value_copy(ndata, F_STONITH_DELEGATE); ++ } else { ++ op->delegate = crm_element_value_copy(data, F_ORIG); ++ } + } + + if (data == NULL) { +diff --git a/lib/fencing/st_client.c b/lib/fencing/st_client.c +index 95ea0f7..2be854f 100644 +--- a/lib/fencing/st_client.c ++++ b/lib/fencing/st_client.c +@@ -1981,6 +1981,8 @@ xml_to_event(xmlNode * msg) + event->executioner = crm_element_value_copy(data, F_STONITH_DELEGATE); + event->id = crm_element_value_copy(data, F_STONITH_REMOTE_OP_ID); + event->client_origin = crm_element_value_copy(data, F_STONITH_CLIENTNAME); ++ event->device = crm_element_value_copy(data, F_STONITH_DEVICE); ++ + } else { + crm_err("No data for %s event", ntype); + crm_log_xml_notice(msg, "BadEvent"); diff --git a/SOURCES/bz1078078-pcmk-filter_self_fencing_at_the_peers_to_allow_unfencing_to_work_correctly.patch b/SOURCES/bz1078078-pcmk-filter_self_fencing_at_the_peers_to_allow_unfencing_to_work_correctly.patch new file mode 100644 index 0000000..71fca5e --- /dev/null +++ b/SOURCES/bz1078078-pcmk-filter_self_fencing_at_the_peers_to_allow_unfencing_to_work_correctly.patch @@ -0,0 +1,222 @@ +commit 5fbd9decbca3f40a8b29e9d1ef5d158808f71cf6 +Author: Andrew Beekhof +Date: Wed Apr 9 14:40:30 2014 +1000 + + Fix: fencing: Filter self-fencing at the peers to allow unfencing to work correctly + + (cherry picked from commit e1e7d7fad2b5ade3c6e433cb1b631d071b8d5e70) + + Conflicts: + fencing/commands.c + fencing/remote.c + +diff --git a/fencing/commands.c b/fencing/commands.c +index 14933a5..abbb1ad 100644 +--- a/fencing/commands.c ++++ b/fencing/commands.c +@@ -60,6 +60,7 @@ struct device_search_s { + int per_device_timeout; + int replies_needed; + int replies_received; ++ bool allow_suicide; + + void *user_data; + void (*callback) (GList * devices, void *user_data); +@@ -1072,8 +1073,17 @@ can_fence_host_with_device(stonith_device_t * dev, struct device_search_s *searc + + if (dev->on_target_actions && + search->action && +- strstr(dev->on_target_actions, search->action) && safe_str_neq(host, stonith_our_uname)) { ++ strstr(dev->on_target_actions, search->action)) { + /* this device can only execute this action on the target node */ ++ ++ if(safe_str_neq(host, stonith_our_uname)) { ++ crm_trace("%s operation with %s can only be executed for localhost not %s", ++ search->action, dev->id, host); ++ goto search_report_results; ++ } ++ ++ } else if(safe_str_eq(host, stonith_our_uname) && search->allow_suicide == FALSE) { ++ crm_trace("%s operation does not support self-fencing", search->action); + goto search_report_results; + } + +@@ -1146,7 +1156,7 @@ search_devices(gpointer key, gpointer value, gpointer user_data) + + #define DEFAULT_QUERY_TIMEOUT 20 + static void +-get_capable_devices(const char *host, const char *action, int timeout, void *user_data, ++get_capable_devices(const char *host, const char *action, int timeout, bool suicide, void *user_data, + void (*callback) (GList * devices, void *user_data)) + { + struct device_search_s *search; +@@ -1199,6 +1209,7 @@ get_capable_devices(const char *host, const char *action, int timeout, void *use + * unregistered some how during the async search, we will get + * the correct number of replies. */ + search->replies_needed = g_hash_table_size(device_list); ++ search->allow_suicide = suicide; + search->callback = callback; + search->user_data = user_data; + /* kick off the search */ +@@ -1313,7 +1324,9 @@ stonith_query(xmlNode * msg, const char *remote_peer, const char *client_id, int + query->action = action ? strdup(action) : NULL; + query->call_options = call_options; + +- get_capable_devices(target, action, timeout, query, stonith_query_capable_device_cb); ++ get_capable_devices(target, action, timeout, ++ is_set(call_options, st_opt_allow_suicide), ++ query, stonith_query_capable_device_cb); + } + + #define ST_LOG_OUTPUT_MAX 512 +@@ -1632,8 +1645,10 @@ stonith_fence(xmlNode * msg) + host = node->uname; + } + } +- get_capable_devices(host, cmd->action, cmd->default_timeout, cmd, +- stonith_fence_get_devices_cb); ++ ++ /* If we get to here, then self-fencing is implicitly allowed */ ++ get_capable_devices(host, cmd->action, cmd->default_timeout, ++ TRUE, cmd, stonith_fence_get_devices_cb); + } + + return -EINPROGRESS; +diff --git a/fencing/remote.c b/fencing/remote.c +index 8c8df6d..399dce5 100644 +--- a/fencing/remote.c ++++ b/fencing/remote.c +@@ -558,6 +558,7 @@ create_remote_stonith_op(const char *client, xmlNode * request, gboolean peer) + { + remote_fencing_op_t *op = NULL; + xmlNode *dev = get_xpath_object("//@" F_STONITH_TARGET, request, LOG_TRACE); ++ int call_options = 0; + + if (remote_op_list == NULL) { + remote_op_list = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, free_remote_op); +@@ -612,7 +613,9 @@ create_remote_stonith_op(const char *client, xmlNode * request, gboolean peer) + + op->target = crm_element_value_copy(dev, F_STONITH_TARGET); + op->request = copy_xml(request); /* TODO: Figure out how to avoid this */ +- crm_element_value_int(request, F_STONITH_CALLOPTS, (int *)&(op->call_options)); ++ crm_element_value_int(request, F_STONITH_CALLOPTS, &call_options); ++ op->call_options = call_options; ++ + crm_element_value_int(request, F_STONITH_CALLID, (int *)&(op->client_callid)); + + crm_trace("%s new stonith op: %s - %s of %s for %s", +@@ -662,7 +665,7 @@ initiate_remote_stonith_op(crm_client_t * client, xmlNode * request, gboolean ma + op->target, op->id); + return op; + } +- ++ + CRM_CHECK(op->action, return NULL); + + if (stonith_topology_next(op) != pcmk_ok) { +@@ -695,6 +698,7 @@ initiate_remote_stonith_op(crm_client_t * client, xmlNode * request, gboolean ma + crm_xml_add(query, F_STONITH_CLIENTID, op->client_id); + crm_xml_add(query, F_STONITH_CLIENTNAME, op->client_name); + crm_xml_add_int(query, F_STONITH_TIMEOUT, op->base_timeout); ++ crm_xml_add_int(query, F_STONITH_CALLOPTS, op->call_options); + + send_cluster_message(NULL, crm_msg_stonith_ng, query, FALSE); + free_xml(query); +@@ -964,29 +968,30 @@ call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer) + + if (peer) { + int timeout_one = 0; +- xmlNode *query = stonith_create_op(op->client_callid, op->id, STONITH_OP_FENCE, NULL, 0); ++ xmlNode *remote_op = stonith_create_op(op->client_callid, op->id, STONITH_OP_FENCE, NULL, 0); + +- crm_xml_add(query, F_STONITH_REMOTE_OP_ID, op->id); +- crm_xml_add(query, F_STONITH_TARGET, op->target); +- crm_xml_add(query, F_STONITH_ACTION, op->action); +- crm_xml_add(query, F_STONITH_ORIGIN, op->originator); +- crm_xml_add(query, F_STONITH_CLIENTID, op->client_id); +- crm_xml_add(query, F_STONITH_CLIENTNAME, op->client_name); +- crm_xml_add_int(query, F_STONITH_TIMEOUT, timeout); ++ crm_xml_add(remote_op, F_STONITH_REMOTE_OP_ID, op->id); ++ crm_xml_add(remote_op, F_STONITH_TARGET, op->target); ++ crm_xml_add(remote_op, F_STONITH_ACTION, op->action); ++ crm_xml_add(remote_op, F_STONITH_ORIGIN, op->originator); ++ crm_xml_add(remote_op, F_STONITH_CLIENTID, op->client_id); ++ crm_xml_add(remote_op, F_STONITH_CLIENTNAME, op->client_name); ++ crm_xml_add_int(remote_op, F_STONITH_TIMEOUT, timeout); ++ crm_xml_add_int(remote_op, F_STONITH_CALLOPTS, op->call_options); + + if (device) { + timeout_one = + TIMEOUT_MULTIPLY_FACTOR * get_device_timeout(peer, device, op->base_timeout); + crm_info("Requesting that %s perform op %s %s with %s for %s (%ds)", peer->host, + op->action, op->target, device, op->client_name, timeout_one); +- crm_xml_add(query, F_STONITH_DEVICE, device); +- crm_xml_add(query, F_STONITH_MODE, "slave"); ++ crm_xml_add(remote_op, F_STONITH_DEVICE, device); ++ crm_xml_add(remote_op, F_STONITH_MODE, "slave"); + + } else { + timeout_one = TIMEOUT_MULTIPLY_FACTOR * get_peer_timeout(peer, op->base_timeout); + crm_info("Requesting that %s perform op %s %s for %s (%ds)", + peer->host, op->action, op->target, op->client_name, timeout_one); +- crm_xml_add(query, F_STONITH_MODE, "smart"); ++ crm_xml_add(remote_op, F_STONITH_MODE, "smart"); + } + + op->state = st_exec; +@@ -995,9 +1000,9 @@ call_remote_stonith(remote_fencing_op_t * op, st_query_result_t * peer) + } + op->op_timer_one = g_timeout_add((1000 * timeout_one), remote_op_timeout_one, op); + +- send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, query, FALSE); ++ send_cluster_message(crm_get_peer(0, peer->host), crm_msg_stonith_ng, remote_op, FALSE); + peer->tried = TRUE; +- free_xml(query); ++ free_xml(remote_op); + return; + + } else if (op->owner == FALSE) { +@@ -1131,24 +1136,20 @@ process_remote_stonith_query(xmlNode * msg) + + if (devices <= 0) { + /* If we're doing 'known' then we might need to fire anyway */ +- crm_trace("Query result from %s (%d devices)", host, devices); ++ crm_trace("Query result %d of %d from %s for %s/%s (%d devices) %s", ++ op->replies, op->replies_expected, host, ++ op->target, op->action, devices, id); + if(op->state == st_query && (op->replies >= op->replies_expected || op->replies >= active)) { +- crm_info("All queries have arrived, continuing (%d, %d, %d) ", op->replies_expected, active, op->replies); ++ crm_info("All queries have arrived, continuing (%d, %d, %d, %s)", ++ op->replies_expected, active, op->replies, id); + call_remote_stonith(op, NULL); + } + return pcmk_ok; +- +- } else if (host_is_target) { +- if (op->call_options & st_opt_allow_suicide) { +- crm_trace("Allowing %s to potentialy fence itself", op->target); +- } else { +- crm_info("Ignoring reply from %s, hosts are not permitted to commit suicide", +- op->target); +- return pcmk_ok; +- } + } + +- crm_info("Query result %d of %d from %s (%d devices)", op->replies, op->replies_expected, host, devices); ++ crm_info("Query result %d of %d from %s for %s/%s (%d devices) %s", ++ op->replies, op->replies_expected, host, ++ op->target, op->action, devices, id); + result = calloc(1, sizeof(st_query_result_t)); + result->host = strdup(host); + result->devices = devices; +@@ -1206,10 +1207,6 @@ process_remote_stonith_query(xmlNode * msg) + crm_trace("Found %d verified devices", g_hash_table_size(result->verified_devices)); + call_remote_stonith(op, result); + +- } else if (safe_str_eq(op->action, "on")) { +- crm_trace("Unfencing %s", op->target); +- call_remote_stonith(op, result); +- + } else if(op->replies >= op->replies_expected || op->replies >= active) { + crm_info("All queries have arrived, continuing (%d, %d, %d) ", op->replies_expected, active, op->replies); + call_remote_stonith(op, NULL); diff --git a/SOURCES/bz1078078-pcmk-logging_daemons_always_get_a_log_file_unless_explicitly_set_to_configured_none.patch b/SOURCES/bz1078078-pcmk-logging_daemons_always_get_a_log_file_unless_explicitly_set_to_configured_none.patch new file mode 100644 index 0000000..9e30fea --- /dev/null +++ b/SOURCES/bz1078078-pcmk-logging_daemons_always_get_a_log_file_unless_explicitly_set_to_configured_none.patch @@ -0,0 +1,40 @@ +commit 63547179498c939dc1c3450c85fac23b3c86742c +Author: Andrew Beekhof +Date: Tue Apr 8 11:26:56 2014 +1000 + + Feature: logging: daemons always get a log file, unless explicitly set to configured 'none' + + (cherry picked from commit 8410680fbd7d5d1fea8e411fdf32f9c3ebe889cb) + +diff --git a/lib/common/logging.c b/lib/common/logging.c +index 571cd03..ad6bcfd 100644 +--- a/lib/common/logging.c ++++ b/lib/common/logging.c +@@ -696,7 +696,12 @@ crm_log_init(const char *entity, int level, gboolean daemon, gboolean to_stderr, + + crm_enable_stderr(to_stderr); + +- if (logfile) { ++ if (safe_str_eq("none", logfile)) { ++ /* No soup^Hlogs for you! */ ++ } else if(crm_is_daemon) { ++ /* The daemons always get a log file, unless explicitly set to configured 'none' */ ++ crm_add_logfile(logfile); ++ } else if(logfile) { + crm_add_logfile(logfile); + } + +@@ -718,13 +723,6 @@ crm_log_init(const char *entity, int level, gboolean daemon, gboolean to_stderr, + set_daemon_option("logfacility", facility); + } + +- if (crm_is_daemon && crm_tracing_enabled() +- && qb_log_ctl(QB_LOG_STDERR, QB_LOG_CONF_STATE_GET, 0) != QB_LOG_STATE_ENABLED +- && qb_log_ctl(QB_LOG_BLACKBOX, QB_LOG_CONF_STATE_GET, 0) != QB_LOG_STATE_ENABLED) { +- /* Make sure tracing goes somewhere */ +- crm_add_logfile(NULL); +- } +- + crm_update_callsites(); + + /* Ok, now we can start logging... */ diff --git a/SOURCES/bz1078078-pcmk-logging_simplify_processing_of_logging_directives.patch b/SOURCES/bz1078078-pcmk-logging_simplify_processing_of_logging_directives.patch new file mode 100644 index 0000000..5a3de7a --- /dev/null +++ b/SOURCES/bz1078078-pcmk-logging_simplify_processing_of_logging_directives.patch @@ -0,0 +1,349 @@ +commit 3cd57be790a6455b115d07c31bdb9a72767d7ee8 +Author: Andrew Beekhof +Date: Tue Apr 8 14:45:24 2014 +1000 + + Refactor: logging: Simplify processing of logging directives + + (cherry picked from commit b3b5a57bca646377f644ff91ff724062ec8ec460) + + Conflicts: + mcp/corosync.c + +diff --git a/lib/ais/utils.c b/lib/ais/utils.c +index 465e381..0a3f95e 100644 +--- a/lib/ais/utils.c ++++ b/lib/ais/utils.c +@@ -237,7 +237,7 @@ spawn_child(crm_child_t * child) + /* *INDENT-ON* */ + + if (pcmk_env.logfile) { +- setenv("HA_debugfile", pcmk_env.logfile, 1); ++ setenv("HA_logfile", pcmk_env.logfile, 1); + } + + if (use_valgrind) { +diff --git a/lib/common/logging.c b/lib/common/logging.c +index ad6bcfd..167583e 100644 +--- a/lib/common/logging.c ++++ b/lib/common/logging.c +@@ -234,6 +234,10 @@ crm_add_logfile(const char *filename) + + if (filename == NULL) { + return FALSE; /* Nothing to do */ ++ } else if(safe_str_eq(filename, "none")) { ++ return FALSE; /* Nothing to do */ ++ } else if(safe_str_eq(filename, "/dev/null")) { ++ return FALSE; /* Nothing to do */ + } + + /* Check the parent directory */ +@@ -317,6 +321,7 @@ crm_add_logfile(const char *filename) + /* Enable callsites */ + crm_update_callsites(); + have_logfile = TRUE; ++ + return TRUE; + } + +@@ -623,7 +628,8 @@ crm_log_init(const char *entity, int level, gboolean daemon, gboolean to_stderr, + int argc, char **argv, gboolean quiet) + { + int lpc = 0; +- const char *logfile = daemon_option("debugfile"); ++ int32_t qb_facility = 0; ++ const char *logfile = daemon_option("logfile"); + const char *facility = daemon_option("logfacility"); + const char *f_copy = facility; + +@@ -643,14 +649,7 @@ crm_log_init(const char *entity, int level, gboolean daemon, gboolean to_stderr, + /* and for good measure... - this enum is a bit field (!) */ + g_log_set_always_fatal((GLogLevelFlags) 0); /*value out of range */ + +- if (facility == NULL) { +- facility = "daemon"; +- +- } else if (safe_str_eq(facility, "none")) { +- facility = "daemon"; +- quiet = TRUE; +- } +- ++ /* Who do we log as */ + if (entity) { + free(crm_system_name); + crm_system_name = strdup(entity); +@@ -673,29 +672,54 @@ crm_log_init(const char *entity, int level, gboolean daemon, gboolean to_stderr, + + setenv("PCMK_service", crm_system_name, 1); + +- if (daemon_option_enabled(crm_system_name, "debug")) { +- /* Override the default setting */ +- level = LOG_DEBUG; ++ /* Should we log to syslog */ ++ if (facility == NULL) { ++ if(crm_is_daemon) { ++ facility = "daemon"; ++ } else { ++ facility = "none"; ++ } ++ set_daemon_option("logfacility", facility); + } + +- if (daemon_option_enabled(crm_system_name, "stderr")) { ++ if (safe_str_eq(facility, "none")) { ++ quiet = TRUE; ++ qb_facility = qb_log_facility2int("daemon"); ++ ++ } else { ++ qb_facility = qb_log_facility2int(facility); ++ } ++ ++ if (daemon_option_enabled(crm_system_name, "debug")) { + /* Override the default setting */ +- to_stderr = TRUE; ++ level = LOG_DEBUG; + } + ++ /* What lower threshold do we have for sending to syslog */ + crm_log_priority = crm_priority2int(daemon_option("logpriority")); + + crm_log_level = level; +- qb_log_init(crm_system_name, qb_log_facility2int(facility), level); +- qb_log_tags_stringify_fn_set(crm_quark_to_string); ++ qb_log_init(crm_system_name, qb_facility, crm_log_level); + +- /* Set default format strings */ ++ if (quiet) { ++ /* Nuke any syslog activity */ ++ qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE); ++ } ++ ++ /* Set format strings */ ++ qb_log_tags_stringify_fn_set(crm_quark_to_string); + for (lpc = QB_LOG_SYSLOG; lpc < QB_LOG_TARGET_MAX; lpc++) { + set_format_string(lpc, crm_system_name); + } + ++ /* Should we log to stderr */ ++ if (daemon_option_enabled(crm_system_name, "stderr")) { ++ /* Override the default setting */ ++ to_stderr = TRUE; ++ } + crm_enable_stderr(to_stderr); + ++ /* Should we log to a file */ + if (safe_str_eq("none", logfile)) { + /* No soup^Hlogs for you! */ + } else if(crm_is_daemon) { +@@ -709,20 +733,11 @@ crm_log_init(const char *entity, int level, gboolean daemon, gboolean to_stderr, + crm_enable_blackbox(0); + } + ++ /* Summary */ + crm_trace("Quiet: %d, facility %s", quiet, f_copy); +- daemon_option("debugfile"); ++ daemon_option("logfile"); + daemon_option("logfacility"); + +- if (quiet) { +- /* Nuke any syslog activity */ +- facility = NULL; +- qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE); +- } +- +- if (crm_is_daemon) { +- set_daemon_option("logfacility", facility); +- } +- + crm_update_callsites(); + + /* Ok, now we can start logging... */ +diff --git a/mcp/corosync.c b/mcp/corosync.c +index a97c8d8..24f5c22 100644 +--- a/mcp/corosync.c ++++ b/mcp/corosync.c +@@ -266,16 +266,9 @@ mcp_read_config(void) + { + int rc = CS_OK; + int retries = 0; +- gboolean have_log = FALSE; + + const char *const_value = NULL; + +- char *logging_debug = NULL; +- char *logging_logfile = NULL; +- char *logging_to_logfile = NULL; +- char *logging_to_syslog = NULL; +- char *logging_syslog_facility = NULL; +- + #if HAVE_CONFDB + char *value = NULL; + confdb_handle_t config = 0; +@@ -379,35 +372,83 @@ mcp_read_config(void) + #endif + + /* =::=::= Logging =::=::= */ +- get_config_opt(config, local_handle, KEY_PREFIX "debug", &logging_debug, "off"); ++ if (daemon_option("debug")) { ++ /* Syslog logging is already setup by crm_log_init() */ ++ ++ } else { ++ /* Check corosync */ ++ char *debug_enabled = NULL; ++ ++ get_config_opt(config, local_handle, KEY_PREFIX "debug", &debug_enabled, "off"); ++ ++ if (crm_is_true(debug_enabled)) { ++ set_daemon_option("debug", "1"); ++ if (get_crm_log_level() < LOG_DEBUG) { ++ set_crm_log_level(LOG_DEBUG); ++ } ++ ++ } else { ++ set_daemon_option("debug", "0"); ++ } ++ ++ free(debug_enabled); ++ } + + const_value = daemon_option("debugfile"); +- if (const_value) { +- logging_to_logfile = strdup("on"); +- logging_logfile = strdup(const_value); +- crm_trace("Using debugfile setting from the environment: %s", logging_logfile); ++ if (daemon_option("logfile")) { ++ /* File logging is already setup by crm_log_init() */ ++ ++ } else if(const_value) { ++ /* From when we cared what options heartbeat used */ ++ set_daemon_option("logfile", const_value); ++ crm_add_logfile(const_value); + + } else { +- get_config_opt(config, local_handle, KEY_PREFIX "to_logfile", &logging_to_logfile, "off"); +- get_config_opt(config, local_handle, KEY_PREFIX "logfile", &logging_logfile, +- "/var/log/pacemaker"); +- } ++ /* Check corosync */ ++ char *logfile = NULL; ++ char *logfile_enabled = NULL; ++ ++ get_config_opt(config, local_handle, KEY_PREFIX "to_logfile", &logfile_enabled, "on"); ++ get_config_opt(config, local_handle, KEY_PREFIX "logfile", &logfile, "/var/log/pacemaker.log"); ++ ++ if (crm_is_true(logfile_enabled) == FALSE) { ++ crm_trace("File logging disabled in corosync"); + +- const_value = daemon_option("logfacility"); +- if (const_value) { +- logging_syslog_facility = strdup(const_value); +- crm_trace("Using logfacility setting from the environment: %s", logging_syslog_facility); ++ } else if (crm_add_logfile(logfile)) { ++ set_daemon_option("logfile", logfile); + +- if (safe_str_eq(logging_syslog_facility, "none")) { +- logging_to_syslog = strdup("off"); + } else { +- logging_to_syslog = strdup("on"); ++ crm_err("Couldn't create logfile: %s", logfile); ++ set_daemon_option("logfile", "none"); + } + ++ free(logfile); ++ free(logfile_enabled); ++ } ++ ++ if (daemon_option("logfacility")) { ++ /* Syslog logging is already setup by crm_log_init() */ ++ + } else { +- get_config_opt(config, local_handle, KEY_PREFIX "to_syslog", &logging_to_syslog, "on"); +- get_config_opt(config, local_handle, KEY_PREFIX "syslog_facility", &logging_syslog_facility, +- "daemon"); ++ /* Check corosync */ ++ char *syslog_enabled = NULL; ++ char *syslog_facility = NULL; ++ ++ get_config_opt(config, local_handle, KEY_PREFIX "to_syslog", &syslog_enabled, "on"); ++ get_config_opt(config, local_handle, KEY_PREFIX "syslog_facility", &syslog_facility, "daemon"); ++ ++ if (crm_is_true(syslog_enabled) == FALSE) { ++ qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE); ++ set_daemon_option("logfacility", "none"); ++ ++ } else { ++ qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_FACILITY, qb_log_facility2int(syslog_facility)); ++ qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_TRUE); ++ set_daemon_option("logfacility", syslog_facility); ++ } ++ ++ free(syslog_enabled); ++ free(syslog_facility); + } + + #if HAVE_CONFDB +@@ -429,52 +470,5 @@ mcp_read_config(void) + cmap_finalize(local_handle); + #endif + +- if (daemon_option("debug")) { +- crm_trace("Using debug setting from the environment: %s", daemon_option("debug")); +- if (get_crm_log_level() < LOG_DEBUG && daemon_option_enabled("pacemakerd", "debug")) { +- set_crm_log_level(LOG_DEBUG); +- } +- +- } else if (crm_is_true(logging_debug)) { +- set_daemon_option("debug", "1"); +- if (get_crm_log_level() < LOG_DEBUG) { +- set_crm_log_level(LOG_DEBUG); +- } +- +- } else { +- set_daemon_option("debug", "0"); +- } +- +- if (crm_is_true(logging_to_logfile)) { +- if (crm_add_logfile(logging_logfile)) { +- /* What a cluster fsck, eventually we need to mandate /one/ */ +- set_daemon_option("debugfile", logging_logfile); +- set_daemon_option("DEBUGLOG", logging_logfile); +- have_log = TRUE; +- +- } else { +- crm_err("Couldn't create logfile: %s", logging_logfile); +- } +- } +- +- if (have_log && crm_is_true(logging_to_syslog) == FALSE) { +- qb_log_ctl(QB_LOG_SYSLOG, QB_LOG_CONF_ENABLED, QB_FALSE); +- free(logging_syslog_facility); +- logging_syslog_facility = strdup("none"); +- crm_info("User configured file based logging and explicitly disabled syslog."); +- +- } else if (crm_is_true(logging_to_syslog) == FALSE) { +- crm_err("Please enable some sort of logging, either 'to_logfile: on' or 'to_syslog: on'."); +- crm_err("If you use file logging, be sure to also define a value for 'logfile'"); +- } +- +- set_daemon_option("logfacility", logging_syslog_facility); +- setenv("HA_LOGFACILITY", logging_syslog_facility, 1); +- +- free(logging_debug); +- free(logging_logfile); +- free(logging_to_logfile); +- free(logging_to_syslog); +- free(logging_syslog_facility); + return TRUE; + } +diff --git a/mcp/pacemaker.sysconfig b/mcp/pacemaker.sysconfig +index 7f12111..50b4814 100644 +--- a/mcp/pacemaker.sysconfig ++++ b/mcp/pacemaker.sysconfig +@@ -14,7 +14,7 @@ + # Send INFO (and higher) messages to the named log file + # Additional messages may also appear here depending on any configured debug and trace settings + # By default Pacemaker will inherit the logfile specified in corosync.conf +-# PCMK_debugfile=/var/log/pacemaker.log ++# PCMK_logfile=/var/log/pacemaker.log + + # Specify an alternate syslog target for NOTICE (and higher) messages + # Use 'none' to disable - not recommended diff --git a/SOURCES/bz1078078-pcmk-lrmd_expose_logging_variables_expected_by_ocf_agents.patch b/SOURCES/bz1078078-pcmk-lrmd_expose_logging_variables_expected_by_ocf_agents.patch new file mode 100644 index 0000000..31ff71b --- /dev/null +++ b/SOURCES/bz1078078-pcmk-lrmd_expose_logging_variables_expected_by_ocf_agents.patch @@ -0,0 +1,41 @@ +commit 20d6b6e542b9814b081355ed19874f75445b0901 +Author: Andrew Beekhof +Date: Tue Apr 8 15:44:00 2014 +1000 + + Fix: lrmd: Expose logging variables expected by OCF agents + + (cherry picked from commit fa8bd56b776b97c3c845986d084c01b4aaf4063c) + +diff --git a/lrmd/main.c b/lrmd/main.c +index 38c5466..8e00f11 100644 +--- a/lrmd/main.c ++++ b/lrmd/main.c +@@ -246,6 +246,7 @@ main(int argc, char **argv) + int rc = 0; + int flag = 0; + int index = 0; ++ const char *option = NULL; + + #ifdef ENABLE_PCMK_REMOTE + crm_log_init("pacemaker_remoted", LOG_INFO, TRUE, FALSE, argc, argv, FALSE); +@@ -282,6 +283,20 @@ main(int argc, char **argv) + } + } + ++ option = daemon_option("logfacility"); ++ if(safe_str_neq(option, "none")) { ++ setenv("HA_LOGFACILITY", option, 1); /* Used by the ocf_log/ha_log OCF macro */ ++ } ++ ++ option = daemon_option("logfile"); ++ if(safe_str_neq(option, "none")) { ++ setenv("HA_LOGFILE", option, 1); /* Used by the ocf_log/ha_log OCF macro */ ++ ++ if (daemon_option_enabled(crm_system_name, "debug")) { ++ setenv("HA_DEBUGLOG", option, 1); /* Used by the ocf_log/ha_debug OCF macro */ ++ } ++ } ++ + /* Used by RAs - Leave owned by root */ + crm_build_path(CRM_RSCTMP_DIR, 0755); + diff --git a/SOURCES/bz1078078-pcmk-lrmd_provide_stderr_output_from_agents_if_available_otherwise_fall_back_to_stdout.patch b/SOURCES/bz1078078-pcmk-lrmd_provide_stderr_output_from_agents_if_available_otherwise_fall_back_to_stdout.patch new file mode 100644 index 0000000..9ad7774 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-lrmd_provide_stderr_output_from_agents_if_available_otherwise_fall_back_to_stdout.patch @@ -0,0 +1,151 @@ +commit 2c6e6e1ad3ecbb186f9a43f89280d0d4391277e5 +Author: Andrew Beekhof +Date: Tue Apr 8 12:13:44 2014 +1000 + + Fix: lrmd: Provide stderr output from agents if available, otherwise fall back to stdout + + (cherry picked from commit d9cc751625ff3adfa158a2f769ab9038f6e317fd) + +diff --git a/lib/services/services_linux.c b/lib/services/services_linux.c +index ac0138f..534e362 100644 +--- a/lib/services/services_linux.c ++++ b/lib/services/services_linux.c +@@ -57,34 +57,37 @@ set_fd_opts(int fd, int opts) + } + + static gboolean +-read_output(int fd, svc_action_t * op) ++svc_read_output(int fd, svc_action_t * op, bool is_stderr) + { + char *data = NULL; + int rc = 0, len = 0; +- gboolean is_err = FALSE; + char buf[500]; + static const size_t buf_read_len = sizeof(buf) - 1; + +- crm_trace("%p", op); + + if (fd < 0) { ++ crm_trace("No fd for %s", op->id); + return FALSE; + } + +- if (fd == op->opaque->stderr_fd) { +- is_err = TRUE; +- if (op->stderr_data) { +- len = strlen(op->stderr_data); +- data = op->stderr_data; +- } +- } else if (op->stdout_data) { ++ if (is_stderr && op->stderr_data) { ++ len = strlen(op->stderr_data); ++ data = op->stderr_data; ++ crm_trace("Reading %s stderr into offset %d", op->id, len); ++ ++ } else if (is_stderr == FALSE && op->stdout_data) { + len = strlen(op->stdout_data); + data = op->stdout_data; ++ crm_trace("Reading %s stdout into offset %d", op->id, len); ++ ++ } else { ++ crm_trace("Reading %s %s", op->id, is_stderr?"stderr":"stdout", len); + } + + do { + rc = read(fd, buf, buf_read_len); + if (rc > 0) { ++ crm_trace("Got %d characters starting with %.20s", rc, buf); + buf[rc] = 0; + data = realloc(data, len + rc + 1); + sprintf(data + len, "%s", buf); +@@ -99,7 +102,7 @@ read_output(int fd, svc_action_t * op) + + } while (rc == buf_read_len || rc < 0); + +- if (data != NULL && is_err) { ++ if (data != NULL && is_stderr) { + op->stderr_data = data; + } else if (data != NULL) { + op->stdout_data = data; +@@ -113,7 +116,7 @@ dispatch_stdout(gpointer userdata) + { + svc_action_t *op = (svc_action_t *) userdata; + +- return read_output(op->opaque->stdout_fd, op); ++ return svc_read_output(op->opaque->stdout_fd, op, FALSE); + } + + static int +@@ -121,7 +124,7 @@ dispatch_stderr(gpointer userdata) + { + svc_action_t *op = (svc_action_t *) userdata; + +- return read_output(op->opaque->stderr_fd, op); ++ return svc_read_output(op->opaque->stderr_fd, op, TRUE); + } + + static void +@@ -264,18 +267,23 @@ operation_finished(mainloop_child_t * p, pid_t pid, int core, int signo, int exi + op->status = PCMK_LRM_OP_DONE; + CRM_ASSERT(op->pid == pid); + ++ crm_trace("%s %p %p", prefix, op->opaque->stderr_gsource, op->opaque->stdout_gsource); + if (op->opaque->stderr_gsource) { + /* Make sure we have read everything from the buffer. + * Depending on the priority mainloop gives the fd, operation_finished + * could occur before all the reads are done. Force the read now.*/ ++ crm_trace("%s dispatching stderr", prefix); + dispatch_stderr(op); ++ crm_trace("%s: %p", op->stderr_data); + } + + if (op->opaque->stdout_gsource) { + /* Make sure we have read everything from the buffer. + * Depending on the priority mainloop gives the fd, operation_finished + * could occur before all the reads are done. Force the read now.*/ ++ crm_trace("%s dispatching stdout", prefix); + dispatch_stdout(op); ++ crm_trace("%s: %p", op->stdout_data); + } + + if (signo) { +@@ -495,11 +503,11 @@ services_os_action_execute(svc_action_t * op, gboolean synchronous) + + if (poll_rc > 0) { + if (fds[0].revents & POLLIN) { +- read_output(op->opaque->stdout_fd, op); ++ svc_read_output(op->opaque->stdout_fd, op, FALSE); + } + + if (fds[1].revents & POLLIN) { +- read_output(op->opaque->stderr_fd, op); ++ svc_read_output(op->opaque->stderr_fd, op, TRUE); + } + + if (fds[2].revents & POLLIN) { +@@ -578,8 +586,8 @@ services_os_action_execute(svc_action_t * op, gboolean synchronous) + } + #endif + +- read_output(op->opaque->stdout_fd, op); +- read_output(op->opaque->stderr_fd, op); ++ svc_read_output(op->opaque->stdout_fd, op, FALSE); ++ svc_read_output(op->opaque->stderr_fd, op, TRUE); + + close(op->opaque->stdout_fd); + close(op->opaque->stderr_fd); +diff --git a/lrmd/lrmd.c b/lrmd/lrmd.c +index 65421d0..517e98f 100644 +--- a/lrmd/lrmd.c ++++ b/lrmd/lrmd.c +@@ -599,7 +599,9 @@ action_complete(svc_action_t * action) + cmd->lrmd_op_status = action->status; + rsc = cmd->rsc_id ? g_hash_table_lookup(rsc_list, cmd->rsc_id) : NULL; + +- if (action->stdout_data) { ++ if (action->stderr_data) { ++ cmd->output = strdup(action->stderr_data); ++ } else if (action->stdout_data) { + cmd->output = strdup(action->stdout_data); + } + #if SUPPORT_NAGIOS diff --git a/SOURCES/bz1078078-pcmk-pe_automatically_re_unfence_a_node_if_the_fencing_device_definition_changes.patch b/SOURCES/bz1078078-pcmk-pe_automatically_re_unfence_a_node_if_the_fencing_device_definition_changes.patch new file mode 100644 index 0000000..aa43324 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_automatically_re_unfence_a_node_if_the_fencing_device_definition_changes.patch @@ -0,0 +1,577 @@ +commit 5efd173dac6b22eee48f1f9372b2e86cb6d27282 +Author: Andrew Beekhof +Date: Thu Apr 3 16:02:23 2014 +1100 + + Feature: PE: Automatically re-unfence a node if the fencing device definition changes + + Ensures all resources that require unfencing are stopped first + If unfencing is enabled, this is the new default for resource.requires + + (cherry picked from commit 26fe2e39b9e060597cddeeb69d8b846784dfac55) + + Conflicts: + include/crm/pengine/internal.h + include/crm/pengine/status.h + lib/pengine/complex.c + +diff --git a/fencing/main.c b/fencing/main.c +index c40610e..e002125 100644 +--- a/fencing/main.c ++++ b/fencing/main.c +@@ -573,7 +573,6 @@ fencing_topology_init(xmlNode * msg) + } + + #define rsc_name(x) x->clone_name?x->clone_name:x->id +-static bool have_fence_scsi = FALSE; + + static void cib_device_update(resource_t *rsc, pe_working_set_t *data_set) + { +@@ -662,34 +661,6 @@ static void cib_device_update(resource_t *rsc, pe_working_set_t *data_set) + data = create_device_registration_xml(rsc_name(rsc), provider, agent, params); + stonith_device_register(data, NULL, TRUE); + +- /* If required, unfence ourselves on cluster startup +- * +- * Make this generic/smarter if/when more than a single agent needs this +- */ +- if(have_fence_scsi == FALSE && safe_str_eq(agent, "fence_scsi")) { +- stonith_device_t *device = g_hash_table_lookup(device_list, rsc->id); +- +- if(stonith_our_uname == NULL) { +- crm_trace("Cannot unfence ourselves: no local host name"); +- +- } else if(device == NULL) { +- crm_err("Cannot unfence ourselves: no such device '%s'", rsc->id); +- +- } else { +- const char *alias = g_hash_table_lookup(device->aliases, stonith_our_uname); +- +- if (!alias) { +- alias = stonith_our_uname; +- } +- +- if (device->targets && string_in_list(device->targets, alias)) { +- have_fence_scsi = TRUE; +- crm_notice("Unfencing ourselves with %s (%s)", agent, device->id); +- schedule_internal_command(__FUNCTION__, device, "on", stonith_our_uname, 0, NULL, unfence_cb); +- } +- } +- } +- + stonith_key_value_freeall(params, 1, 1); + free_xml(data); + } +diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h +index 8575163..055f476 100644 +--- a/include/crm/msg_xml.h ++++ b/include/crm/msg_xml.h +@@ -200,6 +200,7 @@ + # define XML_RSC_ATTR_MULTIPLE "multiple-active" + # define XML_RSC_ATTR_PRIORITY "priority" + # define XML_RSC_ATTR_REQUIRES "requires" ++# define XML_RSC_ATTR_PROVIDES "provides" + # define XML_RSC_ATTR_CONTAINER "container" + # define XML_RSC_ATTR_INTERNAL_RSC "internal_rsc" + # define XML_OP_ATTR_ON_FAIL "on-fail" +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index dd52e7f..657c647 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -261,6 +261,7 @@ op_digest_cache_t *rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, nod + pe_working_set_t * data_set); + + gboolean is_remote_node(xmlNode *xml); ++void clear_bit_recursive(resource_t * rsc, unsigned long long flag); + + resource_t * rsc_contains_remote_node(pe_working_set_t * data_set, resource_t *rsc); + #endif +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index b74bf83..ddf9dd7 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -60,6 +60,7 @@ enum pe_find { + + # define pe_flag_stonith_enabled 0x00000010ULL + # define pe_flag_have_stonith_resource 0x00000020ULL ++# define pe_flag_enable_unfencing 0x00000040ULL + + # define pe_flag_stop_rsc_orphans 0x00000100ULL + # define pe_flag_stop_action_orphans 0x00000200ULL +@@ -165,6 +166,7 @@ struct node_s { + + # define pe_rsc_notify 0x00000010ULL + # define pe_rsc_unique 0x00000020ULL ++# define pe_rsc_fence_device 0x00000040ULL + + # define pe_rsc_provisional 0x00000100ULL + # define pe_rsc_allocating 0x00000200ULL +@@ -329,6 +331,12 @@ enum pe_ordering { + + pe_order_runnable_left = 0x100, /* 'then' requires 'first' to be runnable */ + ++ pe_order_pseudo_left = 0x200, /* 'then' can only be pseudo if 'first' is runnable */ ++ pe_order_implies_then_on_node = 0x400, /* If 'first' is required on 'nodeX', ++ * ensure instances of 'then' on 'nodeX' are too. ++ * Only really useful if 'then' is a clone and 'first' is not ++ */ ++ + pe_order_restart = 0x1000, /* 'then' is runnable if 'first' is optional or runnable */ + pe_order_stonith_stop = 0x2000, /* only applies if the action is non-pseudo */ + pe_order_serialize_only = 0x4000, /* serialize */ +diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c +index c6c69e6..b13cceb 100644 +--- a/lib/pengine/clone.c ++++ b/lib/pengine/clone.c +@@ -46,7 +46,7 @@ mark_as_orphan(resource_t * rsc) + } + } + +-static void ++void + clear_bit_recursive(resource_t * rsc, unsigned long long flag) + { + GListPtr gIter = rsc->children; +diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c +index 8a6049d..8b774c0 100644 +--- a/lib/pengine/complex.c ++++ b/lib/pengine/complex.c +@@ -340,12 +340,13 @@ gboolean + common_unpack(xmlNode * xml_obj, resource_t ** rsc, + resource_t * parent, pe_working_set_t * data_set) + { ++ bool isdefault = FALSE; + xmlNode *expanded_xml = NULL; + xmlNode *ops = NULL; + resource_t *top = NULL; + const char *value = NULL; ++ const char *rclass = NULL; /* Look for this after any templates have been expanded */ + const char *id = crm_element_value(xml_obj, XML_ATTR_ID); +- const char *class = crm_element_value(xml_obj, XML_AGENT_ATTR_CLASS); + + crm_log_xml_trace(xml_obj, "Processing resource input..."); + +@@ -375,6 +376,8 @@ common_unpack(xmlNode * xml_obj, resource_t ** rsc, + (*rsc)->orig_xml = NULL; + } + ++ /* Do not use xml_obj from here on, use (*rsc)->xml in case templates are involved */ ++ rclass = crm_element_value((*rsc)->xml, XML_AGENT_ATTR_CLASS); + (*rsc)->parent = parent; + + ops = find_xml_node((*rsc)->xml, "operations", FALSE); +@@ -538,47 +541,67 @@ common_unpack(xmlNode * xml_obj, resource_t ** rsc, + } + } + ++ if (safe_str_eq(rclass, "stonith")) { ++ set_bit(data_set->flags, pe_flag_have_stonith_resource); ++ set_bit((*rsc)->flags, pe_rsc_fence_device); ++ } ++ + value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_REQUIRES); ++ ++ handle_requires_pref: + if (safe_str_eq(value, "nothing")) { + + } else if (safe_str_eq(value, "quorum")) { + set_bit((*rsc)->flags, pe_rsc_needs_quorum); + + } else if (safe_str_eq(value, "unfencing")) { +- set_bit((*rsc)->flags, pe_rsc_needs_fencing); +- set_bit((*rsc)->flags, pe_rsc_needs_unfencing); +- if (is_set(data_set->flags, pe_flag_stonith_enabled)) { +- crm_notice("%s requires (un)fencing but fencing is disabled", (*rsc)->id); ++ if (is_set((*rsc)->flags, pe_rsc_fence_device)) { ++ crm_config_warn("%s is a fencing device but requires (un)fencing", (*rsc)->id); ++ value = "quorum"; ++ isdefault = TRUE; ++ goto handle_requires_pref; ++ ++ } else if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) { ++ crm_config_warn("%s requires (un)fencing but fencing is disabled", (*rsc)->id); ++ value = "quorum"; ++ isdefault = TRUE; ++ goto handle_requires_pref; ++ ++ } else { ++ set_bit((*rsc)->flags, pe_rsc_needs_fencing); ++ set_bit((*rsc)->flags, pe_rsc_needs_unfencing); + } + + } else if (safe_str_eq(value, "fencing")) { + set_bit((*rsc)->flags, pe_rsc_needs_fencing); + if (is_set(data_set->flags, pe_flag_stonith_enabled)) { +- crm_notice("%s requires fencing but fencing is disabled", (*rsc)->id); ++ crm_config_warn("%s requires (un)fencing but fencing is disabled", (*rsc)->id); + } + + } else { + if (value) { + crm_config_err("Invalid value for %s->requires: %s%s", + (*rsc)->id, value, +- is_set(data_set->flags, +- pe_flag_stonith_enabled) ? "" : " (stonith-enabled=false)"); ++ is_set(data_set->flags, pe_flag_stonith_enabled) ? "" : " (stonith-enabled=false)"); + } + +- if (is_set(data_set->flags, pe_flag_stonith_enabled)) { +- set_bit((*rsc)->flags, pe_rsc_needs_fencing); +- value = "fencing (default)"; ++ isdefault = TRUE; ++ if (is_set(data_set->flags, pe_flag_enable_unfencing)) { ++ value = "unfencing"; ++ ++ } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) { ++ value = "fencing"; + + } else if (data_set->no_quorum_policy == no_quorum_ignore) { +- value = "nothing (default)"; ++ value = "nothing"; + + } else { +- set_bit((*rsc)->flags, pe_rsc_needs_quorum); +- value = "quorum (default)"; ++ value = "quorum"; + } ++ goto handle_requires_pref; + } + +- pe_rsc_trace((*rsc), "\tRequired to start: %s", value); ++ pe_rsc_trace((*rsc), "\tRequired to start: %s%s", value, isdefault?" (default)":""); + + value = g_hash_table_lookup((*rsc)->meta, XML_RSC_ATTR_FAIL_TIMEOUT); + if (value != NULL) { +@@ -606,10 +629,6 @@ common_unpack(xmlNode * xml_obj, resource_t ** rsc, + pe_rsc_trace((*rsc), "\tAction notification: %s", + is_set((*rsc)->flags, pe_rsc_notify) ? "required" : "not required"); + +- if (safe_str_eq(class, "stonith")) { +- set_bit(data_set->flags, pe_flag_have_stonith_resource); +- } +- + (*rsc)->utilization = + g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 56ed291..39ded96 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -77,6 +77,35 @@ unpack_config(xmlNode * config, pe_working_set_t * data_set) + GHashTable *config_hash = + g_hash_table_new_full(crm_str_hash, g_str_equal, g_hash_destroy_str, g_hash_destroy_str); + ++ xmlXPathObjectPtr xpathObj = NULL; ++ ++ if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) { ++ xpathObj = xpath_search(data_set->input, "//nvpair[@name='provides' and @value='unfencing']"); ++ if(xpathObj && numXpathResults(xpathObj) > 0) { ++ set_bit(data_set->flags, pe_flag_enable_unfencing); ++ } ++ freeXpathObject(xpathObj); ++ } ++ ++ if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) { ++ xpathObj = xpath_search(data_set->input, "//nvpair[@name='requires' and @value='unfencing']"); ++ if(xpathObj && numXpathResults(xpathObj) > 0) { ++ set_bit(data_set->flags, pe_flag_enable_unfencing); ++ } ++ freeXpathObject(xpathObj); ++ } ++ ++ ++#ifdef REDHAT_COMPAT_6 ++ if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) { ++ xpathObj = xpath_search(data_set->input, "//primitive[@type='fence_scsi']"); ++ if(xpathObj && numXpathResults(xpathObj) > 0) { ++ set_bit(data_set->flags, pe_flag_enable_unfencing); ++ } ++ freeXpathObject(xpathObj); ++ } ++#endif ++ + data_set->config_hash = config_hash; + + unpack_instance_attributes(data_set->input, config, XML_CIB_TAG_PROPSET, NULL, config_hash, +diff --git a/pengine/allocate.c b/pengine/allocate.c +index b9ce069..88258f4 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -123,6 +123,7 @@ check_rsc_parameters(resource_t * rsc, node_t * node, xmlNode * rsc_entry, + } + + changed = TRUE; ++ trigger_unfencing(rsc, node, "Device definition changed", NULL, data_set); + if (active_here) { + force_restart = TRUE; + crm_notice("Forcing restart of %s on %s, %s changed: %s -> %s", +@@ -169,6 +170,7 @@ CancelXmlOp(resource_t * rsc, xmlNode * xml_op, node_t * active_node, + crm_info("Action %s on %s will be stopped: %s", + key, active_node->details->uname, reason ? reason : "unknown"); + ++ /* TODO: This looks highly dangerous if we ever try to schedule 'key' too */ + cancel = custom_action(rsc, strdup(key), RSC_CANCEL, active_node, FALSE, TRUE, data_set); + + free(cancel->task); +@@ -227,6 +229,7 @@ check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op + key = NULL; + } + ++ crm_trace("Testing %s_%s_%d on %s", rsc->id, task, interval, active_node?active_node->details->uname:"N/A"); + if (interval == 0 && safe_str_eq(task, RSC_STATUS)) { + /* Reload based on the start action not a probe */ + task = RSC_START; +@@ -252,6 +255,7 @@ check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op + op_version, crm_element_value(xml_op, XML_ATTR_TRANSITION_MAGIC)); + + custom_action(rsc, key, task, NULL, FALSE, TRUE, data_set); ++ trigger_unfencing(rsc, NULL, "Device parameters changed", NULL, data_set); + + } else if ((digest_data->rc == RSC_DIGEST_ALL) || (digest_data->rc == RSC_DIGEST_UNKNOWN)) { + /* Changes that can potentially be handled by a reload */ +@@ -259,6 +263,7 @@ check_action_definition(resource_t * rsc, node_t * active_node, xmlNode * xml_op + const char *digest_all = crm_element_value(xml_op, XML_LRM_ATTR_OP_DIGEST); + + did_change = TRUE; ++ trigger_unfencing(rsc, NULL, "Device parameters changed (reload)", NULL, data_set); + crm_log_xml_info(digest_data->params_all, "params:reload"); + key = generate_op_key(rsc->id, task, interval); + pe_rsc_info(rsc, "Parameters to %s on %s changed: was %s vs. now %s (reload:%s) %s", +@@ -1262,8 +1267,40 @@ any_managed_resources(pe_working_set_t * data_set) + return FALSE; + } + ++void ++trigger_unfencing( ++ resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set) ++{ ++ if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) { ++ /* No resources require it */ ++ return; ++ ++ } else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) { ++ /* Wasnt a stonith device */ ++ return; ++ ++ } else if(node) { ++ action_t *unfence = pe_fence_op(node, "on", FALSE, data_set); ++ ++ crm_notice("Unfencing %s: %s", node->details->uname, reason); ++ if(FALSE && dependancy) { ++ order_actions(dependancy, unfence, pe_order_optional); ++ } ++ ++ } else if(rsc) { ++ GHashTableIter iter; ++ ++ g_hash_table_iter_init(&iter, rsc->allowed_nodes); ++ while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { ++ if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) { ++ trigger_unfencing(rsc, node, reason, dependancy, data_set); ++ } ++ } ++ } ++} ++ + action_t * +-pe_fence_op(node_t * node, const char *op, pe_working_set_t * data_set) ++pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set) + { + char *key = NULL; + action_t *stonith_op = NULL; +@@ -1279,13 +1316,18 @@ pe_fence_op(node_t * node, const char *op, pe_working_set_t * data_set) + } + + if(stonith_op == NULL) { +- stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, FALSE, TRUE, data_set); ++ stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, optional, TRUE, data_set); + + add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname); + add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id); + add_hash_param(stonith_op->meta, "stonith_action", op); + } + ++ if(optional == FALSE) { ++ crm_trace("%s is no longer optional", stonith_op->uuid); ++ pe_clear_action_bit(stonith_op, pe_action_optional); ++ } ++ + return stonith_op; + } + +@@ -1323,7 +1365,7 @@ stage6(pe_working_set_t * data_set) + if (need_stonith && node->details->unclean && pe_can_fence(data_set, node)) { + pe_warn("Scheduling Node %s for STONITH", node->details->uname); + +- stonith_op = pe_fence_op(node, NULL, data_set); ++ stonith_op = pe_fence_op(node, NULL, FALSE, data_set); + + stonith_constraints(node, stonith_op, data_set); + +diff --git a/pengine/allocate.h b/pengine/allocate.h +index 042d3aa..712de60 100644 +--- a/pengine/allocate.h ++++ b/pengine/allocate.h +@@ -51,7 +51,7 @@ struct resource_alloc_functions_s { + void (*append_meta) (resource_t * rsc, xmlNode * xml); + }; + +-action_t *pe_fence_op(node_t * node, const char *op, pe_working_set_t * data_set); ++action_t *pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set); + + extern GHashTable *rsc_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes, + const char *attr, float factor, enum pe_weights flags); +@@ -161,6 +161,8 @@ extern void rsc_migrate_reload(resource_t * rsc, pe_working_set_t * data_set); + extern void rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, + pe_working_set_t * data_set); + ++void trigger_unfencing(resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set); ++ + extern enum pe_graph_flags native_update_actions(action_t * first, action_t * then, node_t * node, + enum pe_action_flags flags, + enum pe_action_flags filter, +diff --git a/pengine/graph.c b/pengine/graph.c +index 779f0e0..9c82d23 100644 +--- a/pengine/graph.c ++++ b/pengine/graph.c +@@ -430,11 +430,49 @@ update_action(action_t * then) + uname : "", other->type); + + if (first == other->action) { ++ /* ++ * 'first' was not expanded (ie. from 'start' to 'running'), which could mean it: ++ * - has no associated resource, ++ * - was a primitive, ++ * - was pre-expanded (ie. 'running' instead of 'start') ++ * ++ * The third argument here to graph_update_action() is a node which is used under two conditions: ++ * - Interleaving, in which case first->node and ++ * then->node are equal (and NULL) ++ * - If 'then' is a clone, to limit the scope of the ++ * constraint to instances on the supplied node ++ * ++ */ ++ int otype = other->type; ++ node_t *node = then->node; ++ ++ if(is_set(otype, pe_order_implies_then_on_node)) { ++ /* Normally we want the _whole_ 'then' clone to ++ * restart if 'first' is restarted, so then->node is ++ * needed. ++ * ++ * However for unfencing, we want to limit this to ++ * instances on the same node as 'first' (the ++ * unfencing operation), so first->node is supplied. ++ * ++ * Swap the node, from then on we can can treat it ++ * like any other 'pe_order_implies_then' ++ */ ++ ++ clear_bit(otype, pe_order_implies_then_on_node); ++ set_bit(otype, pe_order_implies_then); ++ node = first->node; ++ } + clear_bit(first_flags, pe_action_pseudo); +- changed |= graph_update_action(first, then, then->node, first_flags, other->type); ++ changed |= graph_update_action(first, then, node, first_flags, otype); + ++ /* 'first' was for a complex resource (clone, group, etc), ++ * create a new dependancy if necessary ++ */ + } else if (order_actions(first, then, other->type)) { +- /* Start again to get the new actions_before list */ ++ /* This was the first time 'first' and 'then' were associated, ++ * start again to get the new actions_before list ++ */ + changed |= (pe_graph_updated_then | pe_graph_disable); + } + +diff --git a/pengine/native.c b/pengine/native.c +index 4016a3c..d0eb950 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -1236,9 +1236,7 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + + resource_t *top = uber_parent(rsc); + int type = pe_order_optional | pe_order_implies_then | pe_order_restart; +- gboolean is_stonith = +- (rsc->xml && safe_str_eq(crm_element_value(rsc->xml, XML_AGENT_ATTR_CLASS), "stonith")) ? +- TRUE : FALSE; ++ gboolean is_stonith = is_set(rsc->flags, pe_rsc_fence_device); + + custom_action_order(rsc, generate_op_key(rsc->id, RSC_STOP, 0), NULL, + rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, type, data_set); +@@ -1253,6 +1251,34 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + pe_order_runnable_left, data_set); + } + ++ if (is_stonith == FALSE ++ && is_set(data_set->flags, pe_flag_enable_unfencing) ++ && is_set(rsc->flags, pe_rsc_needs_unfencing)) { ++ /* Check if the node needs to be unfenced first */ ++ node_t *node = NULL; ++ GHashTableIter iter; ++ ++ if(rsc != top) { ++ /* Only create these constraints once, rsc is almost certainly cloned */ ++ clear_bit_recursive(top, pe_rsc_needs_unfencing); ++ } ++ ++ g_hash_table_iter_init(&iter, rsc->allowed_nodes); ++ while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { ++ action_t *unfence = pe_fence_op(node, "on", TRUE, data_set); ++ ++ custom_action_order(top, generate_op_key(top->id, top == rsc?RSC_STOP:RSC_STOPPED, 0), NULL, ++ NULL, strdup(unfence->uuid), unfence, ++ pe_order_optional, data_set); ++ ++ crm_debug("Stopping %s prior to unfencing %s", top->id, unfence->uuid); ++ ++ custom_action_order(NULL, strdup(unfence->uuid), unfence, ++ top, generate_op_key(top->id, RSC_START, 0), NULL, ++ pe_order_implies_then_on_node, data_set); ++ } ++ } ++ + if (is_not_set(rsc->flags, pe_rsc_managed)) { + pe_rsc_trace(rsc, "Skipping fencing constraints for unmanaged resource: %s", rsc->id); + return; +@@ -2456,20 +2482,9 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set); + update_action_flags(probe, pe_action_optional | pe_action_clear); + +- /* Check if the node needs to be unfenced first */ +- if (is_set(rsc->flags, pe_rsc_needs_unfencing)) { +- action_t *unfence = pe_fence_op(node, "on", data_set); +- +- crm_notice("Unfencing %s for %s", node->details->uname, rsc->id); +- order_actions(unfence, probe, pe_order_implies_then); +- +- /* The lack of ordering constraints on STONITH_UP would +- * traditionally mean unfencing is initiated /before/ the +- * devices are started. +- * +- * However this is a non-issue as stonithd is now smart +- * enough to be able to use devices directly from the cib +- */ ++ if(is_set(rsc->flags, pe_rsc_fence_device)) { ++ crm_crit("TODO: %s: Trigger when probing the device or the resource that needs unfencing?", rsc->id); ++ trigger_unfencing(NULL, node, "node discovery", probe, data_set); + } + + /* +diff --git a/pengine/regression.sh b/pengine/regression.sh +index aa50854..9cb6c18 100755 +--- a/pengine/regression.sh ++++ b/pengine/regression.sh +@@ -317,6 +317,11 @@ do_test bug-cl-5170 "Prevent clone from starting with on-fail=block" + do_test clone-fail-block-colocation "Move colocated group when failed clone has on-fail=block" + + echo "" ++do_test unfence-startup "Clean unfencing" ++do_test unfence-definition "Unfencing when the agent changes" ++do_test unfence-parameters "Unfencing when the agent parameters changes" ++ ++echo "" + do_test master-0 "Stopped -> Slave" + do_test master-1 "Stopped -> Promote" + do_test master-2 "Stopped -> Promote : notify" diff --git a/SOURCES/bz1078078-pcmk-pe_delay_unfencing_until_after_we_know_the_state_of_all_resources_that_require_unfencing.patch b/SOURCES/bz1078078-pcmk-pe_delay_unfencing_until_after_we_know_the_state_of_all_resources_that_require_unfencing.patch new file mode 100644 index 0000000..3503c88 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_delay_unfencing_until_after_we_know_the_state_of_all_resources_that_require_unfencing.patch @@ -0,0 +1,2251 @@ +commit cf7988cb02e96f7a278569dcee410691cb7b85e0 +Author: Andrew Beekhof +Date: Fri Apr 4 11:59:13 2014 +1100 + + Fix: PE: Delay unfencing until after we know the state of all resources that require unfencing + + (cherry picked from commit 22666466bd9e50efdd8efb6c1775c71785c900b1) + + Conflicts: + include/crm/pengine/internal.h + include/crm/pengine/status.h + lib/pengine/unpack.c + lib/pengine/utils.c + pengine/test10/unfence-definition.dot + pengine/test10/unfence-definition.exp + pengine/test10/unfence-definition.summary + pengine/test10/unfence-parameters.dot + pengine/test10/unfence-parameters.exp + pengine/test10/unfence-parameters.summary + pengine/test10/unfence-startup.dot + pengine/test10/unfence-startup.exp + pengine/test10/unfence-startup.xml + +diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h +index 657c647..b4907c1 100644 +--- a/include/crm/pengine/internal.h ++++ b/include/crm/pengine/internal.h +@@ -260,8 +260,13 @@ typedef struct op_digest_cache_s { + op_digest_cache_t *rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node, + pe_working_set_t * data_set); + +-gboolean is_remote_node(xmlNode *xml); ++action_t *pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set); ++void trigger_unfencing( ++ resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set); ++ ++void set_bit_recursive(resource_t * rsc, unsigned long long flag); + void clear_bit_recursive(resource_t * rsc, unsigned long long flag); + ++gboolean is_remote_node(xmlNode *xml); + resource_t * rsc_contains_remote_node(pe_working_set_t * data_set, resource_t *rsc); + #endif +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index ddf9dd7..6542737 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -191,6 +191,7 @@ struct node_s { + # define pe_rsc_needs_quorum 0x10000000ULL + # define pe_rsc_needs_fencing 0x20000000ULL + # define pe_rsc_needs_unfencing 0x40000000ULL ++# define pe_rsc_have_unfencing 0x80000000ULL + + enum pe_graph_flags { + pe_graph_none = 0x00000, +diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c +index b13cceb..90d7eca 100644 +--- a/lib/pengine/clone.c ++++ b/lib/pengine/clone.c +@@ -47,19 +47,6 @@ mark_as_orphan(resource_t * rsc) + } + + void +-clear_bit_recursive(resource_t * rsc, unsigned long long flag) +-{ +- GListPtr gIter = rsc->children; +- +- clear_bit(rsc->flags, flag); +- for (; gIter != NULL; gIter = gIter->next) { +- resource_t *child_rsc = (resource_t *) gIter->data; +- +- clear_bit_recursive(child_rsc, flag); +- } +-} +- +-void + force_non_unique_clone(resource_t * rsc, const char *rid, pe_working_set_t * data_set) + { + if (rsc->variant == pe_clone || rsc->variant == pe_master) { +diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c +index 0f20481..a3133e7 100644 +--- a/lib/pengine/complex.c ++++ b/lib/pengine/complex.c +@@ -575,7 +575,7 @@ common_unpack(xmlNode * xml_obj, resource_t ** rsc, + } else if (safe_str_eq(value, "fencing")) { + set_bit((*rsc)->flags, pe_rsc_needs_fencing); + if (is_set(data_set->flags, pe_flag_stonith_enabled)) { +- crm_config_warn("%s requires (un)fencing but fencing is disabled", (*rsc)->id); ++ crm_config_warn("%s requires fencing but fencing is disabled", (*rsc)->id); + } + + } else { +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index 39ded96..eb10bd8 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -2325,6 +2325,31 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, + * didnt include target_rc and liked to remap status + */ + switch (actual_rc_i) { ++ case PCMK_OCF_OK: ++ if (is_probe && target_rc == 7) { ++ task_status_i = PCMK_LRM_OP_DONE; ++ set_bit(rsc->flags, pe_rsc_unexpectedly_running); ++ ++ if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { ++ /* _Require_ unfencing after probing resources that need unfencing and was found active */ ++ crm_warn("Operation %s found resource %s active on %s: triggering unfencing", ++ task, rsc->id, node->details->uname); ++ trigger_unfencing(NULL, node, "Unexpected state", NULL, data_set); ++ ++ } else { ++ pe_rsc_info(rsc, "Operation %s found resource %s active on %s", ++ task, rsc->id, node->details->uname); ++ } ++ ++ /* legacy code for pre-0.6.5 operations */ ++ } else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) { ++ /* catch status ops that return 0 instead of 8 while they ++ * are supposed to be in master mode ++ */ ++ task_status_i = PCMK_LRM_OP_ERROR; ++ } ++ break; ++ + case PCMK_OCF_NOT_RUNNING: + if (is_probe || target_rc == actual_rc_i) { + task_status_i = PCMK_LRM_OP_DONE; +@@ -2415,23 +2440,6 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, + } + break; + +- case PCMK_OCF_OK: +- if (is_probe && target_rc == 7) { +- task_status_i = PCMK_LRM_OP_DONE; +- set_bit(rsc->flags, pe_rsc_unexpectedly_running); +- pe_rsc_info(rsc, "Operation %s found resource %s active on %s", +- task, rsc->id, node->details->uname); +- +- /* legacy code for pre-0.6.5 operations */ +- } else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) { +- /* catch status ops that return 0 instead of 8 while they +- * are supposed to be in master mode +- */ +- task_status_i = PCMK_LRM_OP_ERROR; +- } +- +- break; +- + default: + if (task_status_i == PCMK_LRM_OP_DONE) { + crm_info("Remapping %s (rc=%d) on %s to an ERROR", +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 2520eeb..e84d341 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1807,3 +1807,92 @@ rsc_contains_remote_node(pe_working_set_t * data_set, resource_t *rsc) + return NULL; + } + ++void ++clear_bit_recursive(resource_t * rsc, unsigned long long flag) ++{ ++ GListPtr gIter = rsc->children; ++ ++ clear_bit(rsc->flags, flag); ++ for (; gIter != NULL; gIter = gIter->next) { ++ resource_t *child_rsc = (resource_t *) gIter->data; ++ ++ clear_bit_recursive(child_rsc, flag); ++ } ++} ++ ++void ++set_bit_recursive(resource_t * rsc, unsigned long long flag) ++{ ++ GListPtr gIter = rsc->children; ++ ++ set_bit(rsc->flags, flag); ++ for (; gIter != NULL; gIter = gIter->next) { ++ resource_t *child_rsc = (resource_t *) gIter->data; ++ ++ set_bit_recursive(child_rsc, flag); ++ } ++} ++ ++action_t * ++pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set) ++{ ++ char *key = NULL; ++ action_t *stonith_op = NULL; ++ ++ if(op == NULL) { ++ op = data_set->stonith_action; ++ } ++ ++ key = g_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op); ++ ++ if(data_set->singletons) { ++ stonith_op = g_hash_table_lookup(data_set->singletons, key); ++ } ++ ++ if(stonith_op == NULL) { ++ stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, optional, TRUE, data_set); ++ ++ add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname); ++ add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id); ++ add_hash_param(stonith_op->meta, "stonith_action", op); ++ } ++ ++ if(optional == FALSE) { ++ crm_trace("%s is no longer optional", stonith_op->uuid); ++ pe_clear_action_bit(stonith_op, pe_action_optional); ++ } ++ ++ return stonith_op; ++} ++ ++void ++trigger_unfencing( ++ resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set) ++{ ++ if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) { ++ /* No resources require it */ ++ return; ++ ++ } else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) { ++ /* Wasnt a stonith device */ ++ return; ++ ++ } else if(node) { ++ action_t *unfence = pe_fence_op(node, "on", FALSE, data_set); ++ ++ crm_notice("Unfencing %s: %s", node->details->uname, reason); ++ if(FALSE && dependancy) { ++ order_actions(unfence, dependancy, pe_order_optional); ++ } ++ ++ } else if(rsc) { ++ GHashTableIter iter; ++ ++ g_hash_table_iter_init(&iter, rsc->allowed_nodes); ++ while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { ++ if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) { ++ trigger_unfencing(rsc, node, reason, dependancy, data_set); ++ } ++ } ++ } ++} +diff --git a/pengine/allocate.c b/pengine/allocate.c +index 88258f4..3480eb9 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1267,70 +1267,6 @@ any_managed_resources(pe_working_set_t * data_set) + return FALSE; + } + +-void +-trigger_unfencing( +- resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set) +-{ +- if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) { +- /* No resources require it */ +- return; +- +- } else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) { +- /* Wasnt a stonith device */ +- return; +- +- } else if(node) { +- action_t *unfence = pe_fence_op(node, "on", FALSE, data_set); +- +- crm_notice("Unfencing %s: %s", node->details->uname, reason); +- if(FALSE && dependancy) { +- order_actions(dependancy, unfence, pe_order_optional); +- } +- +- } else if(rsc) { +- GHashTableIter iter; +- +- g_hash_table_iter_init(&iter, rsc->allowed_nodes); +- while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) { +- if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) { +- trigger_unfencing(rsc, node, reason, dependancy, data_set); +- } +- } +- } +-} +- +-action_t * +-pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set) +-{ +- char *key = NULL; +- action_t *stonith_op = NULL; +- +- if(op == NULL) { +- op = data_set->stonith_action; +- } +- +- key = g_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op); +- +- if(data_set->singletons) { +- stonith_op = g_hash_table_lookup(data_set->singletons, key); +- } +- +- if(stonith_op == NULL) { +- stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, optional, TRUE, data_set); +- +- add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname); +- add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id); +- add_hash_param(stonith_op->meta, "stonith_action", op); +- } +- +- if(optional == FALSE) { +- crm_trace("%s is no longer optional", stonith_op->uuid); +- pe_clear_action_bit(stonith_op, pe_action_optional); +- } +- +- return stonith_op; +-} +- + /* + * Create dependancies for stonith and shutdown operations + */ +diff --git a/pengine/allocate.h b/pengine/allocate.h +index 712de60..0a352fd 100644 +--- a/pengine/allocate.h ++++ b/pengine/allocate.h +@@ -51,8 +51,6 @@ struct resource_alloc_functions_s { + void (*append_meta) (resource_t * rsc, xmlNode * xml); + }; + +-action_t *pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set); +- + extern GHashTable *rsc_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes, + const char *attr, float factor, enum pe_weights flags); + +@@ -161,8 +159,6 @@ extern void rsc_migrate_reload(resource_t * rsc, pe_working_set_t * data_set); + extern void rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, + pe_working_set_t * data_set); + +-void trigger_unfencing(resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set); +- + extern enum pe_graph_flags native_update_actions(action_t * first, action_t * then, node_t * node, + enum pe_action_flags flags, + enum pe_action_flags filter, +diff --git a/pengine/native.c b/pengine/native.c +index d0eb950..3d75dc9 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -1253,14 +1253,15 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set) + + if (is_stonith == FALSE + && is_set(data_set->flags, pe_flag_enable_unfencing) +- && is_set(rsc->flags, pe_rsc_needs_unfencing)) { ++ && is_set(rsc->flags, pe_rsc_needs_unfencing) ++ && is_not_set(rsc->flags, pe_rsc_have_unfencing)) { + /* Check if the node needs to be unfenced first */ + node_t *node = NULL; + GHashTableIter iter; + + if(rsc != top) { + /* Only create these constraints once, rsc is almost certainly cloned */ +- clear_bit_recursive(top, pe_rsc_needs_unfencing); ++ clear_bit_recursive(top, pe_rsc_have_unfencing); + } + + g_hash_table_iter_init(&iter, rsc->allowed_nodes); +@@ -2482,9 +2483,26 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set); + update_action_flags(probe, pe_action_optional | pe_action_clear); + +- if(is_set(rsc->flags, pe_rsc_fence_device)) { +- crm_crit("TODO: %s: Trigger when probing the device or the resource that needs unfencing?", rsc->id); ++ /* If enabled, require unfencing before probing any fence devices ++ * but ensure it happens after any resources that require ++ * unfencing have been probed. ++ * ++ * Doing it the other way (requiring unfencing after probing ++ * resources that need it) would result in the node being ++ * unfenced, and all its resources being stopped, whenever a new ++ * resource is added. Which would be highly suboptimal. ++ * ++ * So essentially, at the point the fencing device(s) have been ++ * probed, we know the state of all resources that require ++ * unfencing and that unfencing occurred. ++ */ ++ if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) { + trigger_unfencing(NULL, node, "node discovery", probe, data_set); ++ ++ } else if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { ++ action_t *unfence = pe_fence_op(node, "on", TRUE, data_set); ++ ++ order_actions(probe, unfence, pe_order_optional); + } + + /* +diff --git a/pengine/test10/unfence-definition.dot b/pengine/test10/unfence-definition.dot +new file mode 100644 +index 0000000..e737687 +--- /dev/null ++++ b/pengine/test10/unfence-definition.dot +@@ -0,0 +1,102 @@ ++digraph "g" { ++"all_stopped" -> "fencing_start_0 virt-1" [ style = bold] ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_start_0" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd-clone_start_0" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"clvmd-clone_start_0" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"clvmd-clone_start_0" -> "clvmd_start_0 virt-1" [ style = bold] ++"clvmd-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_stop_0" -> "clvmd-clone_stopped_0" [ style = bold] ++"clvmd-clone_stop_0" -> "clvmd_stop_0 virt-1" [ style = bold] ++"clvmd-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_stopped_0" -> "clvmd-clone_start_0" [ style = bold] ++"clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] ++"clvmd-clone_stopped_0" -> "stonith 'on' virt-1" [ style = bold] ++"clvmd-clone_stopped_0" -> "stonith 'on' virt-3" [ style = bold] ++"clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd:1_monitor_0 virt-2" -> "probe_complete virt-2" [ style = bold] ++"clvmd:1_monitor_0 virt-2" [ style=bold color="green" fontcolor="black"] ++"clvmd:1_start_0 virt-2" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd:1_start_0 virt-2" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"clvmd:1_start_0 virt-2" [ style=bold color="green" fontcolor="black"] ++"clvmd:2_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"clvmd:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] ++"clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"clvmd_start_0 virt-1" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd_start_0 virt-1" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"clvmd_start_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"clvmd_stop_0 virt-1" -> "all_stopped" [ style = bold] ++"clvmd_stop_0 virt-1" -> "clvmd-clone_stopped_0" [ style = bold] ++"clvmd_stop_0 virt-1" -> "clvmd_start_0 virt-1" [ style = bold] ++"clvmd_stop_0 virt-1" -> "dlm_stop_0 virt-1" [ style = bold] ++"clvmd_stop_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"dlm-clone_running_0" -> "clvmd-clone_start_0" [ style = bold] ++"dlm-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"dlm-clone_start_0" -> "dlm-clone_running_0" [ style = bold] ++"dlm-clone_start_0" -> "dlm:2_start_0 virt-3" [ style = bold] ++"dlm-clone_start_0" -> "dlm_start_0 virt-1" [ style = bold] ++"dlm-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"dlm-clone_stop_0" -> "dlm-clone_stopped_0" [ style = bold] ++"dlm-clone_stop_0" -> "dlm_stop_0 virt-1" [ style = bold] ++"dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"dlm-clone_stopped_0" -> "dlm-clone_start_0" [ style = bold] ++"dlm-clone_stopped_0" -> "stonith 'on' virt-1" [ style = bold] ++"dlm-clone_stopped_0" -> "stonith 'on' virt-3" [ style = bold] ++"dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"dlm:2_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"dlm:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] ++"dlm:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold] ++"dlm:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"dlm_start_0 virt-1" -> "clvmd_start_0 virt-1" [ style = bold] ++"dlm_start_0 virt-1" -> "dlm-clone_running_0" [ style = bold] ++"dlm_start_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"dlm_stop_0 virt-1" -> "all_stopped" [ style = bold] ++"dlm_stop_0 virt-1" -> "dlm-clone_stopped_0" [ style = bold] ++"dlm_stop_0 virt-1" -> "dlm_start_0 virt-1" [ style = bold] ++"dlm_stop_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"fencing_delete_0 virt-1" -> "fencing_start_0 virt-1" [ style = bold] ++"fencing_delete_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"fencing_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"fencing_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"fencing_start_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"fencing_stop_0 virt-1" -> "all_stopped" [ style = bold] ++"fencing_stop_0 virt-1" -> "fencing_delete_0 virt-1" [ style = bold] ++"fencing_stop_0 virt-1" -> "fencing_start_0 virt-1" [ style = bold] ++"fencing_stop_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"probe_complete virt-1" -> "probe_complete" [ style = bold] ++"probe_complete virt-1" [ style=bold color="green" fontcolor="black"] ++"probe_complete virt-2" -> "probe_complete" [ style = bold] ++"probe_complete virt-2" [ style=bold color="green" fontcolor="black"] ++"probe_complete virt-3" -> "probe_complete" [ style = bold] ++"probe_complete virt-3" [ style=bold color="green" fontcolor="black"] ++"probe_complete" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"probe_complete" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"probe_complete" -> "clvmd_start_0 virt-1" [ style = bold] ++"probe_complete" -> "clvmd_stop_0 virt-1" [ style = bold] ++"probe_complete" -> "dlm:2_start_0 virt-3" [ style = bold] ++"probe_complete" -> "dlm_start_0 virt-1" [ style = bold] ++"probe_complete" -> "dlm_stop_0 virt-1" [ style = bold] ++"probe_complete" -> "fencing_start_0 virt-1" [ style = bold] ++"probe_complete" -> "fencing_stop_0 virt-1" [ style = bold] ++"probe_complete" [ style=bold color="green" fontcolor="orange"] ++"stonith 'on' virt-1" -> "clvmd-clone_start_0" [ style = bold] ++"stonith 'on' virt-1" -> "dlm-clone_start_0" [ style = bold] ++"stonith 'on' virt-1" [ style=bold color="green" fontcolor="black"] ++"stonith 'on' virt-3" -> "clvmd-clone_start_0" [ style = bold] ++"stonith 'on' virt-3" -> "dlm-clone_start_0" [ style = bold] ++"stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"stonith_complete" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"stonith_complete" -> "clvmd_start_0 virt-1" [ style = bold] ++"stonith_complete" -> "dlm:2_start_0 virt-3" [ style = bold] ++"stonith_complete" -> "dlm_start_0 virt-1" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] ++} +diff --git a/pengine/test10/unfence-definition.exp b/pengine/test10/unfence-definition.exp +new file mode 100644 +index 0000000..9075347 +--- /dev/null ++++ b/pengine/test10/unfence-definition.exp +@@ -0,0 +1,495 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary +new file mode 100644 +index 0000000..c954c4b +--- /dev/null ++++ b/pengine/test10/unfence-definition.summary +@@ -0,0 +1,63 @@ ++ ++Current cluster status: ++Node virt-4 (4): UNCLEAN (offline) ++Online: [ virt-1 virt-2 virt-3 ] ++ ++ fencing (stonith:fence_scsi): Started virt-1 ++ Clone Set: dlm-clone [dlm] ++ Started: [ virt-1 virt-2 ] ++ Stopped: [ virt-3 virt-4 ] ++ Clone Set: clvmd-clone [clvmd] ++ Started: [ virt-1 ] ++ Stopped: [ virt-2 virt-3 virt-4 ] ++ ++Transition Summary: ++ * Restart fencing (Started virt-1) ++ * Restart dlm:0 (Started virt-1) ++ * Start dlm:2 (virt-3) ++ * Restart clvmd:0 (Started virt-1) ++ * Start clvmd:1 (virt-2) ++ * Start clvmd:2 (virt-3) ++ ++Executing cluster transition: ++ * Resource action: fencing monitor on virt-3 ++ * Resource action: dlm monitor on virt-3 ++ * Resource action: clvmd monitor on virt-2 ++ * Resource action: clvmd monitor on virt-3 ++ * Pseudo action: clvmd-clone_stop_0 ++ * Fencing virt-4 (reboot) ++ * Pseudo action: stonith_complete ++ * Pseudo action: probe_complete ++ * Resource action: fencing stop on virt-1 ++ * Resource action: clvmd stop on virt-1 ++ * Pseudo action: clvmd-clone_stopped_0 ++ * Resource action: fencing delete on virt-1 ++ * Pseudo action: dlm-clone_stop_0 ++ * Resource action: dlm stop on virt-1 ++ * Pseudo action: dlm-clone_stopped_0 ++ * Fencing virt-3 (on) ++ * Pseudo action: all_stopped ++ * Fencing virt-1 (on) ++ * Resource action: fencing start on virt-1 ++ * Pseudo action: dlm-clone_start_0 ++ * Resource action: dlm start on virt-1 ++ * Resource action: dlm start on virt-3 ++ * Pseudo action: dlm-clone_running_0 ++ * Pseudo action: clvmd-clone_start_0 ++ * Resource action: clvmd start on virt-1 ++ * Resource action: clvmd start on virt-2 ++ * Resource action: clvmd start on virt-3 ++ * Pseudo action: clvmd-clone_running_0 ++ ++Revised cluster status: ++Online: [ virt-1 virt-2 virt-3 ] ++OFFLINE: [ virt-4 ] ++ ++ fencing (stonith:fence_scsi): Started virt-1 ++ Clone Set: dlm-clone [dlm] ++ Started: [ virt-1 virt-2 virt-3 ] ++ Stopped: [ virt-4 ] ++ Clone Set: clvmd-clone [clvmd] ++ Started: [ virt-1 virt-2 virt-3 ] ++ Stopped: [ virt-4 ] ++ +diff --git a/pengine/test10/unfence-parameters.dot b/pengine/test10/unfence-parameters.dot +new file mode 100644 +index 0000000..e67eb5a +--- /dev/null ++++ b/pengine/test10/unfence-parameters.dot +@@ -0,0 +1,125 @@ ++digraph "g" { ++"all_stopped" -> "fencing_start_0 virt-1" [ style = bold] ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_start_0" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd-clone_start_0" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"clvmd-clone_start_0" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"clvmd-clone_start_0" -> "clvmd_start_0 virt-1" [ style = bold] ++"clvmd-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_stop_0" -> "clvmd-clone_stopped_0" [ style = bold] ++"clvmd-clone_stop_0" -> "clvmd_stop_0 virt-1" [ style = bold] ++"clvmd-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_stopped_0" -> "clvmd-clone_start_0" [ style = bold] ++"clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold] ++"clvmd-clone_stopped_0" -> "stonith 'on' virt-1" [ style = bold] ++"clvmd-clone_stopped_0" -> "stonith 'on' virt-2" [ style = bold] ++"clvmd-clone_stopped_0" -> "stonith 'on' virt-3" [ style = bold] ++"clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd:1_monitor_0 virt-2" -> "probe_complete virt-2" [ style = bold] ++"clvmd:1_monitor_0 virt-2" -> "stonith 'on' virt-2" [ style = bold] ++"clvmd:1_monitor_0 virt-2" [ style=bold color="green" fontcolor="black"] ++"clvmd:1_start_0 virt-2" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd:1_start_0 virt-2" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"clvmd:1_start_0 virt-2" [ style=bold color="green" fontcolor="black"] ++"clvmd:2_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"clvmd:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] ++"clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd:2_start_0 virt-3" -> "clvmd:3_start_0 " [ style = dashed] ++"clvmd:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"clvmd:3_start_0 " -> "clvmd-clone_running_0" [ style = dashed] ++"clvmd:3_start_0 " [ style=dashed color="red" fontcolor="black"] ++"clvmd_start_0 virt-1" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd_start_0 virt-1" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"clvmd_start_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"clvmd_stop_0 virt-1" -> "all_stopped" [ style = bold] ++"clvmd_stop_0 virt-1" -> "clvmd-clone_stopped_0" [ style = bold] ++"clvmd_stop_0 virt-1" -> "clvmd_start_0 virt-1" [ style = bold] ++"clvmd_stop_0 virt-1" -> "dlm_stop_0 virt-1" [ style = bold] ++"clvmd_stop_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"dlm-clone_running_0" -> "clvmd-clone_start_0" [ style = bold] ++"dlm-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"dlm-clone_start_0" -> "dlm-clone_running_0" [ style = bold] ++"dlm-clone_start_0" -> "dlm:2_start_0 virt-3" [ style = bold] ++"dlm-clone_start_0" -> "dlm_start_0 virt-1" [ style = bold] ++"dlm-clone_start_0" -> "dlm_start_0 virt-2" [ style = bold] ++"dlm-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"dlm-clone_stop_0" -> "dlm-clone_stopped_0" [ style = bold] ++"dlm-clone_stop_0" -> "dlm_stop_0 virt-1" [ style = bold] ++"dlm-clone_stop_0" -> "dlm_stop_0 virt-2" [ style = bold] ++"dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"] ++"dlm-clone_stopped_0" -> "dlm-clone_start_0" [ style = bold] ++"dlm-clone_stopped_0" -> "stonith 'on' virt-1" [ style = bold] ++"dlm-clone_stopped_0" -> "stonith 'on' virt-2" [ style = bold] ++"dlm-clone_stopped_0" -> "stonith 'on' virt-3" [ style = bold] ++"dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"] ++"dlm:2_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"dlm:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] ++"dlm:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold] ++"dlm:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"dlm_start_0 virt-1" -> "clvmd_start_0 virt-1" [ style = bold] ++"dlm_start_0 virt-1" -> "dlm-clone_running_0" [ style = bold] ++"dlm_start_0 virt-1" -> "dlm_start_0 virt-2" [ style = bold] ++"dlm_start_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"dlm_start_0 virt-2" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"dlm_start_0 virt-2" -> "clvmd:3_start_0 " [ style = dashed] ++"dlm_start_0 virt-2" -> "dlm-clone_running_0" [ style = bold] ++"dlm_start_0 virt-2" -> "dlm:2_start_0 virt-3" [ style = bold] ++"dlm_start_0 virt-2" [ style=bold color="green" fontcolor="black"] ++"dlm_stop_0 virt-1" -> "all_stopped" [ style = bold] ++"dlm_stop_0 virt-1" -> "dlm-clone_stopped_0" [ style = bold] ++"dlm_stop_0 virt-1" -> "dlm_start_0 virt-1" [ style = bold] ++"dlm_stop_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"dlm_stop_0 virt-2" -> "all_stopped" [ style = bold] ++"dlm_stop_0 virt-2" -> "dlm-clone_stopped_0" [ style = bold] ++"dlm_stop_0 virt-2" -> "dlm_start_0 virt-2" [ style = bold] ++"dlm_stop_0 virt-2" -> "dlm_stop_0 virt-1" [ style = bold] ++"dlm_stop_0 virt-2" [ style=bold color="green" fontcolor="black"] ++"fencing_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"fencing_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"fencing_start_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"fencing_stop_0 virt-1" -> "all_stopped" [ style = bold] ++"fencing_stop_0 virt-1" -> "fencing_start_0 virt-1" [ style = bold] ++"fencing_stop_0 virt-1" [ style=bold color="green" fontcolor="black"] ++"probe_complete virt-1" -> "probe_complete" [ style = bold] ++"probe_complete virt-1" [ style=bold color="green" fontcolor="black"] ++"probe_complete virt-2" -> "probe_complete" [ style = bold] ++"probe_complete virt-2" [ style=bold color="green" fontcolor="black"] ++"probe_complete virt-3" -> "probe_complete" [ style = bold] ++"probe_complete virt-3" [ style=bold color="green" fontcolor="black"] ++"probe_complete" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"probe_complete" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"probe_complete" -> "clvmd:3_start_0 " [ style = dashed] ++"probe_complete" -> "clvmd_start_0 virt-1" [ style = bold] ++"probe_complete" -> "clvmd_stop_0 virt-1" [ style = bold] ++"probe_complete" -> "dlm:2_start_0 virt-3" [ style = bold] ++"probe_complete" -> "dlm_start_0 virt-1" [ style = bold] ++"probe_complete" -> "dlm_start_0 virt-2" [ style = bold] ++"probe_complete" -> "dlm_stop_0 virt-1" [ style = bold] ++"probe_complete" -> "dlm_stop_0 virt-2" [ style = bold] ++"probe_complete" -> "fencing_start_0 virt-1" [ style = bold] ++"probe_complete" -> "fencing_stop_0 virt-1" [ style = bold] ++"probe_complete" [ style=bold color="green" fontcolor="orange"] ++"stonith 'on' virt-1" -> "clvmd-clone_start_0" [ style = bold] ++"stonith 'on' virt-1" -> "dlm-clone_start_0" [ style = bold] ++"stonith 'on' virt-1" [ style=bold color="green" fontcolor="black"] ++"stonith 'on' virt-2" -> "clvmd-clone_start_0" [ style = bold] ++"stonith 'on' virt-2" -> "dlm-clone_start_0" [ style = bold] ++"stonith 'on' virt-2" [ style=bold color="green" fontcolor="black"] ++"stonith 'on' virt-3" -> "clvmd-clone_start_0" [ style = bold] ++"stonith 'on' virt-3" -> "dlm-clone_start_0" [ style = bold] ++"stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"stonith_complete" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"stonith_complete" -> "clvmd_start_0 virt-1" [ style = bold] ++"stonith_complete" -> "dlm:2_start_0 virt-3" [ style = bold] ++"stonith_complete" -> "dlm_start_0 virt-1" [ style = bold] ++"stonith_complete" -> "dlm_start_0 virt-2" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] ++} +diff --git a/pengine/test10/unfence-parameters.exp b/pengine/test10/unfence-parameters.exp +new file mode 100644 +index 0000000..491d78e +--- /dev/null ++++ b/pengine/test10/unfence-parameters.exp +@@ -0,0 +1,562 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +new file mode 100644 +index 0000000..17bbc62 +--- /dev/null ++++ b/pengine/test10/unfence-parameters.summary +@@ -0,0 +1,66 @@ ++ ++Current cluster status: ++Node virt-4 (4): UNCLEAN (offline) ++Online: [ virt-1 virt-2 virt-3 ] ++ ++ fencing (stonith:fence_scsi): Started virt-1 ++ Clone Set: dlm-clone [dlm] ++ Started: [ virt-1 virt-2 ] ++ Stopped: [ virt-3 virt-4 ] ++ Clone Set: clvmd-clone [clvmd] ++ Started: [ virt-1 ] ++ Stopped: [ virt-2 virt-3 virt-4 ] ++ ++Transition Summary: ++ * Restart fencing (Started virt-1) ++ * Restart dlm:0 (Started virt-1) ++ * Restart dlm:1 (Started virt-2) ++ * Start dlm:2 (virt-3) ++ * Restart clvmd:0 (Started virt-1) ++ * Start clvmd:1 (virt-2) ++ * Start clvmd:2 (virt-3) ++ ++Executing cluster transition: ++ * Resource action: fencing monitor on virt-3 ++ * Resource action: dlm monitor on virt-3 ++ * Resource action: clvmd monitor on virt-2 ++ * Resource action: clvmd monitor on virt-3 ++ * Pseudo action: clvmd-clone_stop_0 ++ * Fencing virt-4 (reboot) ++ * Pseudo action: stonith_complete ++ * Pseudo action: probe_complete ++ * Resource action: fencing stop on virt-1 ++ * Resource action: clvmd stop on virt-1 ++ * Pseudo action: clvmd-clone_stopped_0 ++ * Pseudo action: dlm-clone_stop_0 ++ * Resource action: dlm stop on virt-2 ++ * Resource action: dlm stop on virt-1 ++ * Pseudo action: dlm-clone_stopped_0 ++ * Fencing virt-3 (on) ++ * Fencing virt-2 (on) ++ * Pseudo action: all_stopped ++ * Fencing virt-1 (on) ++ * Resource action: fencing start on virt-1 ++ * Pseudo action: dlm-clone_start_0 ++ * Resource action: dlm start on virt-1 ++ * Resource action: dlm start on virt-2 ++ * Resource action: dlm start on virt-3 ++ * Pseudo action: dlm-clone_running_0 ++ * Pseudo action: clvmd-clone_start_0 ++ * Resource action: clvmd start on virt-1 ++ * Resource action: clvmd start on virt-2 ++ * Resource action: clvmd start on virt-3 ++ * Pseudo action: clvmd-clone_running_0 ++ ++Revised cluster status: ++Online: [ virt-1 virt-2 virt-3 ] ++OFFLINE: [ virt-4 ] ++ ++ fencing (stonith:fence_scsi): Started virt-1 ++ Clone Set: dlm-clone [dlm] ++ Started: [ virt-1 virt-2 virt-3 ] ++ Stopped: [ virt-4 ] ++ Clone Set: clvmd-clone [clvmd] ++ Started: [ virt-1 virt-2 virt-3 ] ++ Stopped: [ virt-4 ] ++ +diff --git a/pengine/test10/unfence-startup.dot b/pengine/test10/unfence-startup.dot +new file mode 100644 +index 0000000..a5cc606 +--- /dev/null ++++ b/pengine/test10/unfence-startup.dot +@@ -0,0 +1,51 @@ ++digraph "g" { ++"all_stopped" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd-clone_start_0" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd-clone_start_0" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"clvmd-clone_start_0" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"clvmd-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"clvmd:1_monitor_0 virt-2" -> "probe_complete virt-2" [ style = bold] ++"clvmd:1_monitor_0 virt-2" [ style=bold color="green" fontcolor="black"] ++"clvmd:1_start_0 virt-2" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd:1_start_0 virt-2" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"clvmd:1_start_0 virt-2" [ style=bold color="green" fontcolor="black"] ++"clvmd:2_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"clvmd:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] ++"clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold] ++"clvmd:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"dlm-clone_running_0" -> "clvmd-clone_start_0" [ style = bold] ++"dlm-clone_running_0" [ style=bold color="green" fontcolor="orange"] ++"dlm-clone_start_0" -> "dlm-clone_running_0" [ style = bold] ++"dlm-clone_start_0" -> "dlm:2_start_0 virt-3" [ style = bold] ++"dlm-clone_start_0" [ style=bold color="green" fontcolor="orange"] ++"dlm:2_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"dlm:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold] ++"dlm:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold] ++"dlm:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"fencing_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"fencing_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] ++"probe_complete virt-1" -> "probe_complete" [ style = bold] ++"probe_complete virt-1" [ style=bold color="green" fontcolor="black"] ++"probe_complete virt-2" -> "probe_complete" [ style = bold] ++"probe_complete virt-2" [ style=bold color="green" fontcolor="black"] ++"probe_complete virt-3" -> "probe_complete" [ style = bold] ++"probe_complete virt-3" [ style=bold color="green" fontcolor="black"] ++"probe_complete" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"probe_complete" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"probe_complete" -> "dlm:2_start_0 virt-3" [ style = bold] ++"probe_complete" [ style=bold color="green" fontcolor="orange"] ++"stonith 'on' virt-3" -> "clvmd-clone_start_0" [ style = bold] ++"stonith 'on' virt-3" -> "dlm-clone_start_0" [ style = bold] ++"stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"] ++"stonith_complete" -> "all_stopped" [ style = bold] ++"stonith_complete" -> "clvmd:1_start_0 virt-2" [ style = bold] ++"stonith_complete" -> "clvmd:2_start_0 virt-3" [ style = bold] ++"stonith_complete" -> "dlm:2_start_0 virt-3" [ style = bold] ++"stonith_complete" [ style=bold color="green" fontcolor="orange"] ++} +diff --git a/pengine/test10/unfence-startup.exp b/pengine/test10/unfence-startup.exp +new file mode 100644 +index 0000000..8a04d54 +--- /dev/null ++++ b/pengine/test10/unfence-startup.exp +@@ -0,0 +1,264 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/unfence-startup.xml b/pengine/test10/unfence-startup.xml +new file mode 100644 +index 0000000..83e9ffe +--- /dev/null ++++ b/pengine/test10/unfence-startup.xml +@@ -0,0 +1,73 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ diff --git a/SOURCES/bz1078078-pcmk-pe_do_not_unfence_nodes_that_are_offline_unclean_or_shutting_down.patch b/SOURCES/bz1078078-pcmk-pe_do_not_unfence_nodes_that_are_offline_unclean_or_shutting_down.patch new file mode 100644 index 0000000..6fce78c --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_do_not_unfence_nodes_that_are_offline_unclean_or_shutting_down.patch @@ -0,0 +1,24 @@ +commit c2fd34cb2f81e795dae211466edd270613cb6543 +Author: Andrew Beekhof +Date: Thu Apr 10 14:50:19 2014 +1000 + + Fix: PE: Do not unfence nodes that are offline, unclean or shutting down + + (cherry picked from commit 12c410557f7685366d8504e57e555624bc6f2a07) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 9cb744a..ccaaa3f 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1877,7 +1877,10 @@ trigger_unfencing( + /* Wasnt a stonith device */ + return; + +- } else if(node) { ++ } else if(node ++ && node->details->online ++ && node->details->unclean == FALSE ++ && node->details->shutdown == FALSE) { + action_t *unfence = pe_fence_op(node, "on", FALSE, data_set); + + crm_notice("Unfencing %s: %s", node->details->uname, reason); diff --git a/SOURCES/bz1078078-pcmk-pe_ensure_actions_like_shutdown_and_fence_have_a_unique_key.patch b/SOURCES/bz1078078-pcmk-pe_ensure_actions_like_shutdown_and_fence_have_a_unique_key.patch new file mode 100644 index 0000000..2a57c2b --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_ensure_actions_like_shutdown_and_fence_have_a_unique_key.patch @@ -0,0 +1,15241 @@ +commit 4b390749bf42df944402e2530a27521e89a45682 +Author: Andrew Beekhof +Date: Wed Apr 2 15:24:21 2014 +1100 + + Refactor: PE: Ensure actions like shutdown and fence have a unique key + + (cherry picked from commit 30bb942116c8e9a1e0ace47a98a0447889407d50) + + Conflicts: + pengine/regression.sh + pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp + pengine/test10/594.exp + pengine/test10/662.exp + pengine/test10/696.exp + pengine/test10/726.exp + pengine/test10/735.exp + pengine/test10/764.exp + pengine/test10/797.exp + pengine/test10/829.exp + pengine/test10/anti-colocation-order.exp + pengine/test10/asymmetric.exp + pengine/test10/attrs1.exp + pengine/test10/attrs2.exp + pengine/test10/attrs3.exp + pengine/test10/attrs4.exp + pengine/test10/attrs5.exp + pengine/test10/attrs6.exp + pengine/test10/attrs7.exp + pengine/test10/attrs8.exp + pengine/test10/balanced.exp + pengine/test10/base-score.exp + pengine/test10/bug-1572-1.exp + pengine/test10/bug-1572-2.exp + pengine/test10/bug-1573.exp + pengine/test10/bug-1820-1.exp + pengine/test10/bug-1820.exp + pengine/test10/bug-1822.exp + pengine/test10/bug-5014-A-start-B-start.exp + pengine/test10/bug-5014-A-stopped-B-stopped.exp + pengine/test10/bug-5014-CLONE-A-start-B-start.exp + pengine/test10/bug-5014-CthenAthenB-C-stopped.exp + pengine/test10/bug-5028-detach.exp + pengine/test10/bug-lf-2160.exp + pengine/test10/bug-lf-2435.exp + pengine/test10/bug-lf-2445.exp + pengine/test10/bug-lf-2581.exp + pengine/test10/bug-lf-2606.exp + pengine/test10/bug-n-387749.exp + pengine/test10/bug-pm-11.exp + pengine/test10/clone-anon-dup.exp + pengine/test10/clone-anon-probe-1.exp + pengine/test10/clone-colocate-instance-1.exp + pengine/test10/clone-colocate-instance-2.exp + pengine/test10/clone-interleave-1.exp + pengine/test10/clone-interleave-2.exp + pengine/test10/clone-interleave-3.exp + pengine/test10/clone-no-shuffle.exp + pengine/test10/clone-order-instance.exp + pengine/test10/clone-order-primitive.exp + pengine/test10/coloc-group.exp + pengine/test10/coloc-list.exp + pengine/test10/coloc-loop.exp + pengine/test10/coloc-many-one.exp + pengine/test10/colocate-primitive-with-clone.exp + pengine/test10/comments.exp + pengine/test10/container-1.exp + pengine/test10/container-2.exp + pengine/test10/container-3.exp + pengine/test10/container-4.exp + pengine/test10/container-group-1.exp + pengine/test10/container-group-2.exp + pengine/test10/container-group-3.exp + pengine/test10/container-group-4.exp + pengine/test10/date-1.exp + pengine/test10/domain.exp + pengine/test10/group-fail.exp + pengine/test10/group-unmanaged-stopped.exp + pengine/test10/group-unmanaged.exp + pengine/test10/group1.exp + pengine/test10/group10.exp + pengine/test10/group11.exp + pengine/test10/group15.exp + pengine/test10/group2.exp + pengine/test10/group3.exp + pengine/test10/group4.exp + pengine/test10/group5.exp + pengine/test10/group6.exp + pengine/test10/group7.exp + pengine/test10/group8.exp + pengine/test10/group9.exp + pengine/test10/honor_stonith_rsc_order1.exp + pengine/test10/honor_stonith_rsc_order2.exp + pengine/test10/honor_stonith_rsc_order3.exp + pengine/test10/honor_stonith_rsc_order4.exp + pengine/test10/ignore_stonith_rsc_order1.exp + pengine/test10/ignore_stonith_rsc_order2.exp + pengine/test10/ignore_stonith_rsc_order3.exp + pengine/test10/ignore_stonith_rsc_order4.exp + pengine/test10/inc0.exp + pengine/test10/inc1.exp + pengine/test10/inc11.exp + pengine/test10/inc12.exp + pengine/test10/inc2.exp + pengine/test10/inc3.exp + pengine/test10/inc4.exp + pengine/test10/inc5.exp + pengine/test10/inc6.exp + pengine/test10/inc7.exp + pengine/test10/inc8.exp + pengine/test10/interleave-0.exp + pengine/test10/interleave-1.exp + pengine/test10/interleave-2.exp + pengine/test10/interleave-3.exp + pengine/test10/location-sets-templates.exp + pengine/test10/managed-0.exp + pengine/test10/managed-1.exp + pengine/test10/managed-2.exp + pengine/test10/master-0.exp + pengine/test10/master-1.exp + pengine/test10/master-10.exp + pengine/test10/master-11.exp + pengine/test10/master-2.exp + pengine/test10/master-3.exp + pengine/test10/master-4.exp + pengine/test10/master-5.exp + pengine/test10/master-6.exp + pengine/test10/master-7.exp + pengine/test10/master-8.exp + pengine/test10/master-9.exp + pengine/test10/master-depend.exp + pengine/test10/master-ordering.exp + pengine/test10/master-probed-score.exp + pengine/test10/master-reattach.exp + pengine/test10/master-stop.exp + pengine/test10/master-unmanaged-monitor.exp + pengine/test10/migrate-1.exp + pengine/test10/migrate-2.exp + pengine/test10/migrate-3.exp + pengine/test10/migrate-4.exp + pengine/test10/migrate-shutdown.exp + pengine/test10/migrate-start-complex.exp + pengine/test10/minimal.exp + pengine/test10/mon-rsc-1.exp + pengine/test10/mon-rsc-2.exp + pengine/test10/mon-rsc-3.exp + pengine/test10/mon-rsc-4.exp + pengine/test10/multi1.exp + pengine/test10/node-maintenance-1.exp + pengine/test10/node-maintenance-2.exp + pengine/test10/notify-0.exp + pengine/test10/notify-1.exp + pengine/test10/notify-2.exp + pengine/test10/notify-3.exp + pengine/test10/novell-239082.exp + pengine/test10/novell-252693-2.exp + pengine/test10/novell-252693-3.exp + pengine/test10/novell-252693.exp + pengine/test10/obsolete-lrm-resource.exp + pengine/test10/one-or-more-0.exp + pengine/test10/one-or-more-1.exp + pengine/test10/one-or-more-2.exp + pengine/test10/one-or-more-3.exp + pengine/test10/one-or-more-4.exp + pengine/test10/one-or-more-5.exp + pengine/test10/order-mandatory.exp + pengine/test10/order-optional-keyword.exp + pengine/test10/order-optional.exp + pengine/test10/order-required.exp + pengine/test10/order-wrong-kind.exp + pengine/test10/order1.exp + pengine/test10/order2.exp + pengine/test10/order3.exp + pengine/test10/order4.exp + pengine/test10/order5.exp + pengine/test10/order6.exp + pengine/test10/order7.exp + pengine/test10/ordered-set-basic-startup.exp + pengine/test10/orphan-0.exp + pengine/test10/orphan-1.exp + pengine/test10/orphan-2.exp + pengine/test10/params-0.exp + pengine/test10/params-1.exp + pengine/test10/params-2.exp + pengine/test10/params-4.exp + pengine/test10/params-5.exp + pengine/test10/per-node-attrs.exp + pengine/test10/placement-capacity.exp + pengine/test10/placement-location.exp + pengine/test10/placement-priority.exp + pengine/test10/placement-stickiness.exp + pengine/test10/probe-0.exp + pengine/test10/probe-1.exp + pengine/test10/probe-timeout.exp + pengine/test10/quorum-1.exp + pengine/test10/quorum-2.exp + pengine/test10/quorum-3.exp + pengine/test10/quorum-4.exp + pengine/test10/quorum-5.exp + pengine/test10/quorum-6.exp + pengine/test10/rec-node-1.exp + pengine/test10/rec-node-10.exp + pengine/test10/rec-node-11.exp + pengine/test10/rec-node-12.exp + pengine/test10/rec-node-15.exp + pengine/test10/rec-node-2.exp + pengine/test10/rec-node-3.exp + pengine/test10/rec-node-4.exp + pengine/test10/rec-node-5.exp + pengine/test10/rec-node-6.exp + pengine/test10/rec-node-7.exp + pengine/test10/rec-node-8.exp + pengine/test10/rec-node-9.exp + pengine/test10/rec-rsc-0.exp + pengine/test10/rec-rsc-1.exp + pengine/test10/rec-rsc-2.exp + pengine/test10/rec-rsc-3.exp + pengine/test10/rec-rsc-4.exp + pengine/test10/rec-rsc-5.exp + pengine/test10/rec-rsc-6.exp + pengine/test10/rec-rsc-7.exp + pengine/test10/rec-rsc-8.exp + pengine/test10/rec-rsc-9.exp + pengine/test10/remote-startup-probes.exp + pengine/test10/remote-startup.exp + pengine/test10/reprobe-target_rc.exp + pengine/test10/rsc-maintenance.exp + pengine/test10/rsc-sets-clone-1.exp + pengine/test10/rsc-sets-clone.exp + pengine/test10/rsc-sets-master.exp + pengine/test10/rsc-sets-seq-false.exp + pengine/test10/rsc-sets-seq-true.exp + pengine/test10/rsc_dep1.exp + pengine/test10/rsc_dep10.exp + pengine/test10/rsc_dep2.exp + pengine/test10/rsc_dep3.exp + pengine/test10/rsc_dep4.exp + pengine/test10/rsc_dep5.exp + pengine/test10/rsc_dep7.exp + pengine/test10/rsc_dep8.exp + pengine/test10/simple11.exp + pengine/test10/simple12.exp + pengine/test10/simple2.exp + pengine/test10/simple3.exp + pengine/test10/simple4.exp + pengine/test10/simple6.exp + pengine/test10/simple7.exp + pengine/test10/simple8.exp + pengine/test10/stonith-3.exp + pengine/test10/stop-failure-no-fencing.exp + pengine/test10/stop-failure-with-fencing.exp + pengine/test10/stopped-monitor-00.exp + pengine/test10/stopped-monitor-01.exp + pengine/test10/stopped-monitor-02.exp + pengine/test10/stopped-monitor-03.exp + pengine/test10/stopped-monitor-04.exp + pengine/test10/stopped-monitor-05.exp + pengine/test10/stopped-monitor-06.exp + pengine/test10/stopped-monitor-07.exp + pengine/test10/stopped-monitor-08.exp + pengine/test10/stopped-monitor-09.exp + pengine/test10/stopped-monitor-10.exp + pengine/test10/stopped-monitor-11.exp + pengine/test10/stopped-monitor-12.exp + pengine/test10/stopped-monitor-20.exp + pengine/test10/stopped-monitor-21.exp + pengine/test10/stopped-monitor-22.exp + pengine/test10/stopped-monitor-23.exp + pengine/test10/stopped-monitor-24.exp + pengine/test10/stopped-monitor-25.exp + pengine/test10/stopped-monitor-26.exp + pengine/test10/stopped-monitor-27.exp + pengine/test10/stopped-monitor-30.exp + pengine/test10/stopped-monitor-31.exp + pengine/test10/systemhealth2.exp + pengine/test10/systemhealth3.exp + pengine/test10/systemhealthm2.exp + pengine/test10/systemhealthm3.exp + pengine/test10/systemhealthn2.exp + pengine/test10/systemhealthn3.exp + pengine/test10/systemhealtho2.exp + pengine/test10/systemhealtho3.exp + pengine/test10/systemhealthp2.exp + pengine/test10/systemhealthp3.exp + pengine/test10/target-0.exp + pengine/test10/target-1.exp + pengine/test10/target-2.exp + pengine/test10/template-1.exp + pengine/test10/template-2.exp + pengine/test10/template-3.exp + pengine/test10/template-clone-group.exp + pengine/test10/template-clone-primitive.exp + pengine/test10/template-coloc-1.exp + pengine/test10/template-coloc-2.exp + pengine/test10/template-coloc-3.exp + pengine/test10/template-order-1.exp + pengine/test10/template-order-2.exp + pengine/test10/template-order-3.exp + pengine/test10/template-rsc-sets-1.exp + pengine/test10/template-rsc-sets-2.exp + pengine/test10/template-rsc-sets-3.exp + pengine/test10/template-rsc-sets-4.exp + pengine/test10/template-ticket.exp + pengine/test10/ticket-clone-1.exp + pengine/test10/ticket-clone-10.exp + pengine/test10/ticket-clone-11.exp + pengine/test10/ticket-clone-12.exp + pengine/test10/ticket-clone-13.exp + pengine/test10/ticket-clone-14.exp + pengine/test10/ticket-clone-15.exp + pengine/test10/ticket-clone-16.exp + pengine/test10/ticket-clone-17.exp + pengine/test10/ticket-clone-18.exp + pengine/test10/ticket-clone-19.exp + pengine/test10/ticket-clone-2.exp + pengine/test10/ticket-clone-20.exp + pengine/test10/ticket-clone-22.exp + pengine/test10/ticket-clone-23.exp + pengine/test10/ticket-clone-24.exp + pengine/test10/ticket-clone-3.exp + pengine/test10/ticket-clone-4.exp + pengine/test10/ticket-clone-5.exp + pengine/test10/ticket-clone-6.exp + pengine/test10/ticket-clone-7.exp + pengine/test10/ticket-clone-8.exp + pengine/test10/ticket-group-1.exp + pengine/test10/ticket-group-10.exp + pengine/test10/ticket-group-11.exp + pengine/test10/ticket-group-12.exp + pengine/test10/ticket-group-13.exp + pengine/test10/ticket-group-14.exp + pengine/test10/ticket-group-15.exp + pengine/test10/ticket-group-16.exp + pengine/test10/ticket-group-17.exp + pengine/test10/ticket-group-18.exp + pengine/test10/ticket-group-19.exp + pengine/test10/ticket-group-2.exp + pengine/test10/ticket-group-20.exp + pengine/test10/ticket-group-21.exp + pengine/test10/ticket-group-22.exp + pengine/test10/ticket-group-23.exp + pengine/test10/ticket-group-24.exp + pengine/test10/ticket-group-3.exp + pengine/test10/ticket-group-4.exp + pengine/test10/ticket-group-5.exp + pengine/test10/ticket-group-6.exp + pengine/test10/ticket-group-7.exp + pengine/test10/ticket-group-8.exp + pengine/test10/ticket-group-9.exp + pengine/test10/ticket-master-1.exp + pengine/test10/ticket-master-10.exp + pengine/test10/ticket-master-11.exp + pengine/test10/ticket-master-12.exp + pengine/test10/ticket-master-13.exp + pengine/test10/ticket-master-14.exp + pengine/test10/ticket-master-15.exp + pengine/test10/ticket-master-16.exp + pengine/test10/ticket-master-17.exp + pengine/test10/ticket-master-18.exp + pengine/test10/ticket-master-19.exp + pengine/test10/ticket-master-2.exp + pengine/test10/ticket-master-20.exp + pengine/test10/ticket-master-21.exp + pengine/test10/ticket-master-22.exp + pengine/test10/ticket-master-23.exp + pengine/test10/ticket-master-24.exp + pengine/test10/ticket-master-3.exp + pengine/test10/ticket-master-4.exp + pengine/test10/ticket-master-5.exp + pengine/test10/ticket-master-6.exp + pengine/test10/ticket-master-7.exp + pengine/test10/ticket-master-8.exp + pengine/test10/ticket-master-9.exp + pengine/test10/ticket-primitive-1.exp + pengine/test10/ticket-primitive-10.exp + pengine/test10/ticket-primitive-11.exp + pengine/test10/ticket-primitive-12.exp + pengine/test10/ticket-primitive-13.exp + pengine/test10/ticket-primitive-14.exp + pengine/test10/ticket-primitive-15.exp + pengine/test10/ticket-primitive-16.exp + pengine/test10/ticket-primitive-17.exp + pengine/test10/ticket-primitive-18.exp + pengine/test10/ticket-primitive-19.exp + pengine/test10/ticket-primitive-2.exp + pengine/test10/ticket-primitive-20.exp + pengine/test10/ticket-primitive-21.exp + pengine/test10/ticket-primitive-22.exp + pengine/test10/ticket-primitive-23.exp + pengine/test10/ticket-primitive-24.exp + pengine/test10/ticket-primitive-3.exp + pengine/test10/ticket-primitive-4.exp + pengine/test10/ticket-primitive-5.exp + pengine/test10/ticket-primitive-6.exp + pengine/test10/ticket-primitive-7.exp + pengine/test10/ticket-primitive-8.exp + pengine/test10/ticket-primitive-9.exp + pengine/test10/ticket-rsc-sets-1.exp + pengine/test10/ticket-rsc-sets-10.exp + pengine/test10/ticket-rsc-sets-11.exp + pengine/test10/ticket-rsc-sets-12.exp + pengine/test10/ticket-rsc-sets-13.exp + pengine/test10/ticket-rsc-sets-14.exp + pengine/test10/ticket-rsc-sets-2.exp + pengine/test10/ticket-rsc-sets-3.exp + pengine/test10/ticket-rsc-sets-4.exp + pengine/test10/ticket-rsc-sets-5.exp + pengine/test10/ticket-rsc-sets-6.exp + pengine/test10/ticket-rsc-sets-7.exp + pengine/test10/ticket-rsc-sets-8.exp + pengine/test10/ticket-rsc-sets-9.exp + pengine/test10/unmanaged-master.exp + pengine/test10/unmanaged-stop-1.exp + pengine/test10/unmanaged-stop-2.exp + pengine/test10/unmanaged-stop-3.exp + pengine/test10/unmanaged-stop-4.exp + pengine/test10/unrunnable-1.exp + pengine/test10/use-after-free-merge.exp + pengine/test10/utilization-order1.exp + pengine/test10/utilization-order2.exp + pengine/test10/utilization-order3.exp + pengine/test10/utilization.exp + pengine/test10/whitebox-ms-ordering.exp + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index a8e5412..a07fdad 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -851,14 +851,14 @@ probe_resources(pe_working_set_t * data_set) + } + + if (probed != NULL && crm_is_true(probed) == FALSE) { +- action_t *probe_op = custom_action(NULL, strdup(CRM_OP_REPROBE), ++ action_t *probe_op = custom_action(NULL, g_strdup_printf("%s-%s", CRM_OP_REPROBE, node->details->uname), + CRM_OP_REPROBE, node, FALSE, TRUE, data_set); + + add_hash_param(probe_op->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); + continue; + } + +- probe_node_complete = custom_action(NULL, strdup(CRM_OP_PROBED), ++ probe_node_complete = custom_action(NULL, g_strdup_printf("%s-%s", CRM_OP_PROBED, node->details->uname), + CRM_OP_PROBED, node, FALSE, TRUE, data_set); + if (crm_is_true(probed)) { + crm_trace("unset"); +@@ -1265,21 +1265,22 @@ any_managed_resources(pe_working_set_t * data_set) + action_t * + pe_fence_op(node_t * node, const char *op, pe_working_set_t * data_set) + { ++ char *key = NULL; + action_t *stonith_op = NULL; + + if(op == NULL) { + op = data_set->stonith_action; + } + +- stonith_op = custom_action( +- NULL, g_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op), +- CRM_OP_FENCE, node, FALSE, TRUE, data_set); +- +- add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname); ++ key = g_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op); + +- add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id); ++ if(stonith_op == NULL) { ++ stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, FALSE, TRUE, data_set); + +- add_hash_param(stonith_op->meta, "stonith_action", op); ++ add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname); ++ add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id); ++ add_hash_param(stonith_op->meta, "stonith_action", op); ++ } + + return stonith_op; + } +@@ -1338,7 +1339,7 @@ stage6(pe_working_set_t * data_set) + + crm_notice("Scheduling Node %s for shutdown", node->details->uname); + +- down_op = custom_action(NULL, strdup(CRM_OP_SHUTDOWN), ++ down_op = custom_action(NULL, g_strdup_printf("%s-%s", CRM_OP_SHUTDOWN, node->details->uname), + CRM_OP_SHUTDOWN, node, FALSE, TRUE, data_set); + + shutdown_constraints(node, down_op, data_set); +@@ -1367,20 +1368,22 @@ stage6(pe_working_set_t * data_set) + } + + if (dc_down != NULL) { +- GListPtr shutdown_matches = find_actions(data_set->actions, CRM_OP_SHUTDOWN, NULL); ++ GListPtr gIter = NULL; + + crm_trace("Ordering shutdowns before %s on %s (DC)", + dc_down->task, dc_down->node->details->uname); + + add_hash_param(dc_down->meta, XML_ATTR_TE_NOWAIT, XML_BOOLEAN_TRUE); + +- gIter = shutdown_matches; +- for (; gIter != NULL; gIter = gIter->next) { ++ for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) { + action_t *node_stop = (action_t *) gIter->data; + +- if (node_stop->node->details->is_dc) { ++ if (safe_str_neq(CRM_OP_SHUTDOWN, node_stop->task)) { ++ continue; ++ } else if (node_stop->node->details->is_dc) { + continue; + } ++ + crm_debug("Ordering shutdown on %s before %s on %s", + node_stop->node->details->uname, + dc_down->task, dc_down->node->details->uname); +@@ -1391,7 +1394,6 @@ stage6(pe_working_set_t * data_set) + if (last_stonith && dc_down != last_stonith) { + order_actions(last_stonith, dc_down, pe_order_optional); + } +- g_list_free(shutdown_matches); + } + + if (last_stonith) { +diff --git a/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp +new file mode 100644 +index 0000000..6374d93 +--- /dev/null ++++ b/pengine/test10/11-a-then-bm-b-move-a-clone-starting.exp +@@ -0,0 +1,168 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/594.exp b/pengine/test10/594.exp +index f1e77f8..357921c 100644 +--- a/pengine/test10/594.exp ++++ b/pengine/test10/594.exp +@@ -191,7 +191,7 @@ + + + +- ++ + + + +@@ -221,7 +221,7 @@ + + + +- ++ + + + +@@ -236,7 +236,7 @@ + + + +- ++ + + + +@@ -260,10 +260,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/662.exp b/pengine/test10/662.exp +index 90f8a54..910f85f 100644 +--- a/pengine/test10/662.exp ++++ b/pengine/test10/662.exp +@@ -282,7 +282,7 @@ + + + +- ++ + + + +@@ -297,7 +297,7 @@ + + + +- ++ + + + +@@ -321,7 +321,7 @@ + + + +- ++ + + + +@@ -351,7 +351,7 @@ + + + +- ++ + + + +@@ -378,7 +378,7 @@ + + + +- ++ + + + +@@ -408,16 +408,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/696.exp b/pengine/test10/696.exp +index ec6a5f6..45bf881 100644 +--- a/pengine/test10/696.exp ++++ b/pengine/test10/696.exp +@@ -309,7 +309,7 @@ + + + +- ++ + + + +@@ -330,7 +330,7 @@ + + + +- ++ + + + +@@ -351,7 +351,7 @@ + + + +- ++ + + + +@@ -387,13 +387,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/726.exp b/pengine/test10/726.exp +index 0f56ae3..0a10a6d 100644 +--- a/pengine/test10/726.exp ++++ b/pengine/test10/726.exp +@@ -531,7 +531,7 @@ + + + +- ++ + + + +@@ -552,7 +552,7 @@ + + + +- ++ + + + +@@ -585,7 +585,7 @@ + + + +- ++ + + + +@@ -621,7 +621,7 @@ + + + +- ++ + + + +@@ -663,16 +663,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/735.exp b/pengine/test10/735.exp +index fd66dfe..e4b73e5 100644 +--- a/pengine/test10/735.exp ++++ b/pengine/test10/735.exp +@@ -182,7 +182,7 @@ + + + +- ++ + + + +@@ -209,7 +209,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/764.exp b/pengine/test10/764.exp +index 890665a..bbb2d32 100644 +--- a/pengine/test10/764.exp ++++ b/pengine/test10/764.exp +@@ -156,7 +156,7 @@ + + + +- ++ + + + +@@ -177,7 +177,7 @@ + + + +- ++ + + + +@@ -204,10 +204,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/797.exp b/pengine/test10/797.exp +index cac962d..02ef70b 100644 +--- a/pengine/test10/797.exp ++++ b/pengine/test10/797.exp +@@ -324,7 +324,7 @@ + + + +- ++ + + + +@@ -342,7 +342,7 @@ + + + +- ++ + + + +@@ -363,7 +363,7 @@ + + + +- ++ + + + +@@ -381,7 +381,7 @@ + + + +- ++ + + + +@@ -414,13 +414,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/829.exp b/pengine/test10/829.exp +index 749014f..21a9ecd 100644 +--- a/pengine/test10/829.exp ++++ b/pengine/test10/829.exp +@@ -266,7 +266,7 @@ + + + +- ++ + + + +@@ -287,7 +287,7 @@ + + + +- ++ + + + +@@ -314,7 +314,7 @@ + + + +- ++ + + + +@@ -350,13 +350,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/anti-colocation-order.exp b/pengine/test10/anti-colocation-order.exp +new file mode 100644 +index 0000000..99d5f3a +--- /dev/null ++++ b/pengine/test10/anti-colocation-order.exp +@@ -0,0 +1,217 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/asymmetric.exp b/pengine/test10/asymmetric.exp +index 33850dd..52a2327 100644 +--- a/pengine/test10/asymmetric.exp ++++ b/pengine/test10/asymmetric.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -49,7 +49,7 @@ + + + +- ++ + + + +@@ -67,10 +67,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/attrs1.exp b/pengine/test10/attrs1.exp +index 9e83487..8dd6a40 100644 +--- a/pengine/test10/attrs1.exp ++++ b/pengine/test10/attrs1.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/attrs2.exp b/pengine/test10/attrs2.exp +index 9e83487..8dd6a40 100644 +--- a/pengine/test10/attrs2.exp ++++ b/pengine/test10/attrs2.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/attrs3.exp b/pengine/test10/attrs3.exp +index 99b81e7..b6e55a2 100644 +--- a/pengine/test10/attrs3.exp ++++ b/pengine/test10/attrs3.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/attrs4.exp b/pengine/test10/attrs4.exp +index 99b81e7..b6e55a2 100644 +--- a/pengine/test10/attrs4.exp ++++ b/pengine/test10/attrs4.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/attrs5.exp b/pengine/test10/attrs5.exp +index c7f836f..5e7b2c7 100644 +--- a/pengine/test10/attrs5.exp ++++ b/pengine/test10/attrs5.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/attrs6.exp b/pengine/test10/attrs6.exp +index 99b81e7..b6e55a2 100644 +--- a/pengine/test10/attrs6.exp ++++ b/pengine/test10/attrs6.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/attrs7.exp b/pengine/test10/attrs7.exp +index 9e83487..8dd6a40 100644 +--- a/pengine/test10/attrs7.exp ++++ b/pengine/test10/attrs7.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/attrs8.exp b/pengine/test10/attrs8.exp +index 9e83487..8dd6a40 100644 +--- a/pengine/test10/attrs8.exp ++++ b/pengine/test10/attrs8.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/balanced.exp b/pengine/test10/balanced.exp +index e6e96d0..c188632 100644 +--- a/pengine/test10/balanced.exp ++++ b/pengine/test10/balanced.exp +@@ -69,7 +69,7 @@ + + + +- ++ + + + +@@ -84,7 +84,7 @@ + + + +- ++ + + + +@@ -105,10 +105,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/base-score.exp b/pengine/test10/base-score.exp +index b6710f5..ab71c3b 100644 +--- a/pengine/test10/base-score.exp ++++ b/pengine/test10/base-score.exp +@@ -50,7 +50,7 @@ + + + +- ++ + + + +@@ -62,7 +62,7 @@ + + + +- ++ + + + +@@ -74,7 +74,7 @@ + + + +- ++ + + + +@@ -86,7 +86,7 @@ + + + +- ++ + + + +@@ -104,16 +104,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bug-1572-1.exp b/pengine/test10/bug-1572-1.exp +index 2b3bf4c..c0184ae 100644 +--- a/pengine/test10/bug-1572-1.exp ++++ b/pengine/test10/bug-1572-1.exp +@@ -750,7 +750,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1572-2.exp b/pengine/test10/bug-1572-2.exp +index ad91bab..465bf1d 100644 +--- a/pengine/test10/bug-1572-2.exp ++++ b/pengine/test10/bug-1572-2.exp +@@ -371,7 +371,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1573.exp b/pengine/test10/bug-1573.exp +index 86eb893..39f7a61 100644 +--- a/pengine/test10/bug-1573.exp ++++ b/pengine/test10/bug-1573.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1820-1.exp b/pengine/test10/bug-1820-1.exp +index 732ded8..2753fd0 100644 +--- a/pengine/test10/bug-1820-1.exp ++++ b/pengine/test10/bug-1820-1.exp +@@ -178,7 +178,7 @@ + + + +- ++ + + + +@@ -193,7 +193,7 @@ + + + +- ++ + + + +@@ -205,7 +205,7 @@ + + + +- ++ + + + +@@ -223,10 +223,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bug-1820.exp b/pengine/test10/bug-1820.exp +index ad98db1..434d4b6 100644 +--- a/pengine/test10/bug-1820.exp ++++ b/pengine/test10/bug-1820.exp +@@ -175,7 +175,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-1822.exp b/pengine/test10/bug-1822.exp +index 0e101b5..6dc28f8 100644 +--- a/pengine/test10/bug-1822.exp ++++ b/pengine/test10/bug-1822.exp +@@ -166,7 +166,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-A-start-B-start.exp b/pengine/test10/bug-5014-A-start-B-start.exp +index 63c271d..6670814 100644 +--- a/pengine/test10/bug-5014-A-start-B-start.exp ++++ b/pengine/test10/bug-5014-A-start-B-start.exp +@@ -74,7 +74,7 @@ + + + +- ++ + + + +@@ -95,7 +95,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-A-stopped-B-stopped.exp b/pengine/test10/bug-5014-A-stopped-B-stopped.exp +index e1a25cd..8814477 100644 +--- a/pengine/test10/bug-5014-A-stopped-B-stopped.exp ++++ b/pengine/test10/bug-5014-A-stopped-B-stopped.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -40,7 +40,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-CLONE-A-start-B-start.exp b/pengine/test10/bug-5014-CLONE-A-start-B-start.exp +index 30d8952..1fc2c70 100644 +--- a/pengine/test10/bug-5014-CLONE-A-start-B-start.exp ++++ b/pengine/test10/bug-5014-CLONE-A-start-B-start.exp +@@ -127,7 +127,7 @@ + + + +- ++ + + + +@@ -148,7 +148,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5014-CthenAthenB-C-stopped.exp b/pengine/test10/bug-5014-CthenAthenB-C-stopped.exp +index 6961d8a..318ba6f 100644 +--- a/pengine/test10/bug-5014-CthenAthenB-C-stopped.exp ++++ b/pengine/test10/bug-5014-CthenAthenB-C-stopped.exp +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5028-detach.exp b/pengine/test10/bug-5028-detach.exp +index 1c3374b..58d12ec 100644 +--- a/pengine/test10/bug-5028-detach.exp ++++ b/pengine/test10/bug-5028-detach.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2160.exp b/pengine/test10/bug-lf-2160.exp +index ed0b455..3d41f1e 100644 +--- a/pengine/test10/bug-lf-2160.exp ++++ b/pengine/test10/bug-lf-2160.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -40,7 +40,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2435.exp b/pengine/test10/bug-lf-2435.exp +index 7255127..5a12601 100644 +--- a/pengine/test10/bug-lf-2435.exp ++++ b/pengine/test10/bug-lf-2435.exp +@@ -70,7 +70,7 @@ + + + +- ++ + + + +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -94,7 +94,7 @@ + + + +- ++ + + + +@@ -112,13 +112,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2445.exp b/pengine/test10/bug-lf-2445.exp +index 5733768..d8f88c3 100644 +--- a/pengine/test10/bug-lf-2445.exp ++++ b/pengine/test10/bug-lf-2445.exp +@@ -80,7 +80,7 @@ + + + +- ++ + + + +@@ -88,7 +88,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2581.exp b/pengine/test10/bug-lf-2581.exp +index 60667fc..2906613 100644 +--- a/pengine/test10/bug-lf-2581.exp ++++ b/pengine/test10/bug-lf-2581.exp +@@ -230,7 +230,7 @@ + + + +- ++ + + + +@@ -266,7 +266,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2606.exp b/pengine/test10/bug-lf-2606.exp +index 270131a..71d8457 100644 +--- a/pengine/test10/bug-lf-2606.exp ++++ b/pengine/test10/bug-lf-2606.exp +@@ -160,7 +160,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-n-387749.exp b/pengine/test10/bug-n-387749.exp +index 394f8ac..ea0cc27 100644 +--- a/pengine/test10/bug-n-387749.exp ++++ b/pengine/test10/bug-n-387749.exp +@@ -345,7 +345,7 @@ + + + +- ++ + + + +@@ -375,7 +375,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-pm-11.exp b/pengine/test10/bug-pm-11.exp +index 42d405f..bb23360 100644 +--- a/pengine/test10/bug-pm-11.exp ++++ b/pengine/test10/bug-pm-11.exp +@@ -213,7 +213,7 @@ + + + +- ++ + + + +@@ -228,7 +228,7 @@ + + + +- ++ + + + +@@ -249,10 +249,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bug-suse-707150.exp b/pengine/test10/bug-suse-707150.exp +index 270b3fd..ca3d57b 100644 +--- a/pengine/test10/bug-suse-707150.exp ++++ b/pengine/test10/bug-suse-707150.exp +@@ -169,7 +169,7 @@ + + + +- ++ + + + +@@ -187,7 +187,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/clone-anon-dup.exp b/pengine/test10/clone-anon-dup.exp +index 69ed601..1fea098 100644 +--- a/pengine/test10/clone-anon-dup.exp ++++ b/pengine/test10/clone-anon-dup.exp +@@ -107,7 +107,7 @@ + + + +- ++ + + + +@@ -119,7 +119,7 @@ + + + +- ++ + + + +@@ -131,7 +131,7 @@ + + + +- ++ + + + +@@ -149,13 +149,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/clone-anon-probe-1.exp b/pengine/test10/clone-anon-probe-1.exp +index 13b8d3d..6a0b6de 100644 +--- a/pengine/test10/clone-anon-probe-1.exp ++++ b/pengine/test10/clone-anon-probe-1.exp +@@ -77,7 +77,7 @@ + + + +- ++ + + + +@@ -89,7 +89,7 @@ + + + +- ++ + + + +@@ -107,10 +107,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/clone-colocate-instance-1.exp b/pengine/test10/clone-colocate-instance-1.exp +index 0d358be..c1d4629 100644 +--- a/pengine/test10/clone-colocate-instance-1.exp ++++ b/pengine/test10/clone-colocate-instance-1.exp +@@ -139,7 +139,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -181,10 +181,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/clone-colocate-instance-2.exp b/pengine/test10/clone-colocate-instance-2.exp +index a02a737..34d03bd 100644 +--- a/pengine/test10/clone-colocate-instance-2.exp ++++ b/pengine/test10/clone-colocate-instance-2.exp +@@ -139,7 +139,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -181,10 +181,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/clone-interleave-1.exp b/pengine/test10/clone-interleave-1.exp +new file mode 100644 +index 0000000..a0d8995 +--- /dev/null ++++ b/pengine/test10/clone-interleave-1.exp +@@ -0,0 +1,231 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/clone-interleave-2.exp b/pengine/test10/clone-interleave-2.exp +new file mode 100644 +index 0000000..fa59afb +--- /dev/null ++++ b/pengine/test10/clone-interleave-2.exp +@@ -0,0 +1,176 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/clone-interleave-3.exp b/pengine/test10/clone-interleave-3.exp +new file mode 100644 +index 0000000..85f6d8b +--- /dev/null ++++ b/pengine/test10/clone-interleave-3.exp +@@ -0,0 +1,244 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/clone-no-shuffle.exp b/pengine/test10/clone-no-shuffle.exp +index c6437a6..4c9ee1b 100644 +--- a/pengine/test10/clone-no-shuffle.exp ++++ b/pengine/test10/clone-no-shuffle.exp +@@ -436,7 +436,7 @@ + + + +- ++ + + + +@@ -448,7 +448,7 @@ + + + +- ++ + + + +@@ -472,10 +472,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/clone-order-instance.exp b/pengine/test10/clone-order-instance.exp +index dd277ae..f6c51de 100644 +--- a/pengine/test10/clone-order-instance.exp ++++ b/pengine/test10/clone-order-instance.exp +@@ -111,7 +111,7 @@ + + + +- ++ + + + +@@ -126,7 +126,7 @@ + + + +- ++ + + + +@@ -147,10 +147,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/clone-order-primitive.exp b/pengine/test10/clone-order-primitive.exp +index aefaf50..e46f1fc 100644 +--- a/pengine/test10/clone-order-primitive.exp ++++ b/pengine/test10/clone-order-primitive.exp +@@ -66,7 +66,7 @@ + + + +- ++ + + + +@@ -74,7 +74,7 @@ + + + +- ++ + + + +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -90,7 +90,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/coloc-group.exp b/pengine/test10/coloc-group.exp +index f6c95f0..04b3b49 100644 +--- a/pengine/test10/coloc-group.exp ++++ b/pengine/test10/coloc-group.exp +@@ -119,7 +119,7 @@ + + + +- ++ + + + +@@ -137,7 +137,7 @@ + + + +- ++ + + + +@@ -155,7 +155,7 @@ + + + +- ++ + + + +@@ -179,13 +179,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/coloc-list.exp b/pengine/test10/coloc-list.exp +index 673c021..ca9d78e 100644 +--- a/pengine/test10/coloc-list.exp ++++ b/pengine/test10/coloc-list.exp +@@ -139,7 +139,7 @@ + + + +- ++ + + + +@@ -160,7 +160,7 @@ + + + +- ++ + + + +@@ -178,7 +178,7 @@ + + + +- ++ + + + +@@ -205,13 +205,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/coloc-loop.exp b/pengine/test10/coloc-loop.exp +index 12e86a2..1b50bad 100644 +--- a/pengine/test10/coloc-loop.exp ++++ b/pengine/test10/coloc-loop.exp +@@ -121,7 +121,7 @@ + + + +- ++ + + + +@@ -139,7 +139,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -181,13 +181,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/coloc-many-one.exp b/pengine/test10/coloc-many-one.exp +index 0dca637..e8503a4 100644 +--- a/pengine/test10/coloc-many-one.exp ++++ b/pengine/test10/coloc-many-one.exp +@@ -121,7 +121,7 @@ + + + +- ++ + + + +@@ -139,7 +139,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -181,13 +181,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/colocate-primitive-with-clone.exp b/pengine/test10/colocate-primitive-with-clone.exp +index 2dacc06..44fac10 100644 +--- a/pengine/test10/colocate-primitive-with-clone.exp ++++ b/pengine/test10/colocate-primitive-with-clone.exp +@@ -133,7 +133,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/comments.exp b/pengine/test10/comments.exp +index 1ad8819..5d6abb2 100644 +--- a/pengine/test10/comments.exp ++++ b/pengine/test10/comments.exp +@@ -63,7 +63,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -99,10 +99,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/container-1.exp b/pengine/test10/container-1.exp +index 522b0f1..e252625 100644 +--- a/pengine/test10/container-1.exp ++++ b/pengine/test10/container-1.exp +@@ -97,7 +97,7 @@ + + + +- ++ + + + +@@ -109,7 +109,7 @@ + + + +- ++ + + + +@@ -127,10 +127,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/container-2.exp b/pengine/test10/container-2.exp +index c5f9e9e..4b1532b 100644 +--- a/pengine/test10/container-2.exp ++++ b/pengine/test10/container-2.exp +@@ -119,7 +119,7 @@ + + + +- ++ + + + +@@ -127,7 +127,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/container-3.exp b/pengine/test10/container-3.exp +index 3c2703a..e1fb4d9 100644 +--- a/pengine/test10/container-3.exp ++++ b/pengine/test10/container-3.exp +@@ -104,7 +104,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/container-4.exp b/pengine/test10/container-4.exp +index d1da381..ed9bafa 100644 +--- a/pengine/test10/container-4.exp ++++ b/pengine/test10/container-4.exp +@@ -119,7 +119,7 @@ + + + +- ++ + + + +@@ -127,7 +127,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/container-group-1.exp b/pengine/test10/container-group-1.exp +index b14aa4c..af8a7d9 100644 +--- a/pengine/test10/container-group-1.exp ++++ b/pengine/test10/container-group-1.exp +@@ -144,7 +144,7 @@ + + + +- ++ + + + +@@ -156,7 +156,7 @@ + + + +- ++ + + + +@@ -174,10 +174,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/container-group-2.exp b/pengine/test10/container-group-2.exp +index ce32cc3..5acdcbc 100644 +--- a/pengine/test10/container-group-2.exp ++++ b/pengine/test10/container-group-2.exp +@@ -207,7 +207,7 @@ + + + +- ++ + + + +@@ -215,7 +215,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/container-group-3.exp b/pengine/test10/container-group-3.exp +index 96d99c6..d07c9cb 100644 +--- a/pengine/test10/container-group-3.exp ++++ b/pengine/test10/container-group-3.exp +@@ -160,7 +160,7 @@ + + + +- ++ + + + +@@ -168,7 +168,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/container-group-4.exp b/pengine/test10/container-group-4.exp +index 6128599..7970a76 100644 +--- a/pengine/test10/container-group-4.exp ++++ b/pengine/test10/container-group-4.exp +@@ -207,7 +207,7 @@ + + + +- ++ + + + +@@ -215,7 +215,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/date-1.exp b/pengine/test10/date-1.exp +index 2a64721..eba26f2 100644 +--- a/pengine/test10/date-1.exp ++++ b/pengine/test10/date-1.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/domain.exp b/pengine/test10/domain.exp +index 3d9eb9a..1228dde 100644 +--- a/pengine/test10/domain.exp ++++ b/pengine/test10/domain.exp +@@ -50,7 +50,7 @@ + + + +- ++ + + + +@@ -62,7 +62,7 @@ + + + +- ++ + + + +@@ -74,7 +74,7 @@ + + + +- ++ + + + +@@ -86,7 +86,7 @@ + + + +- ++ + + + +@@ -104,16 +104,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group-fail.exp b/pengine/test10/group-fail.exp +index cf4c6eb..79ae80a 100644 +--- a/pengine/test10/group-fail.exp ++++ b/pengine/test10/group-fail.exp +@@ -159,7 +159,7 @@ + + + +- ++ + + + +@@ -167,7 +167,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/group-unmanaged-stopped.exp b/pengine/test10/group-unmanaged-stopped.exp +new file mode 100644 +index 0000000..3ea67ca +--- /dev/null ++++ b/pengine/test10/group-unmanaged-stopped.exp +@@ -0,0 +1,66 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/group-unmanaged.exp b/pengine/test10/group-unmanaged.exp +new file mode 100644 +index 0000000..9ccb828 +--- /dev/null ++++ b/pengine/test10/group-unmanaged.exp +@@ -0,0 +1,18 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/group1.exp b/pengine/test10/group1.exp +index e0c19fe..45e2fcc 100644 +--- a/pengine/test10/group1.exp ++++ b/pengine/test10/group1.exp +@@ -138,7 +138,7 @@ + + + +- ++ + + + +@@ -156,7 +156,7 @@ + + + +- ++ + + + +@@ -180,10 +180,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group10.exp b/pengine/test10/group10.exp +index 4285683..7fab8b3 100644 +--- a/pengine/test10/group10.exp ++++ b/pengine/test10/group10.exp +@@ -291,7 +291,7 @@ + + + +- ++ + + + +@@ -306,7 +306,7 @@ + + + +- ++ + + + +@@ -321,7 +321,7 @@ + + + +- ++ + + + +@@ -336,7 +336,7 @@ + + + +- ++ + + + +@@ -360,16 +360,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group11.exp b/pengine/test10/group11.exp +index 5c24675..b316b01 100644 +--- a/pengine/test10/group11.exp ++++ b/pengine/test10/group11.exp +@@ -68,7 +68,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/group15.exp b/pengine/test10/group15.exp +index 421cd86..47a744a 100644 +--- a/pengine/test10/group15.exp ++++ b/pengine/test10/group15.exp +@@ -192,7 +192,7 @@ + + + +- ++ + + + +@@ -219,7 +219,7 @@ + + + +- ++ + + + +@@ -252,10 +252,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group2.exp b/pengine/test10/group2.exp +index 061798d..7df6735 100644 +--- a/pengine/test10/group2.exp ++++ b/pengine/test10/group2.exp +@@ -207,7 +207,7 @@ + + + +- ++ + + + +@@ -231,7 +231,7 @@ + + + +- ++ + + + +@@ -261,10 +261,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group3.exp b/pengine/test10/group3.exp +index 7cb477e..7cf2962 100644 +--- a/pengine/test10/group3.exp ++++ b/pengine/test10/group3.exp +@@ -279,7 +279,7 @@ + + + +- ++ + + + +@@ -306,7 +306,7 @@ + + + +- ++ + + + +@@ -339,10 +339,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group4.exp b/pengine/test10/group4.exp +index b6b275e..207a905 100644 +--- a/pengine/test10/group4.exp ++++ b/pengine/test10/group4.exp +@@ -46,7 +46,7 @@ + + + +- ++ + + + +@@ -54,7 +54,7 @@ + + + +- ++ + + + +@@ -84,10 +84,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group5.exp b/pengine/test10/group5.exp +index 920af4d..fa85bbe 100644 +--- a/pengine/test10/group5.exp ++++ b/pengine/test10/group5.exp +@@ -296,7 +296,7 @@ + + + +- ++ + + + +@@ -320,7 +320,7 @@ + + + +- ++ + + + +@@ -334,10 +334,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group6.exp b/pengine/test10/group6.exp +index 40d149f..d71f695 100644 +--- a/pengine/test10/group6.exp ++++ b/pengine/test10/group6.exp +@@ -420,7 +420,7 @@ + + + +- ++ + + + +@@ -447,7 +447,7 @@ + + + +- ++ + + + +@@ -461,10 +461,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group7.exp b/pengine/test10/group7.exp +index 18abb01..4834e86 100644 +--- a/pengine/test10/group7.exp ++++ b/pengine/test10/group7.exp +@@ -373,7 +373,7 @@ + + + +- ++ + + + +@@ -403,7 +403,7 @@ + + + +- ++ + + + +@@ -433,7 +433,7 @@ + + + +- ++ + + + +@@ -469,13 +469,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/group8.exp b/pengine/test10/group8.exp +index 6700350..c4887dd 100644 +--- a/pengine/test10/group8.exp ++++ b/pengine/test10/group8.exp +@@ -160,7 +160,7 @@ + + + +- ++ + + + +@@ -196,7 +196,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/group9.exp b/pengine/test10/group9.exp +index cf8dfea..650dfab 100644 +--- a/pengine/test10/group9.exp ++++ b/pengine/test10/group9.exp +@@ -384,7 +384,7 @@ + + + +- ++ + + + +@@ -417,7 +417,7 @@ + + + +- ++ + + + +@@ -431,10 +431,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/honor_stonith_rsc_order1.exp b/pengine/test10/honor_stonith_rsc_order1.exp +index 66a401b..7dd0ffb 100644 +--- a/pengine/test10/honor_stonith_rsc_order1.exp ++++ b/pengine/test10/honor_stonith_rsc_order1.exp +@@ -123,7 +123,7 @@ + + + +- ++ + + + +@@ -147,7 +147,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/honor_stonith_rsc_order2.exp b/pengine/test10/honor_stonith_rsc_order2.exp +index c4b0ee2..2ceba4a 100644 +--- a/pengine/test10/honor_stonith_rsc_order2.exp ++++ b/pengine/test10/honor_stonith_rsc_order2.exp +@@ -185,7 +185,7 @@ + + + +- ++ + + + +@@ -215,7 +215,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/honor_stonith_rsc_order3.exp b/pengine/test10/honor_stonith_rsc_order3.exp +index 8c0973a..780b1ba 100644 +--- a/pengine/test10/honor_stonith_rsc_order3.exp ++++ b/pengine/test10/honor_stonith_rsc_order3.exp +@@ -212,7 +212,7 @@ + + + +- ++ + + + +@@ -242,7 +242,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/honor_stonith_rsc_order4.exp b/pengine/test10/honor_stonith_rsc_order4.exp +index 6ddf8f9..4bf90b1 100644 +--- a/pengine/test10/honor_stonith_rsc_order4.exp ++++ b/pengine/test10/honor_stonith_rsc_order4.exp +@@ -70,7 +70,7 @@ + + + +- ++ + + + +@@ -94,7 +94,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ignore_stonith_rsc_order1.exp b/pengine/test10/ignore_stonith_rsc_order1.exp +index 883ba7a..099022c 100644 +--- a/pengine/test10/ignore_stonith_rsc_order1.exp ++++ b/pengine/test10/ignore_stonith_rsc_order1.exp +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -69,7 +69,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ignore_stonith_rsc_order2.exp b/pengine/test10/ignore_stonith_rsc_order2.exp +index f25ec5a..22850b9 100644 +--- a/pengine/test10/ignore_stonith_rsc_order2.exp ++++ b/pengine/test10/ignore_stonith_rsc_order2.exp +@@ -105,7 +105,7 @@ + + + +- ++ + + + +@@ -129,7 +129,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ignore_stonith_rsc_order3.exp b/pengine/test10/ignore_stonith_rsc_order3.exp +index 2ccc016..4feaa51 100644 +--- a/pengine/test10/ignore_stonith_rsc_order3.exp ++++ b/pengine/test10/ignore_stonith_rsc_order3.exp +@@ -132,7 +132,7 @@ + + + +- ++ + + + +@@ -156,7 +156,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ignore_stonith_rsc_order4.exp b/pengine/test10/ignore_stonith_rsc_order4.exp +index a1a840d..f6967c1 100644 +--- a/pengine/test10/ignore_stonith_rsc_order4.exp ++++ b/pengine/test10/ignore_stonith_rsc_order4.exp +@@ -159,7 +159,7 @@ + + + +- ++ + + + +@@ -183,7 +183,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/inc0.exp b/pengine/test10/inc0.exp +index 71cced1..2d957aa 100644 +--- a/pengine/test10/inc0.exp ++++ b/pengine/test10/inc0.exp +@@ -187,7 +187,7 @@ + + + +- ++ + + + +@@ -211,7 +211,7 @@ + + + +- ++ + + + +@@ -241,10 +241,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/inc1.exp b/pengine/test10/inc1.exp +index 6f5d759..172ae77 100644 +--- a/pengine/test10/inc1.exp ++++ b/pengine/test10/inc1.exp +@@ -256,7 +256,7 @@ + + + +- ++ + + + +@@ -286,7 +286,7 @@ + + + +- ++ + + + +@@ -322,10 +322,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/inc11.exp b/pengine/test10/inc11.exp +index 54314c1..527d335 100644 +--- a/pengine/test10/inc11.exp ++++ b/pengine/test10/inc11.exp +@@ -193,7 +193,7 @@ + + + +- ++ + + + +@@ -211,7 +211,7 @@ + + + +- ++ + + + +@@ -229,7 +229,7 @@ + + + +- ++ + + + +@@ -253,13 +253,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/inc12.exp b/pengine/test10/inc12.exp +index 4fabf43..aa3f992 100644 +--- a/pengine/test10/inc12.exp ++++ b/pengine/test10/inc12.exp +@@ -420,7 +420,7 @@ + + + +- ++ + + + +@@ -441,7 +441,7 @@ + + + +- ++ + + + +@@ -462,7 +462,7 @@ + + + +- ++ + + + +@@ -486,7 +486,7 @@ + + + +- ++ + + + +@@ -510,31 +510,31 @@ + + + +- ++ + + + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + + + +- ++ + + + +diff --git a/pengine/test10/inc2.exp b/pengine/test10/inc2.exp +index 82474dc..f3064aa 100644 +--- a/pengine/test10/inc2.exp ++++ b/pengine/test10/inc2.exp +@@ -191,7 +191,7 @@ + + + +- ++ + + + +@@ -221,7 +221,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/inc3.exp b/pengine/test10/inc3.exp +index 653c377..5e566bb 100644 +--- a/pengine/test10/inc3.exp ++++ b/pengine/test10/inc3.exp +@@ -369,7 +369,7 @@ + + + +- ++ + + + +@@ -393,7 +393,7 @@ + + + +- ++ + + + +@@ -423,10 +423,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/inc4.exp b/pengine/test10/inc4.exp +index 7b73210..6a9e648 100644 +--- a/pengine/test10/inc4.exp ++++ b/pengine/test10/inc4.exp +@@ -384,7 +384,7 @@ + + + +- ++ + + + +@@ -408,7 +408,7 @@ + + + +- ++ + + + +@@ -438,10 +438,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/inc5.exp b/pengine/test10/inc5.exp +index f8be6e4..3b0ad75 100644 +--- a/pengine/test10/inc5.exp ++++ b/pengine/test10/inc5.exp +@@ -629,7 +629,7 @@ + + + +- ++ + + + +@@ -686,7 +686,7 @@ + + + +- ++ + + + +@@ -749,10 +749,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/inc6.exp b/pengine/test10/inc6.exp +index 5b347d0..3b4c8e2 100644 +--- a/pengine/test10/inc6.exp ++++ b/pengine/test10/inc6.exp +@@ -438,7 +438,7 @@ + + + +- ++ + + + +@@ -446,7 +446,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/inc7.exp b/pengine/test10/inc7.exp +index ec1c6d2..2685678 100644 +--- a/pengine/test10/inc7.exp ++++ b/pengine/test10/inc7.exp +@@ -560,7 +560,7 @@ + + + +- ++ + + + +@@ -602,7 +602,7 @@ + + + +- ++ + + + +@@ -644,7 +644,7 @@ + + + +- ++ + + + +@@ -692,13 +692,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/inc8.exp b/pengine/test10/inc8.exp +index e6ac6de..03f3a4c 100644 +--- a/pengine/test10/inc8.exp ++++ b/pengine/test10/inc8.exp +@@ -270,7 +270,7 @@ + + + +- ++ + + + +@@ -312,7 +312,7 @@ + + + +- ++ + + + +@@ -360,10 +360,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/interleave-0.exp b/pengine/test10/interleave-0.exp +index d5523d5..8a3c701 100644 +--- a/pengine/test10/interleave-0.exp ++++ b/pengine/test10/interleave-0.exp +@@ -1591,7 +1591,7 @@ + + + +- ++ + + + +@@ -1660,7 +1660,7 @@ + + + +- ++ + + + +@@ -1735,7 +1735,7 @@ + + + +- ++ + + + +@@ -1807,7 +1807,7 @@ + + + +- ++ + + + +@@ -1849,7 +1849,7 @@ + + + +- ++ + + + +@@ -1918,7 +1918,7 @@ + + + +- ++ + + + +@@ -1984,7 +1984,7 @@ + + + +- ++ + + + +@@ -2041,7 +2041,7 @@ + + + +- ++ + + + +@@ -2107,28 +2107,28 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/interleave-1.exp b/pengine/test10/interleave-1.exp +index d5523d5..8a3c701 100644 +--- a/pengine/test10/interleave-1.exp ++++ b/pengine/test10/interleave-1.exp +@@ -1591,7 +1591,7 @@ + + + +- ++ + + + +@@ -1660,7 +1660,7 @@ + + + +- ++ + + + +@@ -1735,7 +1735,7 @@ + + + +- ++ + + + +@@ -1807,7 +1807,7 @@ + + + +- ++ + + + +@@ -1849,7 +1849,7 @@ + + + +- ++ + + + +@@ -1918,7 +1918,7 @@ + + + +- ++ + + + +@@ -1984,7 +1984,7 @@ + + + +- ++ + + + +@@ -2041,7 +2041,7 @@ + + + +- ++ + + + +@@ -2107,28 +2107,28 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/interleave-2.exp b/pengine/test10/interleave-2.exp +index d5523d5..8a3c701 100644 +--- a/pengine/test10/interleave-2.exp ++++ b/pengine/test10/interleave-2.exp +@@ -1591,7 +1591,7 @@ + + + +- ++ + + + +@@ -1660,7 +1660,7 @@ + + + +- ++ + + + +@@ -1735,7 +1735,7 @@ + + + +- ++ + + + +@@ -1807,7 +1807,7 @@ + + + +- ++ + + + +@@ -1849,7 +1849,7 @@ + + + +- ++ + + + +@@ -1918,7 +1918,7 @@ + + + +- ++ + + + +@@ -1984,7 +1984,7 @@ + + + +- ++ + + + +@@ -2041,7 +2041,7 @@ + + + +- ++ + + + +@@ -2107,28 +2107,28 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/interleave-3.exp b/pengine/test10/interleave-3.exp +index d5523d5..8a3c701 100644 +--- a/pengine/test10/interleave-3.exp ++++ b/pengine/test10/interleave-3.exp +@@ -1591,7 +1591,7 @@ + + + +- ++ + + + +@@ -1660,7 +1660,7 @@ + + + +- ++ + + + +@@ -1735,7 +1735,7 @@ + + + +- ++ + + + +@@ -1807,7 +1807,7 @@ + + + +- ++ + + + +@@ -1849,7 +1849,7 @@ + + + +- ++ + + + +@@ -1918,7 +1918,7 @@ + + + +- ++ + + + +@@ -1984,7 +1984,7 @@ + + + +- ++ + + + +@@ -2041,7 +2041,7 @@ + + + +- ++ + + + +@@ -2107,28 +2107,28 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/location-sets-templates.exp b/pengine/test10/location-sets-templates.exp +new file mode 100644 +index 0000000..9ffacc6 +--- /dev/null ++++ b/pengine/test10/location-sets-templates.exp +@@ -0,0 +1,257 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/managed-0.exp b/pengine/test10/managed-0.exp +index 67fcacb..40f48ed 100644 +--- a/pengine/test10/managed-0.exp ++++ b/pengine/test10/managed-0.exp +@@ -730,7 +730,7 @@ + + + +- ++ + + + +@@ -772,7 +772,7 @@ + + + +- ++ + + + +@@ -823,7 +823,7 @@ + + + +- ++ + + + +@@ -871,7 +871,7 @@ + + + +- ++ + + + +@@ -889,7 +889,7 @@ + + + +- ++ + + + +@@ -934,7 +934,7 @@ + + + +- ++ + + + +@@ -976,7 +976,7 @@ + + + +- ++ + + + +@@ -1009,7 +1009,7 @@ + + + +- ++ + + + +@@ -1051,28 +1051,28 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/managed-1.exp b/pengine/test10/managed-1.exp +index 67fcacb..40f48ed 100644 +--- a/pengine/test10/managed-1.exp ++++ b/pengine/test10/managed-1.exp +@@ -730,7 +730,7 @@ + + + +- ++ + + + +@@ -772,7 +772,7 @@ + + + +- ++ + + + +@@ -823,7 +823,7 @@ + + + +- ++ + + + +@@ -871,7 +871,7 @@ + + + +- ++ + + + +@@ -889,7 +889,7 @@ + + + +- ++ + + + +@@ -934,7 +934,7 @@ + + + +- ++ + + + +@@ -976,7 +976,7 @@ + + + +- ++ + + + +@@ -1009,7 +1009,7 @@ + + + +- ++ + + + +@@ -1051,28 +1051,28 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/managed-2.exp b/pengine/test10/managed-2.exp +index 59dcdaf..5020993 100644 +--- a/pengine/test10/managed-2.exp ++++ b/pengine/test10/managed-2.exp +@@ -1036,7 +1036,7 @@ + + + +- ++ + + + +@@ -1090,7 +1090,7 @@ + + + +- ++ + + + +@@ -1147,7 +1147,7 @@ + + + +- ++ + + + +@@ -1204,7 +1204,7 @@ + + + +- ++ + + + +@@ -1246,7 +1246,7 @@ + + + +- ++ + + + +@@ -1303,7 +1303,7 @@ + + + +- ++ + + + +@@ -1357,7 +1357,7 @@ + + + +- ++ + + + +@@ -1405,7 +1405,7 @@ + + + +- ++ + + + +@@ -1459,28 +1459,28 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-0.exp b/pengine/test10/master-0.exp +index 695f274..a1423de 100644 +--- a/pengine/test10/master-0.exp ++++ b/pengine/test10/master-0.exp +@@ -187,7 +187,7 @@ + + + +- ++ + + + +@@ -211,7 +211,7 @@ + + + +- ++ + + + +@@ -241,10 +241,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-1.exp b/pengine/test10/master-1.exp +index d5e0def..cb2b4a3 100644 +--- a/pengine/test10/master-1.exp ++++ b/pengine/test10/master-1.exp +@@ -227,7 +227,7 @@ + + + +- ++ + + + +@@ -251,7 +251,7 @@ + + + +- ++ + + + +@@ -281,10 +281,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-10.exp b/pengine/test10/master-10.exp +index c34c62c..45b3817 100644 +--- a/pengine/test10/master-10.exp ++++ b/pengine/test10/master-10.exp +@@ -603,7 +603,7 @@ + + + +- ++ + + + +@@ -627,7 +627,7 @@ + + + +- ++ + + + +@@ -657,10 +657,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-11.exp b/pengine/test10/master-11.exp +index ae85aa6..33d3ede 100644 +--- a/pengine/test10/master-11.exp ++++ b/pengine/test10/master-11.exp +@@ -166,7 +166,7 @@ + + + +- ++ + + + +@@ -184,7 +184,7 @@ + + + +- ++ + + + +@@ -208,10 +208,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-2.exp b/pengine/test10/master-2.exp +index ef4095b..6bb01f6 100644 +--- a/pengine/test10/master-2.exp ++++ b/pengine/test10/master-2.exp +@@ -524,7 +524,7 @@ + + + +- ++ + + + +@@ -548,7 +548,7 @@ + + + +- ++ + + + +@@ -578,10 +578,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-3.exp b/pengine/test10/master-3.exp +index d5e0def..cb2b4a3 100644 +--- a/pengine/test10/master-3.exp ++++ b/pengine/test10/master-3.exp +@@ -227,7 +227,7 @@ + + + +- ++ + + + +@@ -251,7 +251,7 @@ + + + +- ++ + + + +@@ -281,10 +281,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-4.exp b/pengine/test10/master-4.exp +index a1e61b3..8ec3c39 100644 +--- a/pengine/test10/master-4.exp ++++ b/pengine/test10/master-4.exp +@@ -302,7 +302,7 @@ + + + +- ++ + + + +@@ -335,7 +335,7 @@ + + + +- ++ + + + +@@ -365,7 +365,7 @@ + + + +- ++ + + + +@@ -392,7 +392,7 @@ + + + +- ++ + + + +@@ -425,16 +425,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-5.exp b/pengine/test10/master-5.exp +index 0196883..1312668 100644 +--- a/pengine/test10/master-5.exp ++++ b/pengine/test10/master-5.exp +@@ -244,7 +244,7 @@ + + + +- ++ + + + +@@ -277,7 +277,7 @@ + + + +- ++ + + + +@@ -307,7 +307,7 @@ + + + +- ++ + + + +@@ -334,7 +334,7 @@ + + + +- ++ + + + +@@ -367,16 +367,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-6.exp b/pengine/test10/master-6.exp +index 381bfd5..b9cdb41 100644 +--- a/pengine/test10/master-6.exp ++++ b/pengine/test10/master-6.exp +@@ -217,7 +217,7 @@ + + + +- ++ + + + +@@ -250,7 +250,7 @@ + + + +- ++ + + + +@@ -277,7 +277,7 @@ + + + +- ++ + + + +@@ -298,7 +298,7 @@ + + + +- ++ + + + +@@ -331,16 +331,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-7.exp b/pengine/test10/master-7.exp +index 4fba172..3f45706 100644 +--- a/pengine/test10/master-7.exp ++++ b/pengine/test10/master-7.exp +@@ -647,7 +647,7 @@ + + + +- ++ + + + +@@ -668,7 +668,7 @@ + + + +- ++ + + + +@@ -692,7 +692,7 @@ + + + +- ++ + + + +@@ -719,13 +719,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-8.exp b/pengine/test10/master-8.exp +index c0cd853..26ba512 100644 +--- a/pengine/test10/master-8.exp ++++ b/pengine/test10/master-8.exp +@@ -703,7 +703,7 @@ + + + +- ++ + + + +@@ -724,7 +724,7 @@ + + + +- ++ + + + +@@ -751,7 +751,7 @@ + + + +- ++ + + + +@@ -778,13 +778,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-9.exp b/pengine/test10/master-9.exp +index 333f272..989f087 100644 +--- a/pengine/test10/master-9.exp ++++ b/pengine/test10/master-9.exp +@@ -190,7 +190,7 @@ + + + +- ++ + + + +@@ -202,7 +202,7 @@ + + + +- ++ + + + +@@ -238,7 +238,7 @@ + + + +- ++ + + + +@@ -277,10 +277,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-depend.exp b/pengine/test10/master-depend.exp +index 87d7d68..d9347f2 100644 +--- a/pengine/test10/master-depend.exp ++++ b/pengine/test10/master-depend.exp +@@ -225,7 +225,7 @@ + + + +- ++ + + + +@@ -261,7 +261,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/master-ordering.exp b/pengine/test10/master-ordering.exp +index 365bb7b..de4fe7e 100644 +--- a/pengine/test10/master-ordering.exp ++++ b/pengine/test10/master-ordering.exp +@@ -446,7 +446,7 @@ + + + +- ++ + + + +@@ -500,7 +500,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/master-probed-score.exp b/pengine/test10/master-probed-score.exp +index 0992921..be0cf84 100644 +--- a/pengine/test10/master-probed-score.exp ++++ b/pengine/test10/master-probed-score.exp +@@ -2752,7 +2752,7 @@ + + + +- ++ + + + +@@ -2818,7 +2818,7 @@ + + + +- ++ + + + +@@ -2893,10 +2893,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/master-reattach.exp b/pengine/test10/master-reattach.exp +index afe989b..edf107e 100644 +--- a/pengine/test10/master-reattach.exp ++++ b/pengine/test10/master-reattach.exp +@@ -46,7 +46,7 @@ + + + +- ++ + + + +@@ -54,7 +54,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/master-stop.exp b/pengine/test10/master-stop.exp +new file mode 100644 +index 0000000..5d9e4c1 +--- /dev/null ++++ b/pengine/test10/master-stop.exp +@@ -0,0 +1,74 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/master-unmanaged-monitor.exp b/pengine/test10/master-unmanaged-monitor.exp +index 2f7c2da..3f650a5 100644 +--- a/pengine/test10/master-unmanaged-monitor.exp ++++ b/pengine/test10/master-unmanaged-monitor.exp +@@ -91,7 +91,7 @@ + + + +- ++ + + + +@@ -109,7 +109,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/migrate-1.exp b/pengine/test10/migrate-1.exp +index 9111b0f..7830a51 100644 +--- a/pengine/test10/migrate-1.exp ++++ b/pengine/test10/migrate-1.exp +@@ -67,7 +67,7 @@ + + + +- ++ + + + +@@ -79,7 +79,7 @@ + + + +- ++ + + + +@@ -93,10 +93,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/migrate-2.exp b/pengine/test10/migrate-2.exp +index 3888892..cf7f123 100644 +--- a/pengine/test10/migrate-2.exp ++++ b/pengine/test10/migrate-2.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/migrate-3.exp b/pengine/test10/migrate-3.exp +index 6c8b5af..6476943 100644 +--- a/pengine/test10/migrate-3.exp ++++ b/pengine/test10/migrate-3.exp +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -51,7 +51,7 @@ + + + +- ++ + + + +@@ -65,10 +65,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/migrate-4.exp b/pengine/test10/migrate-4.exp +index 14ce94d..8cac495 100644 +--- a/pengine/test10/migrate-4.exp ++++ b/pengine/test10/migrate-4.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/migrate-shutdown.exp b/pengine/test10/migrate-shutdown.exp +index b6a4365..e6d068a 100644 +--- a/pengine/test10/migrate-shutdown.exp ++++ b/pengine/test10/migrate-shutdown.exp +@@ -325,7 +325,7 @@ + + + +- ++ + + + +@@ -343,7 +343,7 @@ + + + +- ++ + + + +@@ -355,7 +355,7 @@ + + + +- ++ + + + +@@ -385,7 +385,7 @@ + + + +- ++ + + + +@@ -406,13 +406,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/migrate-start-complex.exp b/pengine/test10/migrate-start-complex.exp +index 7f9e3fa..33e54f6 100644 +--- a/pengine/test10/migrate-start-complex.exp ++++ b/pengine/test10/migrate-start-complex.exp +@@ -241,7 +241,7 @@ + + + +- ++ + + + +@@ -253,7 +253,7 @@ + + + +- ++ + + + +@@ -271,10 +271,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/minimal.exp b/pengine/test10/minimal.exp +index 565452b..ce47557 100644 +--- a/pengine/test10/minimal.exp ++++ b/pengine/test10/minimal.exp +@@ -69,7 +69,7 @@ + + + +- ++ + + + +@@ -84,7 +84,7 @@ + + + +- ++ + + + +@@ -105,10 +105,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/mon-rsc-1.exp b/pengine/test10/mon-rsc-1.exp +index 3b67915..c1cf031 100644 +--- a/pengine/test10/mon-rsc-1.exp ++++ b/pengine/test10/mon-rsc-1.exp +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -57,7 +57,7 @@ + + + +- ++ + + + +@@ -75,10 +75,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/mon-rsc-2.exp b/pengine/test10/mon-rsc-2.exp +index 6a6fd07..317b84c 100644 +--- a/pengine/test10/mon-rsc-2.exp ++++ b/pengine/test10/mon-rsc-2.exp +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -60,7 +60,7 @@ + + + +- ++ + + + +@@ -78,10 +78,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/mon-rsc-3.exp b/pengine/test10/mon-rsc-3.exp +index b32054f..6d93160 100644 +--- a/pengine/test10/mon-rsc-3.exp ++++ b/pengine/test10/mon-rsc-3.exp +@@ -36,7 +36,7 @@ + + + +- ++ + + + +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/mon-rsc-4.exp b/pengine/test10/mon-rsc-4.exp +index 2abfdec..58a64f5 100644 +--- a/pengine/test10/mon-rsc-4.exp ++++ b/pengine/test10/mon-rsc-4.exp +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -60,7 +60,7 @@ + + + +- ++ + + + +@@ -78,10 +78,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/multi1.exp b/pengine/test10/multi1.exp +index 58d9dbf..18fe162 100644 +--- a/pengine/test10/multi1.exp ++++ b/pengine/test10/multi1.exp +@@ -35,7 +35,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/node-maintenance-1.exp b/pengine/test10/node-maintenance-1.exp +index 3c73791..6a24c07 100644 +--- a/pengine/test10/node-maintenance-1.exp ++++ b/pengine/test10/node-maintenance-1.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/node-maintenance-2.exp b/pengine/test10/node-maintenance-2.exp +index 0156835..d8b15e4 100644 +--- a/pengine/test10/node-maintenance-2.exp ++++ b/pengine/test10/node-maintenance-2.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -40,7 +40,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/notify-0.exp b/pengine/test10/notify-0.exp +index 3198a86..626fb59 100644 +--- a/pengine/test10/notify-0.exp ++++ b/pengine/test10/notify-0.exp +@@ -97,7 +97,7 @@ + + + +- ++ + + + +@@ -118,7 +118,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/notify-1.exp b/pengine/test10/notify-1.exp +index 8c290aa..ca02a84 100644 +--- a/pengine/test10/notify-1.exp ++++ b/pengine/test10/notify-1.exp +@@ -263,7 +263,7 @@ + + + +- ++ + + + +@@ -284,7 +284,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/notify-2.exp b/pengine/test10/notify-2.exp +index 8c290aa..ca02a84 100644 +--- a/pengine/test10/notify-2.exp ++++ b/pengine/test10/notify-2.exp +@@ -263,7 +263,7 @@ + + + +- ++ + + + +@@ -284,7 +284,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/notify-3.exp b/pengine/test10/notify-3.exp +index 1615df0..c5708dc 100644 +--- a/pengine/test10/notify-3.exp ++++ b/pengine/test10/notify-3.exp +@@ -438,7 +438,7 @@ + + + +- ++ + + + +@@ -456,7 +456,7 @@ + + + +- ++ + + + +@@ -477,10 +477,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/novell-239082.exp b/pengine/test10/novell-239082.exp +index 3073f7f..538eda6 100644 +--- a/pengine/test10/novell-239082.exp ++++ b/pengine/test10/novell-239082.exp +@@ -454,7 +454,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/novell-252693-2.exp b/pengine/test10/novell-252693-2.exp +index f30cf03..ca3c378 100644 +--- a/pengine/test10/novell-252693-2.exp ++++ b/pengine/test10/novell-252693-2.exp +@@ -696,7 +696,7 @@ + + + +- ++ + + + +@@ -729,7 +729,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/novell-252693-3.exp b/pengine/test10/novell-252693-3.exp +index 48be681..6ebe9a7 100644 +--- a/pengine/test10/novell-252693-3.exp ++++ b/pengine/test10/novell-252693-3.exp +@@ -800,7 +800,7 @@ + + + +- ++ + + + +@@ -833,7 +833,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/novell-252693.exp b/pengine/test10/novell-252693.exp +index 4ab727a..ca4b942 100644 +--- a/pengine/test10/novell-252693.exp ++++ b/pengine/test10/novell-252693.exp +@@ -554,7 +554,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/obsolete-lrm-resource.exp b/pengine/test10/obsolete-lrm-resource.exp +index e56e556..0939b4f 100644 +--- a/pengine/test10/obsolete-lrm-resource.exp ++++ b/pengine/test10/obsolete-lrm-resource.exp +@@ -58,7 +58,7 @@ + + + +- ++ + + + +@@ -76,7 +76,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/one-or-more-0.exp b/pengine/test10/one-or-more-0.exp +index 315c596..01c59c7 100644 +--- a/pengine/test10/one-or-more-0.exp ++++ b/pengine/test10/one-or-more-0.exp +@@ -98,7 +98,7 @@ + + + +- ++ + + + +@@ -125,7 +125,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/one-or-more-1.exp b/pengine/test10/one-or-more-1.exp +index 30e68b2..900fc5d 100644 +--- a/pengine/test10/one-or-more-1.exp ++++ b/pengine/test10/one-or-more-1.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/one-or-more-2.exp b/pengine/test10/one-or-more-2.exp +index d94fd9d..019b8d0 100644 +--- a/pengine/test10/one-or-more-2.exp ++++ b/pengine/test10/one-or-more-2.exp +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -109,7 +109,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/one-or-more-3.exp b/pengine/test10/one-or-more-3.exp +index 2070f51..09776cd 100644 +--- a/pengine/test10/one-or-more-3.exp ++++ b/pengine/test10/one-or-more-3.exp +@@ -50,7 +50,7 @@ + + + +- ++ + + + +@@ -77,7 +77,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/one-or-more-4.exp b/pengine/test10/one-or-more-4.exp +index 002fc27..2496808 100644 +--- a/pengine/test10/one-or-more-4.exp ++++ b/pengine/test10/one-or-more-4.exp +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -109,7 +109,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/one-or-more-5.exp b/pengine/test10/one-or-more-5.exp +index 67d1231..fe94627 100644 +--- a/pengine/test10/one-or-more-5.exp ++++ b/pengine/test10/one-or-more-5.exp +@@ -116,7 +116,7 @@ + + + +- ++ + + + +@@ -149,7 +149,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/order-mandatory.exp b/pengine/test10/order-mandatory.exp +index 7aa18d2..166e578 100644 +--- a/pengine/test10/order-mandatory.exp ++++ b/pengine/test10/order-mandatory.exp +@@ -44,7 +44,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/order-optional-keyword.exp b/pengine/test10/order-optional-keyword.exp +index d398916..aa3da4e 100644 +--- a/pengine/test10/order-optional-keyword.exp ++++ b/pengine/test10/order-optional-keyword.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/order-optional.exp b/pengine/test10/order-optional.exp +index d14157b..76126ad 100644 +--- a/pengine/test10/order-optional.exp ++++ b/pengine/test10/order-optional.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/order-required.exp b/pengine/test10/order-required.exp +index 2d871fa..6fc0257 100644 +--- a/pengine/test10/order-required.exp ++++ b/pengine/test10/order-required.exp +@@ -44,7 +44,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/order-wrong-kind.exp b/pengine/test10/order-wrong-kind.exp +index 68796f8..426153d 100644 +--- a/pengine/test10/order-wrong-kind.exp ++++ b/pengine/test10/order-wrong-kind.exp +@@ -35,7 +35,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/order1.exp b/pengine/test10/order1.exp +index 6db9031..d69e2f6 100644 +--- a/pengine/test10/order1.exp ++++ b/pengine/test10/order1.exp +@@ -100,7 +100,7 @@ + + + +- ++ + + + +@@ -118,7 +118,7 @@ + + + +- ++ + + + +@@ -142,10 +142,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/order2.exp b/pengine/test10/order2.exp +index 9041ba0..9eacda0 100644 +--- a/pengine/test10/order2.exp ++++ b/pengine/test10/order2.exp +@@ -134,7 +134,7 @@ + + + +- ++ + + + +@@ -155,7 +155,7 @@ + + + +- ++ + + + +@@ -182,10 +182,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/order3.exp b/pengine/test10/order3.exp +index 215b49c..0ce0585 100644 +--- a/pengine/test10/order3.exp ++++ b/pengine/test10/order3.exp +@@ -171,7 +171,7 @@ + + + +- ++ + + + +@@ -192,7 +192,7 @@ + + + +- ++ + + + +@@ -206,10 +206,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/order4.exp b/pengine/test10/order4.exp +index 6db9031..d69e2f6 100644 +--- a/pengine/test10/order4.exp ++++ b/pengine/test10/order4.exp +@@ -100,7 +100,7 @@ + + + +- ++ + + + +@@ -118,7 +118,7 @@ + + + +- ++ + + + +@@ -142,10 +142,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/order5.exp b/pengine/test10/order5.exp +index eceee4f..970f13a 100644 +--- a/pengine/test10/order5.exp ++++ b/pengine/test10/order5.exp +@@ -189,7 +189,7 @@ + + + +- ++ + + + +@@ -210,7 +210,7 @@ + + + +- ++ + + + +@@ -237,10 +237,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/order6.exp b/pengine/test10/order6.exp +index 7ebade6..e82ec71 100644 +--- a/pengine/test10/order6.exp ++++ b/pengine/test10/order6.exp +@@ -189,7 +189,7 @@ + + + +- ++ + + + +@@ -210,7 +210,7 @@ + + + +- ++ + + + +@@ -237,10 +237,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/order7.exp b/pengine/test10/order7.exp +index cc7cf86..abfbe43 100644 +--- a/pengine/test10/order7.exp ++++ b/pengine/test10/order7.exp +@@ -76,7 +76,7 @@ + + + +- ++ + + + +@@ -103,7 +103,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ordered-set-basic-startup.exp b/pengine/test10/ordered-set-basic-startup.exp +index 4704f66..22d6dbb 100644 +--- a/pengine/test10/ordered-set-basic-startup.exp ++++ b/pengine/test10/ordered-set-basic-startup.exp +@@ -68,7 +68,7 @@ + + + +- ++ + + + +@@ -101,7 +101,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/orphan-0.exp b/pengine/test10/orphan-0.exp +index afccffb..ca56f40 100644 +--- a/pengine/test10/orphan-0.exp ++++ b/pengine/test10/orphan-0.exp +@@ -118,7 +118,7 @@ + + + +- ++ + + + +@@ -139,7 +139,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -172,7 +172,7 @@ + + + +- ++ + + + +@@ -196,16 +196,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/orphan-1.exp b/pengine/test10/orphan-1.exp +index b3a6470..6e76cba 100644 +--- a/pengine/test10/orphan-1.exp ++++ b/pengine/test10/orphan-1.exp +@@ -162,7 +162,7 @@ + + + +- ++ + + + +@@ -183,7 +183,7 @@ + + + +- ++ + + + +@@ -201,7 +201,7 @@ + + + +- ++ + + + +@@ -216,7 +216,7 @@ + + + +- ++ + + + +@@ -240,16 +240,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/orphan-2.exp b/pengine/test10/orphan-2.exp +index ad9ad59..516422c 100644 +--- a/pengine/test10/orphan-2.exp ++++ b/pengine/test10/orphan-2.exp +@@ -180,7 +180,7 @@ + + + +- ++ + + + +@@ -201,7 +201,7 @@ + + + +- ++ + + + +@@ -219,7 +219,7 @@ + + + +- ++ + + + +@@ -234,7 +234,7 @@ + + + +- ++ + + + +@@ -276,16 +276,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/params-0.exp b/pengine/test10/params-0.exp +index 1489e3e..3b036ab 100644 +--- a/pengine/test10/params-0.exp ++++ b/pengine/test10/params-0.exp +@@ -136,7 +136,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -178,7 +178,7 @@ + + + +- ++ + + + +@@ -196,7 +196,7 @@ + + + +- ++ + + + +@@ -223,16 +223,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/params-1.exp b/pengine/test10/params-1.exp +index 09470ab..6b6ef2d 100644 +--- a/pengine/test10/params-1.exp ++++ b/pengine/test10/params-1.exp +@@ -205,7 +205,7 @@ + + + +- ++ + + + +@@ -226,7 +226,7 @@ + + + +- ++ + + + +@@ -247,7 +247,7 @@ + + + +- ++ + + + +@@ -265,7 +265,7 @@ + + + +- ++ + + + +@@ -292,16 +292,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/params-2.exp b/pengine/test10/params-2.exp +index 7dd199b..fbac4a2 100644 +--- a/pengine/test10/params-2.exp ++++ b/pengine/test10/params-2.exp +@@ -134,7 +134,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +@@ -176,7 +176,7 @@ + + + +- ++ + + + +@@ -197,13 +197,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/params-4.exp b/pengine/test10/params-4.exp +index 50d9b7f..c5987e02 100644 +--- a/pengine/test10/params-4.exp ++++ b/pengine/test10/params-4.exp +@@ -189,7 +189,7 @@ + + + +- ++ + + + +@@ -210,7 +210,7 @@ + + + +- ++ + + + +@@ -231,7 +231,7 @@ + + + +- ++ + + + +@@ -249,7 +249,7 @@ + + + +- ++ + + + +@@ -276,16 +276,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/params-5.exp b/pengine/test10/params-5.exp +index 85736b0..01231e0 100644 +--- a/pengine/test10/params-5.exp ++++ b/pengine/test10/params-5.exp +@@ -205,7 +205,7 @@ + + + +- ++ + + + +@@ -226,7 +226,7 @@ + + + +- ++ + + + +@@ -247,7 +247,7 @@ + + + +- ++ + + + +@@ -265,7 +265,7 @@ + + + +- ++ + + + +@@ -292,16 +292,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/per-node-attrs.exp b/pengine/test10/per-node-attrs.exp +index 1e38557..aff6b4f 100644 +--- a/pengine/test10/per-node-attrs.exp ++++ b/pengine/test10/per-node-attrs.exp +@@ -41,7 +41,7 @@ + + + +- ++ + + + +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -65,7 +65,7 @@ + + + +- ++ + + + +@@ -83,13 +83,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/placement-capacity.exp b/pengine/test10/placement-capacity.exp +index f70c8c5..406d502 100644 +--- a/pengine/test10/placement-capacity.exp ++++ b/pengine/test10/placement-capacity.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/placement-location.exp b/pengine/test10/placement-location.exp +index f70c8c5..406d502 100644 +--- a/pengine/test10/placement-location.exp ++++ b/pengine/test10/placement-location.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/placement-priority.exp b/pengine/test10/placement-priority.exp +index c72759e..557f033 100644 +--- a/pengine/test10/placement-priority.exp ++++ b/pengine/test10/placement-priority.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/placement-stickiness.exp b/pengine/test10/placement-stickiness.exp +index f70c8c5..406d502 100644 +--- a/pengine/test10/placement-stickiness.exp ++++ b/pengine/test10/placement-stickiness.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/probe-0.exp b/pengine/test10/probe-0.exp +index c1310d9..562e855 100644 +--- a/pengine/test10/probe-0.exp ++++ b/pengine/test10/probe-0.exp +@@ -160,7 +160,7 @@ + + + +- ++ + + + +@@ -172,7 +172,7 @@ + + + +- ++ + + + +@@ -190,10 +190,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/probe-1.exp b/pengine/test10/probe-1.exp +index a62f2f3..37d49e3 100644 +--- a/pengine/test10/probe-1.exp ++++ b/pengine/test10/probe-1.exp +@@ -36,7 +36,7 @@ + + + +- ++ + + + +@@ -54,7 +54,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/probe-4.exp b/pengine/test10/probe-4.exp +index 819dd5c..4f2d325 100644 +--- a/pengine/test10/probe-4.exp ++++ b/pengine/test10/probe-4.exp +@@ -33,7 +33,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/probe-timeout.exp b/pengine/test10/probe-timeout.exp +index 2458e7b..baf2440 100644 +--- a/pengine/test10/probe-timeout.exp ++++ b/pengine/test10/probe-timeout.exp +@@ -115,7 +115,7 @@ + + + +- ++ + + + +@@ -130,7 +130,7 @@ + + + +- ++ + + + +@@ -151,10 +151,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/quorum-1.exp b/pengine/test10/quorum-1.exp +index f2de2c8..97dec69 100644 +--- a/pengine/test10/quorum-1.exp ++++ b/pengine/test10/quorum-1.exp +@@ -79,7 +79,7 @@ + + + +- ++ + + + +@@ -97,7 +97,7 @@ + + + +- ++ + + + +@@ -115,10 +115,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/quorum-2.exp b/pengine/test10/quorum-2.exp +index ed60d5e..ac868ff 100644 +--- a/pengine/test10/quorum-2.exp ++++ b/pengine/test10/quorum-2.exp +@@ -66,7 +66,7 @@ + + + +- ++ + + + +@@ -84,7 +84,7 @@ + + + +- ++ + + + +@@ -102,10 +102,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/quorum-3.exp b/pengine/test10/quorum-3.exp +index 175b11d..8ddd88c 100644 +--- a/pengine/test10/quorum-3.exp ++++ b/pengine/test10/quorum-3.exp +@@ -63,7 +63,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -99,10 +99,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/quorum-4.exp b/pengine/test10/quorum-4.exp +index 3b90816..42ba483 100644 +--- a/pengine/test10/quorum-4.exp ++++ b/pengine/test10/quorum-4.exp +@@ -36,7 +36,7 @@ + + + +- ++ + + + +@@ -54,7 +54,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/quorum-5.exp b/pengine/test10/quorum-5.exp +index b2844a9..ee9e28e 100644 +--- a/pengine/test10/quorum-5.exp ++++ b/pengine/test10/quorum-5.exp +@@ -106,7 +106,7 @@ + + + +- ++ + + + +@@ -127,7 +127,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/quorum-6.exp b/pengine/test10/quorum-6.exp +index 685a514..0c46abe 100644 +--- a/pengine/test10/quorum-6.exp ++++ b/pengine/test10/quorum-6.exp +@@ -125,7 +125,7 @@ + + + +- ++ + + + +@@ -164,7 +164,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-1.exp b/pengine/test10/rec-node-1.exp +index f26fd66..2cc4595 100644 +--- a/pengine/test10/rec-node-1.exp ++++ b/pengine/test10/rec-node-1.exp +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -66,7 +66,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-10.exp b/pengine/test10/rec-node-10.exp +index 741bcf1..60ab810 100644 +--- a/pengine/test10/rec-node-10.exp ++++ b/pengine/test10/rec-node-10.exp +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-11.exp b/pengine/test10/rec-node-11.exp +index 173df21..f441ce8 100644 +--- a/pengine/test10/rec-node-11.exp ++++ b/pengine/test10/rec-node-11.exp +@@ -245,7 +245,7 @@ + + + +- ++ + + + +@@ -269,7 +269,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-12.exp b/pengine/test10/rec-node-12.exp +index 9732907..1ac908f 100644 +--- a/pengine/test10/rec-node-12.exp ++++ b/pengine/test10/rec-node-12.exp +@@ -534,7 +534,7 @@ + + + +- ++ + + + +@@ -570,7 +570,7 @@ + + + +- ++ + + + +@@ -606,7 +606,7 @@ + + + +- ++ + + + +@@ -648,13 +648,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-node-15.exp b/pengine/test10/rec-node-15.exp +index 3c79d65..e1330a6 100644 +--- a/pengine/test10/rec-node-15.exp ++++ b/pengine/test10/rec-node-15.exp +@@ -465,7 +465,7 @@ + + + +- ++ + + + +@@ -477,7 +477,7 @@ + + + +- ++ + + + +@@ -495,10 +495,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-node-2.exp b/pengine/test10/rec-node-2.exp +index 901eded..fd04c37 100644 +--- a/pengine/test10/rec-node-2.exp ++++ b/pengine/test10/rec-node-2.exp +@@ -266,7 +266,7 @@ + + + +- ++ + + + +@@ -302,7 +302,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-3.exp b/pengine/test10/rec-node-3.exp +index f26fd66..2cc4595 100644 +--- a/pengine/test10/rec-node-3.exp ++++ b/pengine/test10/rec-node-3.exp +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -66,7 +66,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-4.exp b/pengine/test10/rec-node-4.exp +index 1e98f4d..2cbfecb 100644 +--- a/pengine/test10/rec-node-4.exp ++++ b/pengine/test10/rec-node-4.exp +@@ -132,7 +132,7 @@ + + + +- ++ + + + +@@ -156,7 +156,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-5.exp b/pengine/test10/rec-node-5.exp +index f26fd66..2cc4595 100644 +--- a/pengine/test10/rec-node-5.exp ++++ b/pengine/test10/rec-node-5.exp +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -66,7 +66,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-6.exp b/pengine/test10/rec-node-6.exp +index 1e98f4d..2cbfecb 100644 +--- a/pengine/test10/rec-node-6.exp ++++ b/pengine/test10/rec-node-6.exp +@@ -132,7 +132,7 @@ + + + +- ++ + + + +@@ -156,7 +156,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-7.exp b/pengine/test10/rec-node-7.exp +index 1e98f4d..2cbfecb 100644 +--- a/pengine/test10/rec-node-7.exp ++++ b/pengine/test10/rec-node-7.exp +@@ -132,7 +132,7 @@ + + + +- ++ + + + +@@ -156,7 +156,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-8.exp b/pengine/test10/rec-node-8.exp +index 00e58ef..1237a82 100644 +--- a/pengine/test10/rec-node-8.exp ++++ b/pengine/test10/rec-node-8.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-9.exp b/pengine/test10/rec-node-9.exp +index e2192c8..9868f6f 100644 +--- a/pengine/test10/rec-node-9.exp ++++ b/pengine/test10/rec-node-9.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -40,7 +40,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-0.exp b/pengine/test10/rec-rsc-0.exp +index 93bfb15..8caa0c0 100644 +--- a/pengine/test10/rec-rsc-0.exp ++++ b/pengine/test10/rec-rsc-0.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-1.exp b/pengine/test10/rec-rsc-1.exp +index 3825203..25030c6 100644 +--- a/pengine/test10/rec-rsc-1.exp ++++ b/pengine/test10/rec-rsc-1.exp +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -51,7 +51,7 @@ + + + +- ++ + + + +@@ -65,10 +65,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-2.exp b/pengine/test10/rec-rsc-2.exp +index 7db7a0e..5f64c90 100644 +--- a/pengine/test10/rec-rsc-2.exp ++++ b/pengine/test10/rec-rsc-2.exp +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +@@ -78,10 +78,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-3.exp b/pengine/test10/rec-rsc-3.exp +index 80a6252..7faada9 100644 +--- a/pengine/test10/rec-rsc-3.exp ++++ b/pengine/test10/rec-rsc-3.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-4.exp b/pengine/test10/rec-rsc-4.exp +index 78a90db..c186199 100644 +--- a/pengine/test10/rec-rsc-4.exp ++++ b/pengine/test10/rec-rsc-4.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +@@ -36,10 +36,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-5.exp b/pengine/test10/rec-rsc-5.exp +index 69a85d1..95a3558 100644 +--- a/pengine/test10/rec-rsc-5.exp ++++ b/pengine/test10/rec-rsc-5.exp +@@ -132,7 +132,7 @@ + + + +- ++ + + + +@@ -156,7 +156,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-6.exp b/pengine/test10/rec-rsc-6.exp +index 58d9dbf..18fe162 100644 +--- a/pengine/test10/rec-rsc-6.exp ++++ b/pengine/test10/rec-rsc-6.exp +@@ -35,7 +35,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-7.exp b/pengine/test10/rec-rsc-7.exp +index eb2335e..1454ff3 100644 +--- a/pengine/test10/rec-rsc-7.exp ++++ b/pengine/test10/rec-rsc-7.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-8.exp b/pengine/test10/rec-rsc-8.exp +index 8b8bce3..c53091f 100644 +--- a/pengine/test10/rec-rsc-8.exp ++++ b/pengine/test10/rec-rsc-8.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-9.exp b/pengine/test10/rec-rsc-9.exp +index 4d0e70b..5a48606 100644 +--- a/pengine/test10/rec-rsc-9.exp ++++ b/pengine/test10/rec-rsc-9.exp +@@ -194,7 +194,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/remote-startup-probes.exp b/pengine/test10/remote-startup-probes.exp +new file mode 100644 +index 0000000..3c0f4e5 +--- /dev/null ++++ b/pengine/test10/remote-startup-probes.exp +@@ -0,0 +1,214 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/remote-startup.exp b/pengine/test10/remote-startup.exp +new file mode 100644 +index 0000000..82fd243 +--- /dev/null ++++ b/pengine/test10/remote-startup.exp +@@ -0,0 +1,246 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/reprobe-target_rc.exp b/pengine/test10/reprobe-target_rc.exp +index 9c09de7..f3b5a38 100644 +--- a/pengine/test10/reprobe-target_rc.exp ++++ b/pengine/test10/reprobe-target_rc.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rsc-maintenance.exp b/pengine/test10/rsc-maintenance.exp +new file mode 100644 +index 0000000..146519c +--- /dev/null ++++ b/pengine/test10/rsc-maintenance.exp +@@ -0,0 +1,36 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/rsc-sets-clone-1.exp b/pengine/test10/rsc-sets-clone-1.exp +index 7e441c3..c338d7a 100644 +--- a/pengine/test10/rsc-sets-clone-1.exp ++++ b/pengine/test10/rsc-sets-clone-1.exp +@@ -476,7 +476,7 @@ + + + +- ++ + + + +@@ -536,7 +536,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rsc-sets-clone.exp b/pengine/test10/rsc-sets-clone.exp +index 042eff5..14d9ccb 100644 +--- a/pengine/test10/rsc-sets-clone.exp ++++ b/pengine/test10/rsc-sets-clone.exp +@@ -88,7 +88,7 @@ + + + +- ++ + + + +@@ -96,7 +96,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rsc-sets-master.exp b/pengine/test10/rsc-sets-master.exp +index 18609c9..f801973 100644 +--- a/pengine/test10/rsc-sets-master.exp ++++ b/pengine/test10/rsc-sets-master.exp +@@ -205,7 +205,7 @@ + + + +- ++ + + + +@@ -213,7 +213,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rsc-sets-seq-false.exp b/pengine/test10/rsc-sets-seq-false.exp +index 36109e4..76f5156 100644 +--- a/pengine/test10/rsc-sets-seq-false.exp ++++ b/pengine/test10/rsc-sets-seq-false.exp +@@ -166,7 +166,7 @@ + + + +- ++ + + + +@@ -174,7 +174,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rsc-sets-seq-true.exp b/pengine/test10/rsc-sets-seq-true.exp +index be82497..c37c9a9 100644 +--- a/pengine/test10/rsc-sets-seq-true.exp ++++ b/pengine/test10/rsc-sets-seq-true.exp +@@ -168,7 +168,7 @@ + + + +- ++ + + + +@@ -176,7 +176,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rsc_dep1.exp b/pengine/test10/rsc_dep1.exp +index 9b61a44..96b998a 100644 +--- a/pengine/test10/rsc_dep1.exp ++++ b/pengine/test10/rsc_dep1.exp +@@ -63,7 +63,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -99,10 +99,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rsc_dep10.exp b/pengine/test10/rsc_dep10.exp +index a8f0bdf..727e20d 100644 +--- a/pengine/test10/rsc_dep10.exp ++++ b/pengine/test10/rsc_dep10.exp +@@ -50,7 +50,7 @@ + + + +- ++ + + + +@@ -65,7 +65,7 @@ + + + +- ++ + + + +@@ -86,10 +86,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rsc_dep2.exp b/pengine/test10/rsc_dep2.exp +index 9040091..0e861e6 100644 +--- a/pengine/test10/rsc_dep2.exp ++++ b/pengine/test10/rsc_dep2.exp +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -99,7 +99,7 @@ + + + +- ++ + + + +@@ -123,10 +123,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rsc_dep3.exp b/pengine/test10/rsc_dep3.exp +index 15bf782..4c78ffc 100644 +--- a/pengine/test10/rsc_dep3.exp ++++ b/pengine/test10/rsc_dep3.exp +@@ -63,7 +63,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -99,10 +99,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rsc_dep4.exp b/pengine/test10/rsc_dep4.exp +index 6b4eb82..1ad9e3e 100644 +--- a/pengine/test10/rsc_dep4.exp ++++ b/pengine/test10/rsc_dep4.exp +@@ -110,7 +110,7 @@ + + + +- ++ + + + +@@ -131,7 +131,7 @@ + + + +- ++ + + + +@@ -152,10 +152,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rsc_dep5.exp b/pengine/test10/rsc_dep5.exp +index 64ba084..9119f77 100644 +--- a/pengine/test10/rsc_dep5.exp ++++ b/pengine/test10/rsc_dep5.exp +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -99,7 +99,7 @@ + + + +- ++ + + + +@@ -123,10 +123,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rsc_dep7.exp b/pengine/test10/rsc_dep7.exp +index 8da56fe..64f1140 100644 +--- a/pengine/test10/rsc_dep7.exp ++++ b/pengine/test10/rsc_dep7.exp +@@ -94,7 +94,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +@@ -136,10 +136,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rsc_dep8.exp b/pengine/test10/rsc_dep8.exp +index 9040091..0e861e6 100644 +--- a/pengine/test10/rsc_dep8.exp ++++ b/pengine/test10/rsc_dep8.exp +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -99,7 +99,7 @@ + + + +- ++ + + + +@@ -123,10 +123,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/simple11.exp b/pengine/test10/simple11.exp +index 315a7fd..f94ad53 100644 +--- a/pengine/test10/simple11.exp ++++ b/pengine/test10/simple11.exp +@@ -63,7 +63,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -99,10 +99,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/simple12.exp b/pengine/test10/simple12.exp +index 2b0a9f8..fda3c50 100644 +--- a/pengine/test10/simple12.exp ++++ b/pengine/test10/simple12.exp +@@ -63,7 +63,7 @@ + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -99,10 +99,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/simple2.exp b/pengine/test10/simple2.exp +index 99b81e7..b6e55a2 100644 +--- a/pengine/test10/simple2.exp ++++ b/pengine/test10/simple2.exp +@@ -32,7 +32,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +@@ -62,10 +62,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/simple3.exp b/pengine/test10/simple3.exp +index 6e4c8cb..2ff55a8 100644 +--- a/pengine/test10/simple3.exp ++++ b/pengine/test10/simple3.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/simple4.exp b/pengine/test10/simple4.exp +index 73c62e6..8702eab 100644 +--- a/pengine/test10/simple4.exp ++++ b/pengine/test10/simple4.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/simple6.exp b/pengine/test10/simple6.exp +index 79d230d..785c04c 100644 +--- a/pengine/test10/simple6.exp ++++ b/pengine/test10/simple6.exp +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -57,7 +57,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/simple7.exp b/pengine/test10/simple7.exp +index ed0c1db..5cbf960 100644 +--- a/pengine/test10/simple7.exp ++++ b/pengine/test10/simple7.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -22,7 +22,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/simple8.exp b/pengine/test10/simple8.exp +index 0344648..4e8f09b 100644 +--- a/pengine/test10/simple8.exp ++++ b/pengine/test10/simple8.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -58,7 +58,7 @@ + + + +- ++ + + + +@@ -72,10 +72,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/stonith-3.exp b/pengine/test10/stonith-3.exp +index 49e6e81..c4c1e32 100644 +--- a/pengine/test10/stonith-3.exp ++++ b/pengine/test10/stonith-3.exp +@@ -137,7 +137,7 @@ + + + +- ++ + + + +@@ -158,7 +158,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stonith-4.exp b/pengine/test10/stonith-4.exp +index 2cfffc0..2be0a96 100644 +--- a/pengine/test10/stonith-4.exp ++++ b/pengine/test10/stonith-4.exp +@@ -79,13 +79,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/stop-failure-no-fencing.exp b/pengine/test10/stop-failure-no-fencing.exp +index e9d94b8..52fe598 100644 +--- a/pengine/test10/stop-failure-no-fencing.exp ++++ b/pengine/test10/stop-failure-no-fencing.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stop-failure-with-fencing.exp b/pengine/test10/stop-failure-with-fencing.exp +index 335ac37..8a467b4 100644 +--- a/pengine/test10/stop-failure-with-fencing.exp ++++ b/pengine/test10/stop-failure-with-fencing.exp +@@ -72,7 +72,7 @@ + + + +- ++ + + + +@@ -90,7 +90,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-00.exp b/pengine/test10/stopped-monitor-00.exp +index 2788366..c27f870 100644 +--- a/pengine/test10/stopped-monitor-00.exp ++++ b/pengine/test10/stopped-monitor-00.exp +@@ -58,7 +58,7 @@ + + + +- ++ + + + +@@ -70,7 +70,7 @@ + + + +- ++ + + + +@@ -88,10 +88,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-01.exp b/pengine/test10/stopped-monitor-01.exp +index 04e44ee..5ca1496 100644 +--- a/pengine/test10/stopped-monitor-01.exp ++++ b/pengine/test10/stopped-monitor-01.exp +@@ -36,7 +36,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-02.exp b/pengine/test10/stopped-monitor-02.exp +index 236d8ec..a735b0f 100644 +--- a/pengine/test10/stopped-monitor-02.exp ++++ b/pengine/test10/stopped-monitor-02.exp +@@ -61,7 +61,7 @@ + + + +- ++ + + + +@@ -69,7 +69,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-03.exp b/pengine/test10/stopped-monitor-03.exp +index 9598ad7..92a5f57 100644 +--- a/pengine/test10/stopped-monitor-03.exp ++++ b/pengine/test10/stopped-monitor-03.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-04.exp b/pengine/test10/stopped-monitor-04.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/stopped-monitor-04.exp ++++ b/pengine/test10/stopped-monitor-04.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-05.exp b/pengine/test10/stopped-monitor-05.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/stopped-monitor-05.exp ++++ b/pengine/test10/stopped-monitor-05.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-06.exp b/pengine/test10/stopped-monitor-06.exp +index ba6632b..0f663b1 100644 +--- a/pengine/test10/stopped-monitor-06.exp ++++ b/pengine/test10/stopped-monitor-06.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-07.exp b/pengine/test10/stopped-monitor-07.exp +index ba6632b..0f663b1 100644 +--- a/pengine/test10/stopped-monitor-07.exp ++++ b/pengine/test10/stopped-monitor-07.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-08.exp b/pengine/test10/stopped-monitor-08.exp +index d1da812..a291c47 100644 +--- a/pengine/test10/stopped-monitor-08.exp ++++ b/pengine/test10/stopped-monitor-08.exp +@@ -61,7 +61,7 @@ + + + +- ++ + + + +@@ -69,7 +69,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-09.exp b/pengine/test10/stopped-monitor-09.exp +index ba6632b..0f663b1 100644 +--- a/pengine/test10/stopped-monitor-09.exp ++++ b/pengine/test10/stopped-monitor-09.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-10.exp b/pengine/test10/stopped-monitor-10.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/stopped-monitor-10.exp ++++ b/pengine/test10/stopped-monitor-10.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-11.exp b/pengine/test10/stopped-monitor-11.exp +index ba6632b..0f663b1 100644 +--- a/pengine/test10/stopped-monitor-11.exp ++++ b/pengine/test10/stopped-monitor-11.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-12.exp b/pengine/test10/stopped-monitor-12.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/stopped-monitor-12.exp ++++ b/pengine/test10/stopped-monitor-12.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-20.exp b/pengine/test10/stopped-monitor-20.exp +index 0d1c05e..d4a4076 100644 +--- a/pengine/test10/stopped-monitor-20.exp ++++ b/pengine/test10/stopped-monitor-20.exp +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -57,7 +57,7 @@ + + + +- ++ + + + +@@ -75,10 +75,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-21.exp b/pengine/test10/stopped-monitor-21.exp +index 39054cf..0a912fe 100644 +--- a/pengine/test10/stopped-monitor-21.exp ++++ b/pengine/test10/stopped-monitor-21.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-22.exp b/pengine/test10/stopped-monitor-22.exp +index e5dc1de..5a3476e 100644 +--- a/pengine/test10/stopped-monitor-22.exp ++++ b/pengine/test10/stopped-monitor-22.exp +@@ -45,7 +45,7 @@ + + + +- ++ + + + +@@ -53,7 +53,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-23.exp b/pengine/test10/stopped-monitor-23.exp +index eb42407..75bfca2 100644 +--- a/pengine/test10/stopped-monitor-23.exp ++++ b/pengine/test10/stopped-monitor-23.exp +@@ -36,7 +36,7 @@ + + + +- ++ + + + +@@ -44,7 +44,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-24.exp b/pengine/test10/stopped-monitor-24.exp +index ba6632b..0f663b1 100644 +--- a/pengine/test10/stopped-monitor-24.exp ++++ b/pengine/test10/stopped-monitor-24.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-25.exp b/pengine/test10/stopped-monitor-25.exp +index 1cff5b2..cc62d3f 100644 +--- a/pengine/test10/stopped-monitor-25.exp ++++ b/pengine/test10/stopped-monitor-25.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-26.exp b/pengine/test10/stopped-monitor-26.exp +index ba6632b..0f663b1 100644 +--- a/pengine/test10/stopped-monitor-26.exp ++++ b/pengine/test10/stopped-monitor-26.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-27.exp b/pengine/test10/stopped-monitor-27.exp +index 1cff5b2..cc62d3f 100644 +--- a/pengine/test10/stopped-monitor-27.exp ++++ b/pengine/test10/stopped-monitor-27.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-30.exp b/pengine/test10/stopped-monitor-30.exp +index 2dce65f..fb3c88f 100644 +--- a/pengine/test10/stopped-monitor-30.exp ++++ b/pengine/test10/stopped-monitor-30.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -35,7 +35,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -57,13 +57,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/stopped-monitor-31.exp b/pengine/test10/stopped-monitor-31.exp +index 394cbc7..e51c78d 100644 +--- a/pengine/test10/stopped-monitor-31.exp ++++ b/pengine/test10/stopped-monitor-31.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -35,7 +35,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -57,13 +57,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/systemhealth2.exp b/pengine/test10/systemhealth2.exp +index 68b5078..95d8847 100644 +--- a/pengine/test10/systemhealth2.exp ++++ b/pengine/test10/systemhealth2.exp +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealth3.exp b/pengine/test10/systemhealth3.exp +index 68b5078..95d8847 100644 +--- a/pengine/test10/systemhealth3.exp ++++ b/pengine/test10/systemhealth3.exp +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthm2.exp b/pengine/test10/systemhealthm2.exp +index 68b5078..95d8847 100644 +--- a/pengine/test10/systemhealthm2.exp ++++ b/pengine/test10/systemhealthm2.exp +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthm3.exp b/pengine/test10/systemhealthm3.exp +index 7246b5b..4100342 100644 +--- a/pengine/test10/systemhealthm3.exp ++++ b/pengine/test10/systemhealthm3.exp +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthn2.exp b/pengine/test10/systemhealthn2.exp +index 68b5078..95d8847 100644 +--- a/pengine/test10/systemhealthn2.exp ++++ b/pengine/test10/systemhealthn2.exp +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthn3.exp b/pengine/test10/systemhealthn3.exp +index 68b5078..95d8847 100644 +--- a/pengine/test10/systemhealthn3.exp ++++ b/pengine/test10/systemhealthn3.exp +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealtho2.exp b/pengine/test10/systemhealtho2.exp +index 7246b5b..4100342 100644 +--- a/pengine/test10/systemhealtho2.exp ++++ b/pengine/test10/systemhealtho2.exp +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealtho3.exp b/pengine/test10/systemhealtho3.exp +index 7246b5b..4100342 100644 +--- a/pengine/test10/systemhealtho3.exp ++++ b/pengine/test10/systemhealtho3.exp +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthp2.exp b/pengine/test10/systemhealthp2.exp +index e3ba25d..77729c4 100644 +--- a/pengine/test10/systemhealthp2.exp ++++ b/pengine/test10/systemhealthp2.exp +@@ -106,7 +106,7 @@ + + + +- ++ + + + +@@ -130,7 +130,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthp3.exp b/pengine/test10/systemhealthp3.exp +index 7246b5b..4100342 100644 +--- a/pengine/test10/systemhealthp3.exp ++++ b/pengine/test10/systemhealthp3.exp +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/target-0.exp b/pengine/test10/target-0.exp +index 1233c09..825d9ae 100644 +--- a/pengine/test10/target-0.exp ++++ b/pengine/test10/target-0.exp +@@ -136,7 +136,7 @@ + + + +- ++ + + + +@@ -157,7 +157,7 @@ + + + +- ++ + + + +@@ -178,7 +178,7 @@ + + + +- ++ + + + +@@ -196,7 +196,7 @@ + + + +- ++ + + + +@@ -223,16 +223,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/target-1.exp b/pengine/test10/target-1.exp +index 15d6a54..06ac73d 100644 +--- a/pengine/test10/target-1.exp ++++ b/pengine/test10/target-1.exp +@@ -122,7 +122,7 @@ + + + +- ++ + + + +@@ -140,7 +140,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +@@ -176,7 +176,7 @@ + + + +- ++ + + + +@@ -200,16 +200,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/target-2.exp b/pengine/test10/target-2.exp +index 94e1e86..a723118 100644 +--- a/pengine/test10/target-2.exp ++++ b/pengine/test10/target-2.exp +@@ -149,7 +149,7 @@ + + + +- ++ + + + +@@ -170,7 +170,7 @@ + + + +- ++ + + + +@@ -191,7 +191,7 @@ + + + +- ++ + + + +@@ -209,7 +209,7 @@ + + + +- ++ + + + +@@ -236,16 +236,16 @@ + + + +- ++ + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-1.exp b/pengine/test10/template-1.exp +index a65f3c5..0fef2b0 100644 +--- a/pengine/test10/template-1.exp ++++ b/pengine/test10/template-1.exp +@@ -66,7 +66,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -102,10 +102,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-2.exp b/pengine/test10/template-2.exp +index a206d20..0fd08cb 100644 +--- a/pengine/test10/template-2.exp ++++ b/pengine/test10/template-2.exp +@@ -66,7 +66,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -102,10 +102,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-3.exp b/pengine/test10/template-3.exp +index f76a695..6b65404 100644 +--- a/pengine/test10/template-3.exp ++++ b/pengine/test10/template-3.exp +@@ -121,7 +121,7 @@ + + + +- ++ + + + +@@ -136,7 +136,7 @@ + + + +- ++ + + + +@@ -157,10 +157,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-clone-group.exp b/pengine/test10/template-clone-group.exp +index 7d06925..2ec6921 100644 +--- a/pengine/test10/template-clone-group.exp ++++ b/pengine/test10/template-clone-group.exp +@@ -193,7 +193,7 @@ + + + +- ++ + + + +@@ -208,7 +208,7 @@ + + + +- ++ + + + +@@ -229,10 +229,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-clone-primitive.exp b/pengine/test10/template-clone-primitive.exp +index b631908..151edf6 100644 +--- a/pengine/test10/template-clone-primitive.exp ++++ b/pengine/test10/template-clone-primitive.exp +@@ -77,7 +77,7 @@ + + + +- ++ + + + +@@ -89,7 +89,7 @@ + + + +- ++ + + + +@@ -107,10 +107,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-coloc-1.exp b/pengine/test10/template-coloc-1.exp +index 871e4e1..96fce66 100644 +--- a/pengine/test10/template-coloc-1.exp ++++ b/pengine/test10/template-coloc-1.exp +@@ -125,7 +125,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +@@ -173,10 +173,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-coloc-2.exp b/pengine/test10/template-coloc-2.exp +index 871e4e1..96fce66 100644 +--- a/pengine/test10/template-coloc-2.exp ++++ b/pengine/test10/template-coloc-2.exp +@@ -125,7 +125,7 @@ + + + +- ++ + + + +@@ -146,7 +146,7 @@ + + + +- ++ + + + +@@ -173,10 +173,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-coloc-3.exp b/pengine/test10/template-coloc-3.exp +index 37e359b..74e9f90 100644 +--- a/pengine/test10/template-coloc-3.exp ++++ b/pengine/test10/template-coloc-3.exp +@@ -187,7 +187,7 @@ + + + +- ++ + + + +@@ -214,7 +214,7 @@ + + + +- ++ + + + +@@ -247,10 +247,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-order-1.exp b/pengine/test10/template-order-1.exp +index 82c79fa..dc9ecd9 100644 +--- a/pengine/test10/template-order-1.exp ++++ b/pengine/test10/template-order-1.exp +@@ -134,7 +134,7 @@ + + + +- ++ + + + +@@ -155,7 +155,7 @@ + + + +- ++ + + + +@@ -182,10 +182,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-order-2.exp b/pengine/test10/template-order-2.exp +index 261ae44..bbf7d9c 100644 +--- a/pengine/test10/template-order-2.exp ++++ b/pengine/test10/template-order-2.exp +@@ -134,7 +134,7 @@ + + + +- ++ + + + +@@ -155,7 +155,7 @@ + + + +- ++ + + + +@@ -182,10 +182,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-order-3.exp b/pengine/test10/template-order-3.exp +index 7e2e90f..17eab13 100644 +--- a/pengine/test10/template-order-3.exp ++++ b/pengine/test10/template-order-3.exp +@@ -214,7 +214,7 @@ + + + +- ++ + + + +@@ -241,7 +241,7 @@ + + + +- ++ + + + +@@ -274,10 +274,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-rsc-sets-1.exp b/pengine/test10/template-rsc-sets-1.exp +index 0e0f6bd..5166d93 100644 +--- a/pengine/test10/template-rsc-sets-1.exp ++++ b/pengine/test10/template-rsc-sets-1.exp +@@ -168,7 +168,7 @@ + + + +- ++ + + + +@@ -192,7 +192,7 @@ + + + +- ++ + + + +@@ -222,10 +222,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-rsc-sets-2.exp b/pengine/test10/template-rsc-sets-2.exp +index 6e08aad..43c138b 100644 +--- a/pengine/test10/template-rsc-sets-2.exp ++++ b/pengine/test10/template-rsc-sets-2.exp +@@ -174,7 +174,7 @@ + + + +- ++ + + + +@@ -198,7 +198,7 @@ + + + +- ++ + + + +@@ -228,10 +228,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-rsc-sets-3.exp b/pengine/test10/template-rsc-sets-3.exp +index 0e0f6bd..5166d93 100644 +--- a/pengine/test10/template-rsc-sets-3.exp ++++ b/pengine/test10/template-rsc-sets-3.exp +@@ -168,7 +168,7 @@ + + + +- ++ + + + +@@ -192,7 +192,7 @@ + + + +- ++ + + + +@@ -222,10 +222,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-rsc-sets-4.exp b/pengine/test10/template-rsc-sets-4.exp +index f97def2..a00ac64 100644 +--- a/pengine/test10/template-rsc-sets-4.exp ++++ b/pengine/test10/template-rsc-sets-4.exp +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -97,10 +97,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/template-ticket.exp b/pengine/test10/template-ticket.exp +index f97def2..a00ac64 100644 +--- a/pengine/test10/template-ticket.exp ++++ b/pengine/test10/template-ticket.exp +@@ -55,7 +55,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -97,10 +97,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-1.exp b/pengine/test10/ticket-clone-1.exp +index e3a5cb8..e127ef8 100644 +--- a/pengine/test10/ticket-clone-1.exp ++++ b/pengine/test10/ticket-clone-1.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-10.exp b/pengine/test10/ticket-clone-10.exp +index e3a5cb8..e127ef8 100644 +--- a/pengine/test10/ticket-clone-10.exp ++++ b/pengine/test10/ticket-clone-10.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-11.exp b/pengine/test10/ticket-clone-11.exp +index 9fc69df..cc7e89a 100644 +--- a/pengine/test10/ticket-clone-11.exp ++++ b/pengine/test10/ticket-clone-11.exp +@@ -79,7 +79,7 @@ + + + +- ++ + + + +@@ -87,7 +87,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-12.exp b/pengine/test10/ticket-clone-12.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/ticket-clone-12.exp ++++ b/pengine/test10/ticket-clone-12.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-13.exp b/pengine/test10/ticket-clone-13.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-clone-13.exp ++++ b/pengine/test10/ticket-clone-13.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-14.exp b/pengine/test10/ticket-clone-14.exp +index f9dffd7..cd56b44 100644 +--- a/pengine/test10/ticket-clone-14.exp ++++ b/pengine/test10/ticket-clone-14.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-15.exp b/pengine/test10/ticket-clone-15.exp +index f9dffd7..cd56b44 100644 +--- a/pengine/test10/ticket-clone-15.exp ++++ b/pengine/test10/ticket-clone-15.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-16.exp b/pengine/test10/ticket-clone-16.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-clone-16.exp ++++ b/pengine/test10/ticket-clone-16.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-17.exp b/pengine/test10/ticket-clone-17.exp +index f9dffd7..cd56b44 100644 +--- a/pengine/test10/ticket-clone-17.exp ++++ b/pengine/test10/ticket-clone-17.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-18.exp b/pengine/test10/ticket-clone-18.exp +index f9dffd7..cd56b44 100644 +--- a/pengine/test10/ticket-clone-18.exp ++++ b/pengine/test10/ticket-clone-18.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-19.exp b/pengine/test10/ticket-clone-19.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-clone-19.exp ++++ b/pengine/test10/ticket-clone-19.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-2.exp b/pengine/test10/ticket-clone-2.exp +index 9fc69df..cc7e89a 100644 +--- a/pengine/test10/ticket-clone-2.exp ++++ b/pengine/test10/ticket-clone-2.exp +@@ -79,7 +79,7 @@ + + + +- ++ + + + +@@ -87,7 +87,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-20.exp b/pengine/test10/ticket-clone-20.exp +index f9dffd7..cd56b44 100644 +--- a/pengine/test10/ticket-clone-20.exp ++++ b/pengine/test10/ticket-clone-20.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-22.exp b/pengine/test10/ticket-clone-22.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-clone-22.exp ++++ b/pengine/test10/ticket-clone-22.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-23.exp b/pengine/test10/ticket-clone-23.exp +index f9dffd7..cd56b44 100644 +--- a/pengine/test10/ticket-clone-23.exp ++++ b/pengine/test10/ticket-clone-23.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-24.exp b/pengine/test10/ticket-clone-24.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/ticket-clone-24.exp ++++ b/pengine/test10/ticket-clone-24.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-3.exp b/pengine/test10/ticket-clone-3.exp +index f9dffd7..cd56b44 100644 +--- a/pengine/test10/ticket-clone-3.exp ++++ b/pengine/test10/ticket-clone-3.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-4.exp b/pengine/test10/ticket-clone-4.exp +index e3a5cb8..e127ef8 100644 +--- a/pengine/test10/ticket-clone-4.exp ++++ b/pengine/test10/ticket-clone-4.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-5.exp b/pengine/test10/ticket-clone-5.exp +index 9fc69df..cc7e89a 100644 +--- a/pengine/test10/ticket-clone-5.exp ++++ b/pengine/test10/ticket-clone-5.exp +@@ -79,7 +79,7 @@ + + + +- ++ + + + +@@ -87,7 +87,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-6.exp b/pengine/test10/ticket-clone-6.exp +index f9dffd7..cd56b44 100644 +--- a/pengine/test10/ticket-clone-6.exp ++++ b/pengine/test10/ticket-clone-6.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -61,7 +61,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-7.exp b/pengine/test10/ticket-clone-7.exp +index e3a5cb8..e127ef8 100644 +--- a/pengine/test10/ticket-clone-7.exp ++++ b/pengine/test10/ticket-clone-7.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-8.exp b/pengine/test10/ticket-clone-8.exp +index 9fc69df..cc7e89a 100644 +--- a/pengine/test10/ticket-clone-8.exp ++++ b/pengine/test10/ticket-clone-8.exp +@@ -79,7 +79,7 @@ + + + +- ++ + + + +@@ -87,7 +87,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-1.exp b/pengine/test10/ticket-group-1.exp +index 328d7a7..15dd4eb 100644 +--- a/pengine/test10/ticket-group-1.exp ++++ b/pengine/test10/ticket-group-1.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -73,10 +73,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-10.exp b/pengine/test10/ticket-group-10.exp +index 328d7a7..15dd4eb 100644 +--- a/pengine/test10/ticket-group-10.exp ++++ b/pengine/test10/ticket-group-10.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -73,10 +73,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-11.exp b/pengine/test10/ticket-group-11.exp +index e1b6cc1..7048e97 100644 +--- a/pengine/test10/ticket-group-11.exp ++++ b/pengine/test10/ticket-group-11.exp +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -90,7 +90,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-12.exp b/pengine/test10/ticket-group-12.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/ticket-group-12.exp ++++ b/pengine/test10/ticket-group-12.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-13.exp b/pengine/test10/ticket-group-13.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-group-13.exp ++++ b/pengine/test10/ticket-group-13.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-14.exp b/pengine/test10/ticket-group-14.exp +index 250b6ee..b3b1182 100644 +--- a/pengine/test10/ticket-group-14.exp ++++ b/pengine/test10/ticket-group-14.exp +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-15.exp b/pengine/test10/ticket-group-15.exp +index 250b6ee..b3b1182 100644 +--- a/pengine/test10/ticket-group-15.exp ++++ b/pengine/test10/ticket-group-15.exp +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-16.exp b/pengine/test10/ticket-group-16.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-group-16.exp ++++ b/pengine/test10/ticket-group-16.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-17.exp b/pengine/test10/ticket-group-17.exp +index 250b6ee..b3b1182 100644 +--- a/pengine/test10/ticket-group-17.exp ++++ b/pengine/test10/ticket-group-17.exp +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-18.exp b/pengine/test10/ticket-group-18.exp +index 250b6ee..b3b1182 100644 +--- a/pengine/test10/ticket-group-18.exp ++++ b/pengine/test10/ticket-group-18.exp +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-19.exp b/pengine/test10/ticket-group-19.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-group-19.exp ++++ b/pengine/test10/ticket-group-19.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-2.exp b/pengine/test10/ticket-group-2.exp +index e1b6cc1..7048e97 100644 +--- a/pengine/test10/ticket-group-2.exp ++++ b/pengine/test10/ticket-group-2.exp +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -90,7 +90,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-20.exp b/pengine/test10/ticket-group-20.exp +index 250b6ee..b3b1182 100644 +--- a/pengine/test10/ticket-group-20.exp ++++ b/pengine/test10/ticket-group-20.exp +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-21.exp b/pengine/test10/ticket-group-21.exp +index d85b683..c7b5b72 100644 +--- a/pengine/test10/ticket-group-21.exp ++++ b/pengine/test10/ticket-group-21.exp +@@ -84,7 +84,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-22.exp b/pengine/test10/ticket-group-22.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-group-22.exp ++++ b/pengine/test10/ticket-group-22.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-23.exp b/pengine/test10/ticket-group-23.exp +index 250b6ee..b3b1182 100644 +--- a/pengine/test10/ticket-group-23.exp ++++ b/pengine/test10/ticket-group-23.exp +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-24.exp b/pengine/test10/ticket-group-24.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/ticket-group-24.exp ++++ b/pengine/test10/ticket-group-24.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-3.exp b/pengine/test10/ticket-group-3.exp +index 250b6ee..b3b1182 100644 +--- a/pengine/test10/ticket-group-3.exp ++++ b/pengine/test10/ticket-group-3.exp +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-4.exp b/pengine/test10/ticket-group-4.exp +index 328d7a7..15dd4eb 100644 +--- a/pengine/test10/ticket-group-4.exp ++++ b/pengine/test10/ticket-group-4.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -73,10 +73,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-5.exp b/pengine/test10/ticket-group-5.exp +index e1b6cc1..7048e97 100644 +--- a/pengine/test10/ticket-group-5.exp ++++ b/pengine/test10/ticket-group-5.exp +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -90,7 +90,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-6.exp b/pengine/test10/ticket-group-6.exp +index 250b6ee..b3b1182 100644 +--- a/pengine/test10/ticket-group-6.exp ++++ b/pengine/test10/ticket-group-6.exp +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -64,7 +64,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-7.exp b/pengine/test10/ticket-group-7.exp +index 328d7a7..15dd4eb 100644 +--- a/pengine/test10/ticket-group-7.exp ++++ b/pengine/test10/ticket-group-7.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -73,10 +73,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-8.exp b/pengine/test10/ticket-group-8.exp +index e1b6cc1..7048e97 100644 +--- a/pengine/test10/ticket-group-8.exp ++++ b/pengine/test10/ticket-group-8.exp +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -90,7 +90,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-9.exp b/pengine/test10/ticket-group-9.exp +index d85b683..c7b5b72 100644 +--- a/pengine/test10/ticket-group-9.exp ++++ b/pengine/test10/ticket-group-9.exp +@@ -84,7 +84,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-1.exp b/pengine/test10/ticket-master-1.exp +index 3a93a0e..55abf78 100644 +--- a/pengine/test10/ticket-master-1.exp ++++ b/pengine/test10/ticket-master-1.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-10.exp b/pengine/test10/ticket-master-10.exp +index 8f76f85..44cb6a0 100644 +--- a/pengine/test10/ticket-master-10.exp ++++ b/pengine/test10/ticket-master-10.exp +@@ -77,7 +77,7 @@ + + + +- ++ + + + +@@ -89,7 +89,7 @@ + + + +- ++ + + + +@@ -107,10 +107,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-11.exp b/pengine/test10/ticket-master-11.exp +index d160526..c0299ae 100644 +--- a/pengine/test10/ticket-master-11.exp ++++ b/pengine/test10/ticket-master-11.exp +@@ -34,7 +34,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-12.exp b/pengine/test10/ticket-master-12.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/ticket-master-12.exp ++++ b/pengine/test10/ticket-master-12.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-13.exp b/pengine/test10/ticket-master-13.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-master-13.exp ++++ b/pengine/test10/ticket-master-13.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-14.exp b/pengine/test10/ticket-master-14.exp +index 256060e..2e1faee 100644 +--- a/pengine/test10/ticket-master-14.exp ++++ b/pengine/test10/ticket-master-14.exp +@@ -96,7 +96,7 @@ + + + +- ++ + + + +@@ -104,7 +104,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-15.exp b/pengine/test10/ticket-master-15.exp +index 256060e..2e1faee 100644 +--- a/pengine/test10/ticket-master-15.exp ++++ b/pengine/test10/ticket-master-15.exp +@@ -96,7 +96,7 @@ + + + +- ++ + + + +@@ -104,7 +104,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-16.exp b/pengine/test10/ticket-master-16.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/ticket-master-16.exp ++++ b/pengine/test10/ticket-master-16.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-17.exp b/pengine/test10/ticket-master-17.exp +index b1b51b2..47b3cd7 100644 +--- a/pengine/test10/ticket-master-17.exp ++++ b/pengine/test10/ticket-master-17.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-18.exp b/pengine/test10/ticket-master-18.exp +index b1b51b2..47b3cd7 100644 +--- a/pengine/test10/ticket-master-18.exp ++++ b/pengine/test10/ticket-master-18.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-19.exp b/pengine/test10/ticket-master-19.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/ticket-master-19.exp ++++ b/pengine/test10/ticket-master-19.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-2.exp b/pengine/test10/ticket-master-2.exp +index 967072c..c995d00 100644 +--- a/pengine/test10/ticket-master-2.exp ++++ b/pengine/test10/ticket-master-2.exp +@@ -93,7 +93,7 @@ + + + +- ++ + + + +@@ -101,7 +101,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-20.exp b/pengine/test10/ticket-master-20.exp +index b1b51b2..47b3cd7 100644 +--- a/pengine/test10/ticket-master-20.exp ++++ b/pengine/test10/ticket-master-20.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-21.exp b/pengine/test10/ticket-master-21.exp +index 1ed731c..5b21e47 100644 +--- a/pengine/test10/ticket-master-21.exp ++++ b/pengine/test10/ticket-master-21.exp +@@ -135,7 +135,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-22.exp b/pengine/test10/ticket-master-22.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/ticket-master-22.exp ++++ b/pengine/test10/ticket-master-22.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-23.exp b/pengine/test10/ticket-master-23.exp +index b1b51b2..47b3cd7 100644 +--- a/pengine/test10/ticket-master-23.exp ++++ b/pengine/test10/ticket-master-23.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-24.exp b/pengine/test10/ticket-master-24.exp +index 6d57001..641b47d 100644 +--- a/pengine/test10/ticket-master-24.exp ++++ b/pengine/test10/ticket-master-24.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-3.exp b/pengine/test10/ticket-master-3.exp +index 256060e..2e1faee 100644 +--- a/pengine/test10/ticket-master-3.exp ++++ b/pengine/test10/ticket-master-3.exp +@@ -96,7 +96,7 @@ + + + +- ++ + + + +@@ -104,7 +104,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-4.exp b/pengine/test10/ticket-master-4.exp +index 8f76f85..44cb6a0 100644 +--- a/pengine/test10/ticket-master-4.exp ++++ b/pengine/test10/ticket-master-4.exp +@@ -77,7 +77,7 @@ + + + +- ++ + + + +@@ -89,7 +89,7 @@ + + + +- ++ + + + +@@ -107,10 +107,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-5.exp b/pengine/test10/ticket-master-5.exp +index d160526..c0299ae 100644 +--- a/pengine/test10/ticket-master-5.exp ++++ b/pengine/test10/ticket-master-5.exp +@@ -34,7 +34,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-6.exp b/pengine/test10/ticket-master-6.exp +index b1b51b2..47b3cd7 100644 +--- a/pengine/test10/ticket-master-6.exp ++++ b/pengine/test10/ticket-master-6.exp +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -45,7 +45,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-7.exp b/pengine/test10/ticket-master-7.exp +index 8f76f85..44cb6a0 100644 +--- a/pengine/test10/ticket-master-7.exp ++++ b/pengine/test10/ticket-master-7.exp +@@ -77,7 +77,7 @@ + + + +- ++ + + + +@@ -89,7 +89,7 @@ + + + +- ++ + + + +@@ -107,10 +107,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-8.exp b/pengine/test10/ticket-master-8.exp +index d160526..c0299ae 100644 +--- a/pengine/test10/ticket-master-8.exp ++++ b/pengine/test10/ticket-master-8.exp +@@ -34,7 +34,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-9.exp b/pengine/test10/ticket-master-9.exp +index 1ed731c..5b21e47 100644 +--- a/pengine/test10/ticket-master-9.exp ++++ b/pengine/test10/ticket-master-9.exp +@@ -135,7 +135,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-1.exp b/pengine/test10/ticket-primitive-1.exp +index b532698..88a1075 100644 +--- a/pengine/test10/ticket-primitive-1.exp ++++ b/pengine/test10/ticket-primitive-1.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-10.exp b/pengine/test10/ticket-primitive-10.exp +index b532698..88a1075 100644 +--- a/pengine/test10/ticket-primitive-10.exp ++++ b/pengine/test10/ticket-primitive-10.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-11.exp b/pengine/test10/ticket-primitive-11.exp +index 2d13079..ed47627 100644 +--- a/pengine/test10/ticket-primitive-11.exp ++++ b/pengine/test10/ticket-primitive-11.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-12.exp b/pengine/test10/ticket-primitive-12.exp +index ba6632b..0f663b1 100644 +--- a/pengine/test10/ticket-primitive-12.exp ++++ b/pengine/test10/ticket-primitive-12.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-13.exp b/pengine/test10/ticket-primitive-13.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-primitive-13.exp ++++ b/pengine/test10/ticket-primitive-13.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-14.exp b/pengine/test10/ticket-primitive-14.exp +index 3cb5f90..12753e9 100644 +--- a/pengine/test10/ticket-primitive-14.exp ++++ b/pengine/test10/ticket-primitive-14.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-15.exp b/pengine/test10/ticket-primitive-15.exp +index 3cb5f90..12753e9 100644 +--- a/pengine/test10/ticket-primitive-15.exp ++++ b/pengine/test10/ticket-primitive-15.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-16.exp b/pengine/test10/ticket-primitive-16.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-primitive-16.exp ++++ b/pengine/test10/ticket-primitive-16.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-17.exp b/pengine/test10/ticket-primitive-17.exp +index 3cb5f90..12753e9 100644 +--- a/pengine/test10/ticket-primitive-17.exp ++++ b/pengine/test10/ticket-primitive-17.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-18.exp b/pengine/test10/ticket-primitive-18.exp +index 3cb5f90..12753e9 100644 +--- a/pengine/test10/ticket-primitive-18.exp ++++ b/pengine/test10/ticket-primitive-18.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-19.exp b/pengine/test10/ticket-primitive-19.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-primitive-19.exp ++++ b/pengine/test10/ticket-primitive-19.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-2.exp b/pengine/test10/ticket-primitive-2.exp +index 2d13079..ed47627 100644 +--- a/pengine/test10/ticket-primitive-2.exp ++++ b/pengine/test10/ticket-primitive-2.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-20.exp b/pengine/test10/ticket-primitive-20.exp +index 3cb5f90..12753e9 100644 +--- a/pengine/test10/ticket-primitive-20.exp ++++ b/pengine/test10/ticket-primitive-20.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-21.exp b/pengine/test10/ticket-primitive-21.exp +index c88cf38..72fcacd 100644 +--- a/pengine/test10/ticket-primitive-21.exp ++++ b/pengine/test10/ticket-primitive-21.exp +@@ -33,7 +33,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-22.exp b/pengine/test10/ticket-primitive-22.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-primitive-22.exp ++++ b/pengine/test10/ticket-primitive-22.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-23.exp b/pengine/test10/ticket-primitive-23.exp +index 3cb5f90..12753e9 100644 +--- a/pengine/test10/ticket-primitive-23.exp ++++ b/pengine/test10/ticket-primitive-23.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-24.exp b/pengine/test10/ticket-primitive-24.exp +index ba6632b..0f663b1 100644 +--- a/pengine/test10/ticket-primitive-24.exp ++++ b/pengine/test10/ticket-primitive-24.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-3.exp b/pengine/test10/ticket-primitive-3.exp +index 3cb5f90..12753e9 100644 +--- a/pengine/test10/ticket-primitive-3.exp ++++ b/pengine/test10/ticket-primitive-3.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-4.exp b/pengine/test10/ticket-primitive-4.exp +index b532698..88a1075 100644 +--- a/pengine/test10/ticket-primitive-4.exp ++++ b/pengine/test10/ticket-primitive-4.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-5.exp b/pengine/test10/ticket-primitive-5.exp +index 2d13079..ed47627 100644 +--- a/pengine/test10/ticket-primitive-5.exp ++++ b/pengine/test10/ticket-primitive-5.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-6.exp b/pengine/test10/ticket-primitive-6.exp +index 3cb5f90..12753e9 100644 +--- a/pengine/test10/ticket-primitive-6.exp ++++ b/pengine/test10/ticket-primitive-6.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -18,7 +18,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-7.exp b/pengine/test10/ticket-primitive-7.exp +index b532698..88a1075 100644 +--- a/pengine/test10/ticket-primitive-7.exp ++++ b/pengine/test10/ticket-primitive-7.exp +@@ -19,7 +19,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +@@ -49,10 +49,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-8.exp b/pengine/test10/ticket-primitive-8.exp +index 2d13079..ed47627 100644 +--- a/pengine/test10/ticket-primitive-8.exp ++++ b/pengine/test10/ticket-primitive-8.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-9.exp b/pengine/test10/ticket-primitive-9.exp +index c88cf38..72fcacd 100644 +--- a/pengine/test10/ticket-primitive-9.exp ++++ b/pengine/test10/ticket-primitive-9.exp +@@ -33,7 +33,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-1.exp b/pengine/test10/ticket-rsc-sets-1.exp +index 20800e2..869b1e3 100644 +--- a/pengine/test10/ticket-rsc-sets-1.exp ++++ b/pengine/test10/ticket-rsc-sets-1.exp +@@ -149,7 +149,7 @@ + + + +- ++ + + + +@@ -173,7 +173,7 @@ + + + +- ++ + + + +@@ -203,10 +203,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-10.exp b/pengine/test10/ticket-rsc-sets-10.exp +index 35e5a99..36b6cc9 100644 +--- a/pengine/test10/ticket-rsc-sets-10.exp ++++ b/pengine/test10/ticket-rsc-sets-10.exp +@@ -153,7 +153,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-11.exp b/pengine/test10/ticket-rsc-sets-11.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-rsc-sets-11.exp ++++ b/pengine/test10/ticket-rsc-sets-11.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-12.exp b/pengine/test10/ticket-rsc-sets-12.exp +index 2712003..bc5c4d4 100644 +--- a/pengine/test10/ticket-rsc-sets-12.exp ++++ b/pengine/test10/ticket-rsc-sets-12.exp +@@ -65,7 +65,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-13.exp b/pengine/test10/ticket-rsc-sets-13.exp +index 35e5a99..36b6cc9 100644 +--- a/pengine/test10/ticket-rsc-sets-13.exp ++++ b/pengine/test10/ticket-rsc-sets-13.exp +@@ -153,7 +153,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-14.exp b/pengine/test10/ticket-rsc-sets-14.exp +index 35e5a99..36b6cc9 100644 +--- a/pengine/test10/ticket-rsc-sets-14.exp ++++ b/pengine/test10/ticket-rsc-sets-14.exp +@@ -153,7 +153,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-2.exp b/pengine/test10/ticket-rsc-sets-2.exp +index 5618565..0594676 100644 +--- a/pengine/test10/ticket-rsc-sets-2.exp ++++ b/pengine/test10/ticket-rsc-sets-2.exp +@@ -215,7 +215,7 @@ + + + +- ++ + + + +@@ -223,7 +223,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-3.exp b/pengine/test10/ticket-rsc-sets-3.exp +index 35e5a99..36b6cc9 100644 +--- a/pengine/test10/ticket-rsc-sets-3.exp ++++ b/pengine/test10/ticket-rsc-sets-3.exp +@@ -153,7 +153,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-4.exp b/pengine/test10/ticket-rsc-sets-4.exp +index 20800e2..869b1e3 100644 +--- a/pengine/test10/ticket-rsc-sets-4.exp ++++ b/pengine/test10/ticket-rsc-sets-4.exp +@@ -149,7 +149,7 @@ + + + +- ++ + + + +@@ -173,7 +173,7 @@ + + + +- ++ + + + +@@ -203,10 +203,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-5.exp b/pengine/test10/ticket-rsc-sets-5.exp +index 9e20a6b..206959f 100644 +--- a/pengine/test10/ticket-rsc-sets-5.exp ++++ b/pengine/test10/ticket-rsc-sets-5.exp +@@ -104,7 +104,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-6.exp b/pengine/test10/ticket-rsc-sets-6.exp +index 9bd2d3a..91ce17e 100644 +--- a/pengine/test10/ticket-rsc-sets-6.exp ++++ b/pengine/test10/ticket-rsc-sets-6.exp +@@ -112,7 +112,7 @@ + + + +- ++ + + + +@@ -120,7 +120,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-7.exp b/pengine/test10/ticket-rsc-sets-7.exp +index 35e5a99..36b6cc9 100644 +--- a/pengine/test10/ticket-rsc-sets-7.exp ++++ b/pengine/test10/ticket-rsc-sets-7.exp +@@ -153,7 +153,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-8.exp b/pengine/test10/ticket-rsc-sets-8.exp +index ccd7539..cf75a01 100644 +--- a/pengine/test10/ticket-rsc-sets-8.exp ++++ b/pengine/test10/ticket-rsc-sets-8.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-rsc-sets-9.exp b/pengine/test10/ticket-rsc-sets-9.exp +index 35e5a99..36b6cc9 100644 +--- a/pengine/test10/ticket-rsc-sets-9.exp ++++ b/pengine/test10/ticket-rsc-sets-9.exp +@@ -153,7 +153,7 @@ + + + +- ++ + + + +@@ -161,7 +161,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unmanaged-master.exp b/pengine/test10/unmanaged-master.exp +index 9e0f188..58311fd 100644 +--- a/pengine/test10/unmanaged-master.exp ++++ b/pengine/test10/unmanaged-master.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,13 +9,13 @@ + + + +- ++ + + + + + +- ++ + + + +diff --git a/pengine/test10/unmanaged-stop-1.exp b/pengine/test10/unmanaged-stop-1.exp +index 7845919..ed7dcfb 100644 +--- a/pengine/test10/unmanaged-stop-1.exp ++++ b/pengine/test10/unmanaged-stop-1.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unmanaged-stop-2.exp b/pengine/test10/unmanaged-stop-2.exp +index 7845919..ed7dcfb 100644 +--- a/pengine/test10/unmanaged-stop-2.exp ++++ b/pengine/test10/unmanaged-stop-2.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unmanaged-stop-3.exp b/pengine/test10/unmanaged-stop-3.exp +index 2cb2435..a1db8b0 100644 +--- a/pengine/test10/unmanaged-stop-3.exp ++++ b/pengine/test10/unmanaged-stop-3.exp +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unmanaged-stop-4.exp b/pengine/test10/unmanaged-stop-4.exp +index 2cb2435..a1db8b0 100644 +--- a/pengine/test10/unmanaged-stop-4.exp ++++ b/pengine/test10/unmanaged-stop-4.exp +@@ -9,7 +9,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unrunnable-1.exp b/pengine/test10/unrunnable-1.exp +index 62a6e29..35b17ee 100644 +--- a/pengine/test10/unrunnable-1.exp ++++ b/pengine/test10/unrunnable-1.exp +@@ -120,7 +120,7 @@ + + + +- ++ + + + +@@ -168,7 +168,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/use-after-free-merge.exp b/pengine/test10/use-after-free-merge.exp +index e7052b1..178b84e 100644 +--- a/pengine/test10/use-after-free-merge.exp ++++ b/pengine/test10/use-after-free-merge.exp +@@ -144,7 +144,7 @@ + + + +- ++ + + + +@@ -165,7 +165,7 @@ + + + +- ++ + + + +@@ -192,10 +192,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/utilization-order1.exp b/pengine/test10/utilization-order1.exp +index c5e551d..40ba326 100644 +--- a/pengine/test10/utilization-order1.exp ++++ b/pengine/test10/utilization-order1.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -31,7 +31,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/utilization-order2.exp b/pengine/test10/utilization-order2.exp +index f62a8ef..dfa3f60 100644 +--- a/pengine/test10/utilization-order2.exp ++++ b/pengine/test10/utilization-order2.exp +@@ -84,7 +84,7 @@ + + + +- ++ + + + +@@ -92,7 +92,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/utilization-order3.exp b/pengine/test10/utilization-order3.exp +index 570a39e..9075052 100644 +--- a/pengine/test10/utilization-order3.exp ++++ b/pengine/test10/utilization-order3.exp +@@ -64,7 +64,7 @@ + + + +- ++ + + + +@@ -72,7 +72,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/utilization.exp b/pengine/test10/utilization.exp +index ed08a2b..f6c1b69 100644 +--- a/pengine/test10/utilization.exp ++++ b/pengine/test10/utilization.exp +@@ -53,7 +53,7 @@ + + + +- ++ + + + +@@ -68,7 +68,7 @@ + + + +- ++ + + + +@@ -89,10 +89,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/whitebox-fail3.exp b/pengine/test10/whitebox-fail3.exp +index c5a6474..739d71b 100644 +--- a/pengine/test10/whitebox-fail3.exp ++++ b/pengine/test10/whitebox-fail3.exp +@@ -187,7 +187,7 @@ + + + +- ++ + + + +@@ -205,7 +205,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/whitebox-ms-ordering.exp b/pengine/test10/whitebox-ms-ordering.exp +new file mode 100644 +index 0000000..1ff4595 +--- /dev/null ++++ b/pengine/test10/whitebox-ms-ordering.exp +@@ -0,0 +1,503 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c +index 78cab50..2eb68f1 100644 +--- a/tools/crm_simulate.c ++++ b/tools/crm_simulate.c +@@ -760,9 +760,12 @@ create_action_name(action_t * action) + } else if (safe_str_eq(action->task, CRM_OP_FENCE)) { + action_name = g_strdup_printf("%s%s %s", prefix ? prefix : "", action->task, action_host); + +- } else if (action_host) { ++ } else if (action->rsc && action_host) { + action_name = g_strdup_printf("%s%s %s", prefix ? prefix : "", action->uuid, action_host); + ++ } else if (action_host) { ++ action_name = g_strdup_printf("%s%s %s", prefix ? prefix : "", action->task, action_host); ++ + } else { + action_name = g_strdup_printf("%s", action->uuid); + } +@@ -789,7 +792,7 @@ create_dotfile(pe_working_set_t * data_set, const char *dot_file, gboolean all_a + const char *color = "black"; + char *action_name = create_action_name(action); + +- crm_trace("Action %d: %p", action->id, action); ++ crm_trace("Action %d: %s %s %p", action->id, action_name, action->uuid, action); + + if (is_set(action->flags, pe_action_pseudo)) { + font = "orange"; +@@ -819,6 +822,8 @@ create_dotfile(pe_working_set_t * data_set, const char *dot_file, gboolean all_a + } + + set_bit(action->flags, pe_action_dumped); ++ crm_trace("\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]", ++ action_name, style, color, font); + fprintf(dot_strm, "\"%s\" [ style=%s color=\"%s\" fontcolor=\"%s\"]\n", + action_name, style, color, font); + dont_write: +@@ -857,6 +862,8 @@ create_dotfile(pe_working_set_t * data_set, const char *dot_file, gboolean all_a + if (all_actions || optional == FALSE) { + before_name = create_action_name(before->action); + after_name = create_action_name(action); ++ crm_trace("\"%s\" -> \"%s\" [ style = %s]", ++ before_name, after_name, style); + fprintf(dot_strm, "\"%s\" -> \"%s\" [ style = %s]\n", + before_name, after_name, style); + free(before_name); diff --git a/SOURCES/bz1078078-pcmk-pe_ensure_unfencing_occurs_before_fencing_devices_are_re_probed.patch b/SOURCES/bz1078078-pcmk-pe_ensure_unfencing_occurs_before_fencing_devices_are_re_probed.patch new file mode 100644 index 0000000..b7d7017 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_ensure_unfencing_occurs_before_fencing_devices_are_re_probed.patch @@ -0,0 +1,311 @@ +commit 5566c3e6b194cbe682408ef9c88ad3b6a9755f7d +Author: Andrew Beekhof +Date: Wed Apr 9 12:07:46 2014 +1000 + + Fix: PE: Ensure unfencing occurs before fencing devices are (re-)probed + + (cherry picked from commit 6be22945726c204b9c73da843bb04981e46126be) + + Conflicts: + pengine/test10/unfence-startup.summary + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index e84d341..9cb744a 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -1881,7 +1881,7 @@ trigger_unfencing( + action_t *unfence = pe_fence_op(node, "on", FALSE, data_set); + + crm_notice("Unfencing %s: %s", node->details->uname, reason); +- if(FALSE && dependancy) { ++ if(dependancy) { + order_actions(unfence, dependancy, pe_order_optional); + } + +diff --git a/pengine/native.c b/pengine/native.c +index 3d75dc9..5c723c2 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -2518,8 +2518,21 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + } + + pe_rsc_debug(rsc, "Probing %s on %s (%s)", rsc->id, node->details->uname, role2text(rsc->role)); +- order_actions(probe, complete, pe_order_implies_then); + ++ if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) { ++ /* Normally rsc.start depends on probe complete which depends ++ * on rsc.probe. But this can't be the case in this scenario as ++ * it would create graph loops. ++ * ++ * So instead we explicitly order 'rsc.probe then rsc.start' ++ */ ++ custom_action_order(rsc, NULL, probe, ++ rsc, generate_op_key(rsc->id, RSC_START, 0), NULL, ++ pe_order_optional, data_set); ++ ++ } else { ++ order_actions(probe, complete, pe_order_implies_then); ++ } + return TRUE; + } + +diff --git a/pengine/test10/unfence-definition.dot b/pengine/test10/unfence-definition.dot +index e737687..6ab0e28 100644 +--- a/pengine/test10/unfence-definition.dot ++++ b/pengine/test10/unfence-definition.dot +@@ -61,7 +61,7 @@ digraph "g" { + "dlm_stop_0 virt-1" [ style=bold color="green" fontcolor="black"] + "fencing_delete_0 virt-1" -> "fencing_start_0 virt-1" [ style = bold] + "fencing_delete_0 virt-1" [ style=bold color="green" fontcolor="black"] +-"fencing_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"fencing_monitor_0 virt-3" -> "fencing_start_0 virt-1" [ style = bold] + "fencing_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "fencing_start_0 virt-1" [ style=bold color="green" fontcolor="black"] + "fencing_stop_0 virt-1" -> "all_stopped" [ style = bold] +@@ -89,6 +89,7 @@ digraph "g" { + "stonith 'on' virt-1" [ style=bold color="green" fontcolor="black"] + "stonith 'on' virt-3" -> "clvmd-clone_start_0" [ style = bold] + "stonith 'on' virt-3" -> "dlm-clone_start_0" [ style = bold] ++"stonith 'on' virt-3" -> "fencing_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold] + "stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/unfence-definition.exp b/pengine/test10/unfence-definition.exp +index 9075347..0f779da 100644 +--- a/pengine/test10/unfence-definition.exp ++++ b/pengine/test10/unfence-definition.exp +@@ -19,6 +19,9 @@ + + + ++ ++ ++ + + + +@@ -28,7 +31,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -387,9 +394,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary +index c954c4b..3e4b66b 100644 +--- a/pengine/test10/unfence-definition.summary ++++ b/pengine/test10/unfence-definition.summary +@@ -20,7 +20,6 @@ Transition Summary: + * Start clvmd:2 (virt-3) + + Executing cluster transition: +- * Resource action: fencing monitor on virt-3 + * Resource action: dlm monitor on virt-3 + * Resource action: clvmd monitor on virt-2 + * Resource action: clvmd monitor on virt-3 +@@ -38,8 +37,9 @@ Executing cluster transition: + * Fencing virt-3 (on) + * Pseudo action: all_stopped + * Fencing virt-1 (on) +- * Resource action: fencing start on virt-1 ++ * Resource action: fencing monitor on virt-3 + * Pseudo action: dlm-clone_start_0 ++ * Resource action: fencing start on virt-1 + * Resource action: dlm start on virt-1 + * Resource action: dlm start on virt-3 + * Pseudo action: dlm-clone_running_0 +diff --git a/pengine/test10/unfence-parameters.dot b/pengine/test10/unfence-parameters.dot +index e67eb5a..082bbe1 100644 +--- a/pengine/test10/unfence-parameters.dot ++++ b/pengine/test10/unfence-parameters.dot +@@ -78,7 +78,7 @@ digraph "g" { + "dlm_stop_0 virt-2" -> "dlm_start_0 virt-2" [ style = bold] + "dlm_stop_0 virt-2" -> "dlm_stop_0 virt-1" [ style = bold] + "dlm_stop_0 virt-2" [ style=bold color="green" fontcolor="black"] +-"fencing_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] ++"fencing_monitor_0 virt-3" -> "fencing_start_0 virt-1" [ style = bold] + "fencing_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "fencing_start_0 virt-1" [ style=bold color="green" fontcolor="black"] + "fencing_stop_0 virt-1" -> "all_stopped" [ style = bold] +@@ -111,6 +111,7 @@ digraph "g" { + "stonith 'on' virt-2" [ style=bold color="green" fontcolor="black"] + "stonith 'on' virt-3" -> "clvmd-clone_start_0" [ style = bold] + "stonith 'on' virt-3" -> "dlm-clone_start_0" [ style = bold] ++"stonith 'on' virt-3" -> "fencing_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold] + "stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/unfence-parameters.exp b/pengine/test10/unfence-parameters.exp +index 491d78e..d3480f3 100644 +--- a/pengine/test10/unfence-parameters.exp ++++ b/pengine/test10/unfence-parameters.exp +@@ -19,7 +19,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -36,6 +40,9 @@ + + + ++ ++ ++ + + + +@@ -433,9 +440,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +index 17bbc62..5276cba 100644 +--- a/pengine/test10/unfence-parameters.summary ++++ b/pengine/test10/unfence-parameters.summary +@@ -21,7 +21,6 @@ Transition Summary: + * Start clvmd:2 (virt-3) + + Executing cluster transition: +- * Resource action: fencing monitor on virt-3 + * Resource action: dlm monitor on virt-3 + * Resource action: clvmd monitor on virt-2 + * Resource action: clvmd monitor on virt-3 +@@ -40,6 +39,7 @@ Executing cluster transition: + * Fencing virt-2 (on) + * Pseudo action: all_stopped + * Fencing virt-1 (on) ++ * Resource action: fencing monitor on virt-3 + * Resource action: fencing start on virt-1 + * Pseudo action: dlm-clone_start_0 + * Resource action: dlm start on virt-1 +diff --git a/pengine/test10/unfence-startup.dot b/pengine/test10/unfence-startup.dot +index a5cc606..4dbd273 100644 +--- a/pengine/test10/unfence-startup.dot ++++ b/pengine/test10/unfence-startup.dot +@@ -26,7 +26,6 @@ digraph "g" { + "dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold] + "dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold] + "dlm:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"] +-"fencing_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold] + "fencing_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"] + "probe_complete virt-1" -> "probe_complete" [ style = bold] + "probe_complete virt-1" [ style=bold color="green" fontcolor="black"] +@@ -40,6 +39,7 @@ digraph "g" { + "probe_complete" [ style=bold color="green" fontcolor="orange"] + "stonith 'on' virt-3" -> "clvmd-clone_start_0" [ style = bold] + "stonith 'on' virt-3" -> "dlm-clone_start_0" [ style = bold] ++"stonith 'on' virt-3" -> "fencing_monitor_0 virt-3" [ style = bold] + "stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold] + "stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"] +diff --git a/pengine/test10/unfence-startup.exp b/pengine/test10/unfence-startup.exp +index 8a04d54..7c55d2c 100644 +--- a/pengine/test10/unfence-startup.exp ++++ b/pengine/test10/unfence-startup.exp +@@ -6,7 +6,11 @@ + + + +- ++ ++ ++ ++ ++ + + + +@@ -186,9 +190,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/unfence-startup.summary b/pengine/test10/unfence-startup.summary +new file mode 100644 +index 0000000..a03fa15 +--- /dev/null ++++ b/pengine/test10/unfence-startup.summary +@@ -0,0 +1,48 @@ ++ ++Current cluster status: ++Node virt-4 (4): UNCLEAN (offline) ++Online: [ virt-1 virt-2 virt-3 ] ++ ++ fencing (stonith:fence_scsi): Started virt-1 ++ Clone Set: dlm-clone [dlm] ++ Started: [ virt-1 virt-2 ] ++ Stopped: [ virt-3 virt-4 ] ++ Clone Set: clvmd-clone [clvmd] ++ Started: [ virt-1 ] ++ Stopped: [ virt-2 virt-3 virt-4 ] ++ ++Transition Summary: ++ * Start dlm:2 (virt-3) ++ * Start clvmd:1 (virt-2) ++ * Start clvmd:2 (virt-3) ++ ++Executing cluster transition: ++ * Resource action: dlm monitor on virt-3 ++ * Resource action: clvmd monitor on virt-2 ++ * Resource action: clvmd monitor on virt-3 ++ * Fencing virt-4 (reboot) ++ * Pseudo action: stonith_complete ++ * Pseudo action: probe_complete ++ * Fencing virt-3 (on) ++ * Pseudo action: all_stopped ++ * Resource action: fencing monitor on virt-3 ++ * Pseudo action: dlm-clone_start_0 ++ * Resource action: dlm start on virt-3 ++ * Pseudo action: dlm-clone_running_0 ++ * Pseudo action: clvmd-clone_start_0 ++ * Resource action: clvmd start on virt-2 ++ * Resource action: clvmd start on virt-3 ++ * Pseudo action: clvmd-clone_running_0 ++ ++Revised cluster status: ++Online: [ virt-1 virt-2 virt-3 ] ++OFFLINE: [ virt-4 ] ++ ++ fencing (stonith:fence_scsi): Started virt-1 ++ Clone Set: dlm-clone [dlm] ++ Started: [ virt-1 virt-2 virt-3 ] ++ Stopped: [ virt-4 ] ++ Clone Set: clvmd-clone [clvmd] ++ Started: [ virt-1 virt-2 virt-3 ] ++ Stopped: [ virt-4 ] ++ diff --git a/SOURCES/bz1078078-pcmk-pe_ensure_unfencing_only_happens_once_even_if_the_transition_is_interrupted.patch b/SOURCES/bz1078078-pcmk-pe_ensure_unfencing_only_happens_once_even_if_the_transition_is_interrupted.patch new file mode 100644 index 0000000..0cbf2b6 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_ensure_unfencing_only_happens_once_even_if_the_transition_is_interrupted.patch @@ -0,0 +1,57 @@ +commit 6f2e3fc12f64f317280f9cb3d65cc08489516e39 +Author: Andrew Beekhof +Date: Thu Apr 10 14:25:32 2014 +1000 + + Fix: PE: Ensure unfencing only happens once, even if the transition is interrupted + + (cherry picked from commit 76a3fe06ae8fdfa579b2ae6ba730945ef79b0ebc) + +diff --git a/pengine/native.c b/pengine/native.c +index 5c723c2..6294092 100644 +--- a/pengine/native.c ++++ b/pengine/native.c +@@ -2498,6 +2498,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete, + */ + if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) { + trigger_unfencing(NULL, node, "node discovery", probe, data_set); ++ probe->priority = INFINITY; /* Ensure this runs if unfencing succeeds */ + + } else if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { + action_t *unfence = pe_fence_op(node, "on", TRUE, data_set); +diff --git a/pengine/test10/unfence-definition.exp b/pengine/test10/unfence-definition.exp +index 0f779da..f023a0b 100644 +--- a/pengine/test10/unfence-definition.exp ++++ b/pengine/test10/unfence-definition.exp +@@ -24,7 +24,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unfence-parameters.exp b/pengine/test10/unfence-parameters.exp +index d3480f3..8ff3016 100644 +--- a/pengine/test10/unfence-parameters.exp ++++ b/pengine/test10/unfence-parameters.exp +@@ -12,7 +12,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unfence-startup.exp b/pengine/test10/unfence-startup.exp +index 7c55d2c..a0661e7 100644 +--- a/pengine/test10/unfence-startup.exp ++++ b/pengine/test10/unfence-startup.exp +@@ -1,5 +1,5 @@ + +- ++ + + + diff --git a/SOURCES/bz1078078-pcmk-pe_fencing_devices_default_to_only_requiring_quorum_in_order_to_start.patch b/SOURCES/bz1078078-pcmk-pe_fencing_devices_default_to_only_requiring_quorum_in_order_to_start.patch new file mode 100644 index 0000000..d4831fd --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_fencing_devices_default_to_only_requiring_quorum_in_order_to_start.patch @@ -0,0 +1,753 @@ +commit fabf4840a335d199ec921b37692af02344e87565 +Author: Andrew Beekhof +Date: Fri Apr 4 10:14:05 2014 +1100 + + Fix: PE: Fencing devices default to only requiring quorum in order to start + + (cherry picked from commit c19b872efb9ad1007f5bf574abf1928689af9dcc) + + Conflicts: + pengine/test10/bug-5186-partial-migrate.dot + +diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c +index 8b774c0..0f20481 100644 +--- a/lib/pengine/complex.c ++++ b/lib/pengine/complex.c +@@ -586,7 +586,10 @@ common_unpack(xmlNode * xml_obj, resource_t ** rsc, + } + + isdefault = TRUE; +- if (is_set(data_set->flags, pe_flag_enable_unfencing)) { ++ if(is_set((*rsc)->flags, pe_rsc_fence_device)) { ++ value = "quorum"; ++ ++ } else if (is_set(data_set->flags, pe_flag_enable_unfencing)) { + value = "unfencing"; + + } else if (is_set(data_set->flags, pe_flag_stonith_enabled)) { +diff --git a/pengine/test10/bug-5186-partial-migrate.exp b/pengine/test10/bug-5186-partial-migrate.exp +index 7e10e3a..216d962 100644 +--- a/pengine/test10/bug-5186-partial-migrate.exp ++++ b/pengine/test10/bug-5186-partial-migrate.exp +@@ -167,9 +167,6 @@ + + + +- +- +- + + + +@@ -220,9 +217,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/bug-lf-2508.dot b/pengine/test10/bug-lf-2508.dot +index 6bef312..6e88af5 100644 +--- a/pengine/test10/bug-lf-2508.dot ++++ b/pengine/test10/bug-lf-2508.dot +@@ -24,6 +24,7 @@ digraph "g" { + "Group02_stop_0" [ style=bold color="green" fontcolor="orange" ] + "Group02_stopped_0" -> "Group02_start_0" [ style = bold] + "Group02_stopped_0" [ style=bold color="green" fontcolor="orange" ] ++"all_stopped" -> "prmStonith3-3:1_start_0 srv01" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "clnStonith1_stop_0" -> "clnStonith1_stopped_0" [ style = bold] + "clnStonith1_stop_0" -> "grpStonith1:1_stop_0" [ style = bold] +@@ -107,6 +108,5 @@ digraph "g" { + "stonith_complete" -> "Dummy01_start_0 srv01" [ style = bold] + "stonith_complete" -> "Dummy02_start_0 srv04" [ style = bold] + "stonith_complete" -> "all_stopped" [ style = bold] +-"stonith_complete" -> "prmStonith3-3:1_start_0 srv01" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/bug-lf-2508.exp b/pengine/test10/bug-lf-2508.exp +index d56002e..0d84e9e 100644 +--- a/pengine/test10/bug-lf-2508.exp ++++ b/pengine/test10/bug-lf-2508.exp +@@ -363,10 +363,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2551.dot b/pengine/test10/bug-lf-2551.dot +index 3225e00..ed80e15 100644 +--- a/pengine/test10/bug-lf-2551.dot ++++ b/pengine/test10/bug-lf-2551.dot +@@ -80,7 +80,6 @@ digraph "g" { + "stonith 'reboot' hex-9" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "dummy1_start_0 hex-0" [ style = bold] +-"stonith_complete" -> "fencing-sbd_start_0 hex-0" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + "vg1:3_stop_0 hex-9" -> "all_stopped" [ style = bold] + "vg1:3_stop_0 hex-9" -> "base-group:3_stopped_0" [ style = bold] +diff --git a/pengine/test10/bug-lf-2551.exp b/pengine/test10/bug-lf-2551.exp +index 7790199..1912812 100644 +--- a/pengine/test10/bug-lf-2551.exp ++++ b/pengine/test10/bug-lf-2551.exp +@@ -10,9 +10,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/rec-node-11.dot b/pengine/test10/rec-node-11.dot +index 2865f41..bc76888 100644 +--- a/pengine/test10/rec-node-11.dot ++++ b/pengine/test10/rec-node-11.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 node2" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "group1_running_0" [ style=bold color="green" fontcolor="orange" ] + "group1_start_0" -> "group1_running_0" [ style = bold] +@@ -56,6 +57,5 @@ digraph "g" { + "stonith_complete" -> "rsc1_start_0 node2" [ style = bold] + "stonith_complete" -> "rsc2_start_0 node2" [ style = bold] + "stonith_complete" -> "rsc3_start_0 node2" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/rec-node-11.exp b/pengine/test10/rec-node-11.exp +index f441ce8..b6f4c47 100644 +--- a/pengine/test10/rec-node-11.exp ++++ b/pengine/test10/rec-node-11.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-node-11.summary b/pengine/test10/rec-node-11.summary +index d2ad78a..5a37fb8 100644 +--- a/pengine/test10/rec-node-11.summary ++++ b/pengine/test10/rec-node-11.summary +@@ -22,13 +22,13 @@ Executing cluster transition: + * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete +- * Resource action: stonith-1 start on node2 + * Pseudo action: group1_stop_0 + * Pseudo action: rsc2_stop_0 + * Pseudo action: rsc1_stop_0 + * Pseudo action: group1_stopped_0 + * Resource action: rsc3 stop on node2 + * Pseudo action: all_stopped ++ * Resource action: stonith-1 start on node2 + * Resource action: rsc3 start on node2 + * Pseudo action: group1_start_0 + * Resource action: rsc1 start on node2 +diff --git a/pengine/test10/rec-node-15.dot b/pengine/test10/rec-node-15.dot +index 31d0929..71062bf 100644 +--- a/pengine/test10/rec-node-15.dot ++++ b/pengine/test10/rec-node-15.dot +@@ -42,6 +42,7 @@ digraph "g" { + "LVM_22_start_0 sapcl01" -> "LVM_22_monitor_120000 sapcl01" [ style = bold] + "LVM_22_start_0 sapcl01" -> "oracle_running_0" [ style = bold] + "LVM_22_start_0 sapcl01" [ style=bold color="green" fontcolor="black" ] ++"all_stopped" -> "stonith-1_start_0 sapcl01" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "app02_running_0" [ style=bold color="green" fontcolor="orange" ] + "app02_start_0" -> "Filesystem_13_start_0 sapcl01" [ style = bold] +@@ -98,6 +99,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "oracle_24_start_0 sapcl01" [ style = bold] + "stonith_complete" -> "oralsnr_25_start_0 sapcl01" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 sapcl01" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/rec-node-15.exp b/pengine/test10/rec-node-15.exp +index e1330a6..9654d34 100644 +--- a/pengine/test10/rec-node-15.exp ++++ b/pengine/test10/rec-node-15.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-node-15.summary b/pengine/test10/rec-node-15.summary +index 85bfed5..b44a364 100644 +--- a/pengine/test10/rec-node-15.summary ++++ b/pengine/test10/rec-node-15.summary +@@ -39,7 +39,6 @@ Executing cluster transition: + * Fencing sapcl03 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete +- * Resource action: stonith-1 start on sapcl01 + * Resource action: Filesystem_13 stop on sapcl02 + * Resource action: IPaddr_192_168_1_104 start on sapcl01 + * Resource action: LVM_22 start on sapcl01 +@@ -55,6 +54,7 @@ Executing cluster transition: + * Resource action: oralsnr_25 monitor=120000 on sapcl01 + * Resource action: IPaddr_192_168_1_102 stop on sapcl02 + * Pseudo action: all_stopped ++ * Resource action: stonith-1 start on sapcl01 + * Pseudo action: app02_stopped_0 + * Pseudo action: app02_start_0 + * Resource action: IPaddr_192_168_1_102 start on sapcl01 +diff --git a/pengine/test10/rec-node-2.dot b/pengine/test10/rec-node-2.dot +index f8ba887..d889080 100644 +--- a/pengine/test10/rec-node-2.dot ++++ b/pengine/test10/rec-node-2.dot +@@ -1,6 +1,7 @@ + digraph "g" { + "all_stopped" -> "rsc1_start_0 node2" [ style = bold] + "all_stopped" -> "rsc3_start_0 node2" [ style = bold] ++"all_stopped" -> "stonith-1_start_0 node2" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "group1_running_0" [ style=bold color="green" fontcolor="orange" ] + "group1_start_0" -> "group1_running_0" [ style = bold] +@@ -56,6 +57,5 @@ digraph "g" { + "stonith_complete" -> "rsc4_start_0 node2" [ style = bold] + "stonith_complete" -> "rsc5_start_0 node2" [ style = bold] + "stonith_complete" -> "rsc6_start_0 node2" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/rec-node-2.exp b/pengine/test10/rec-node-2.exp +index fd04c37..ea68e3e 100644 +--- a/pengine/test10/rec-node-2.exp ++++ b/pengine/test10/rec-node-2.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-node-4.dot b/pengine/test10/rec-node-4.dot +index e5b31ef..221cefe 100644 +--- a/pengine/test10/rec-node-4.dot ++++ b/pengine/test10/rec-node-4.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 node2" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "probe_complete node2" -> "probe_complete" [ style = bold] + "probe_complete node2" [ style=bold color="green" fontcolor="black" ] +@@ -30,6 +31,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc1_start_0 node2" [ style = bold] + "stonith_complete" -> "rsc2_start_0 node2" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/rec-node-4.exp b/pengine/test10/rec-node-4.exp +index 2cbfecb..4932b98 100644 +--- a/pengine/test10/rec-node-4.exp ++++ b/pengine/test10/rec-node-4.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-node-4.summary b/pengine/test10/rec-node-4.summary +index 295da1d..60f70fc 100644 +--- a/pengine/test10/rec-node-4.summary ++++ b/pengine/test10/rec-node-4.summary +@@ -19,10 +19,10 @@ Executing cluster transition: + * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete +- * Resource action: stonith-1 start on node2 + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 + * Pseudo action: all_stopped ++ * Resource action: stonith-1 start on node2 + * Resource action: rsc1 start on node2 + * Resource action: rsc2 start on node2 + +diff --git a/pengine/test10/rec-node-6.dot b/pengine/test10/rec-node-6.dot +index e5b31ef..221cefe 100644 +--- a/pengine/test10/rec-node-6.dot ++++ b/pengine/test10/rec-node-6.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 node2" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "probe_complete node2" -> "probe_complete" [ style = bold] + "probe_complete node2" [ style=bold color="green" fontcolor="black" ] +@@ -30,6 +31,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc1_start_0 node2" [ style = bold] + "stonith_complete" -> "rsc2_start_0 node2" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/rec-node-6.exp b/pengine/test10/rec-node-6.exp +index 2cbfecb..4932b98 100644 +--- a/pengine/test10/rec-node-6.exp ++++ b/pengine/test10/rec-node-6.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-node-6.summary b/pengine/test10/rec-node-6.summary +index 300d61b..7ae96e5 100644 +--- a/pengine/test10/rec-node-6.summary ++++ b/pengine/test10/rec-node-6.summary +@@ -19,10 +19,10 @@ Executing cluster transition: + * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete +- * Resource action: stonith-1 start on node2 + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 + * Pseudo action: all_stopped ++ * Resource action: stonith-1 start on node2 + * Resource action: rsc1 start on node2 + * Resource action: rsc2 start on node2 + +diff --git a/pengine/test10/rec-node-7.dot b/pengine/test10/rec-node-7.dot +index e5b31ef..221cefe 100644 +--- a/pengine/test10/rec-node-7.dot ++++ b/pengine/test10/rec-node-7.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 node2" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "probe_complete node2" -> "probe_complete" [ style = bold] + "probe_complete node2" [ style=bold color="green" fontcolor="black" ] +@@ -30,6 +31,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc1_start_0 node2" [ style = bold] + "stonith_complete" -> "rsc2_start_0 node2" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/rec-node-7.exp b/pengine/test10/rec-node-7.exp +index 2cbfecb..4932b98 100644 +--- a/pengine/test10/rec-node-7.exp ++++ b/pengine/test10/rec-node-7.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-node-7.summary b/pengine/test10/rec-node-7.summary +index 295da1d..60f70fc 100644 +--- a/pengine/test10/rec-node-7.summary ++++ b/pengine/test10/rec-node-7.summary +@@ -19,10 +19,10 @@ Executing cluster transition: + * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete +- * Resource action: stonith-1 start on node2 + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 + * Pseudo action: all_stopped ++ * Resource action: stonith-1 start on node2 + * Resource action: rsc1 start on node2 + * Resource action: rsc2 start on node2 + +diff --git a/pengine/test10/rec-rsc-5.dot b/pengine/test10/rec-rsc-5.dot +index 53b9521..3ca51f8 100644 +--- a/pengine/test10/rec-rsc-5.dot ++++ b/pengine/test10/rec-rsc-5.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 node1" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "probe_complete node1" -> "probe_complete" [ style = bold] + "probe_complete node1" [ style=bold color="green" fontcolor="black" ] +@@ -30,6 +31,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc1_start_0 node1" [ style = bold] + "stonith_complete" -> "rsc2_start_0 node1" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 node1" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/rec-rsc-5.exp b/pengine/test10/rec-rsc-5.exp +index 95a3558..a99555f 100644 +--- a/pengine/test10/rec-rsc-5.exp ++++ b/pengine/test10/rec-rsc-5.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-5.summary b/pengine/test10/rec-rsc-5.summary +index 549abe6..cdb9011 100644 +--- a/pengine/test10/rec-rsc-5.summary ++++ b/pengine/test10/rec-rsc-5.summary +@@ -19,10 +19,10 @@ Executing cluster transition: + * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete +- * Resource action: stonith-1 start on node1 + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 + * Pseudo action: all_stopped ++ * Resource action: stonith-1 start on node1 + * Resource action: rsc1 start on node1 + * Resource action: rsc2 start on node1 + +diff --git a/pengine/test10/stonith-3.dot b/pengine/test10/stonith-3.dot +index df5f607..86c3bd3 100644 +--- a/pengine/test10/stonith-3.dot ++++ b/pengine/test10/stonith-3.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "prmStonith:0_start_0 rh5node2" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "clnStonith_running_0" [ style=bold color="green" fontcolor="orange" ] + "clnStonith_start_0" -> "clnStonith_running_0" [ style = bold] +@@ -27,6 +28,5 @@ digraph "g" { + "stonith 'reboot' rh5node1" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "prmIpPostgreSQLDB_start_0 rh5node2" [ style = bold] +-"stonith_complete" -> "prmStonith:0_start_0 rh5node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/stonith-3.exp b/pengine/test10/stonith-3.exp +index c4c1e32..4ada6ad 100644 +--- a/pengine/test10/stonith-3.exp ++++ b/pengine/test10/stonith-3.exp +@@ -73,13 +73,13 @@ + + + +- ++ + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/stonith-4.dot b/pengine/test10/stonith-4.dot +index 24243d8..1a7771a 100644 +--- a/pengine/test10/stonith-4.dot ++++ b/pengine/test10/stonith-4.dot +@@ -6,6 +6,7 @@ digraph "g" { + "Fencing_monitor_0 pcmk-3" -> "probe_complete pcmk-3" [ style = dashed] + "Fencing_monitor_0 pcmk-3" [ style=dashed color="red" fontcolor="black"] + "Fencing_start_0 pcmk-1" [ style=bold color="green" fontcolor="black"] ++"all_stopped" -> "Fencing_start_0 pcmk-1" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange"] + "probe_complete pcmk-11" -> "probe_complete" [ style = bold] + "probe_complete pcmk-11" [ style=dashed color="red" fontcolor="black"] +@@ -23,7 +24,6 @@ digraph "g" { + "stonith 'reboot' pcmk-7" [ style=bold color="green" fontcolor="black"] + "stonith 'reboot' pcmk-8" -> "stonith_complete" [ style = bold] + "stonith 'reboot' pcmk-8" [ style=bold color="green" fontcolor="black"] +-"stonith_complete" -> "Fencing_start_0 pcmk-1" [ style = bold] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/stonith-4.exp b/pengine/test10/stonith-4.exp +index 2be0a96..aa61f4d 100644 +--- a/pengine/test10/stonith-4.exp ++++ b/pengine/test10/stonith-4.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/systemhealth2.dot b/pengine/test10/systemhealth2.dot +index dcf49eb..1e029a6 100644 +--- a/pengine/test10/systemhealth2.dot ++++ b/pengine/test10/systemhealth2.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 hs21c" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "apache_1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "apache_1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] +@@ -24,6 +25,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "apache_1_start_0 hs21c" [ style = bold] + "stonith_complete" -> "nfs_1_start_0 hs21c" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealth2.exp b/pengine/test10/systemhealth2.exp +index 95d8847..43ae4e3 100644 +--- a/pengine/test10/systemhealth2.exp ++++ b/pengine/test10/systemhealth2.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/systemhealth3.dot b/pengine/test10/systemhealth3.dot +index dcf49eb..1e029a6 100644 +--- a/pengine/test10/systemhealth3.dot ++++ b/pengine/test10/systemhealth3.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 hs21c" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "apache_1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "apache_1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] +@@ -24,6 +25,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "apache_1_start_0 hs21c" [ style = bold] + "stonith_complete" -> "nfs_1_start_0 hs21c" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealth3.exp b/pengine/test10/systemhealth3.exp +index 95d8847..43ae4e3 100644 +--- a/pengine/test10/systemhealth3.exp ++++ b/pengine/test10/systemhealth3.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/systemhealthm2.dot b/pengine/test10/systemhealthm2.dot +index dcf49eb..1e029a6 100644 +--- a/pengine/test10/systemhealthm2.dot ++++ b/pengine/test10/systemhealthm2.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 hs21c" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "apache_1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "apache_1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] +@@ -24,6 +25,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "apache_1_start_0 hs21c" [ style = bold] + "stonith_complete" -> "nfs_1_start_0 hs21c" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealthm2.exp b/pengine/test10/systemhealthm2.exp +index 95d8847..43ae4e3 100644 +--- a/pengine/test10/systemhealthm2.exp ++++ b/pengine/test10/systemhealthm2.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/systemhealthn2.dot b/pengine/test10/systemhealthn2.dot +index dcf49eb..1e029a6 100644 +--- a/pengine/test10/systemhealthn2.dot ++++ b/pengine/test10/systemhealthn2.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 hs21c" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "apache_1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "apache_1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] +@@ -24,6 +25,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "apache_1_start_0 hs21c" [ style = bold] + "stonith_complete" -> "nfs_1_start_0 hs21c" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealthn2.exp b/pengine/test10/systemhealthn2.exp +index 95d8847..43ae4e3 100644 +--- a/pengine/test10/systemhealthn2.exp ++++ b/pengine/test10/systemhealthn2.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/systemhealthn3.dot b/pengine/test10/systemhealthn3.dot +index dcf49eb..1e029a6 100644 +--- a/pengine/test10/systemhealthn3.dot ++++ b/pengine/test10/systemhealthn3.dot +@@ -1,4 +1,5 @@ + digraph "g" { ++"all_stopped" -> "stonith-1_start_0 hs21c" [ style = bold] + "all_stopped" [ style=bold color="green" fontcolor="orange" ] + "apache_1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "apache_1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] +@@ -24,6 +25,5 @@ digraph "g" { + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "apache_1_start_0 hs21c" [ style = bold] + "stonith_complete" -> "nfs_1_start_0 hs21c" [ style = bold] +-"stonith_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealthn3.exp b/pengine/test10/systemhealthn3.exp +index 95d8847..43ae4e3 100644 +--- a/pengine/test10/systemhealthn3.exp ++++ b/pengine/test10/systemhealthn3.exp +@@ -8,10 +8,10 @@ + + + +- ++ + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-21.dot b/pengine/test10/ticket-master-21.dot +index e1abd95..60386a8 100644 +--- a/pengine/test10/ticket-master-21.dot ++++ b/pengine/test10/ticket-master-21.dot +@@ -27,6 +27,5 @@ digraph "g" { + "stonith 'reboot' node1" -> "stonith_complete" [ style = bold] + "stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] +-"stonith_complete" -> "rsc_stonith_start_0 node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/ticket-master-21.exp b/pengine/test10/ticket-master-21.exp +index 5b21e47..e9482c4 100644 +--- a/pengine/test10/ticket-master-21.exp ++++ b/pengine/test10/ticket-master-21.exp +@@ -10,9 +10,6 @@ + + + +- +- +- + + + +diff --git a/pengine/test10/ticket-master-9.dot b/pengine/test10/ticket-master-9.dot +index 40c2e7f..3a29836 100644 +--- a/pengine/test10/ticket-master-9.dot ++++ b/pengine/test10/ticket-master-9.dot +@@ -27,6 +27,5 @@ digraph "g" { + "stonith 'reboot' node1" -> "stonith_complete" [ style = bold] + "stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] +-"stonith_complete" -> "rsc_stonith_start_0 node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/ticket-master-9.exp b/pengine/test10/ticket-master-9.exp +index 5b21e47..e9482c4 100644 +--- a/pengine/test10/ticket-master-9.exp ++++ b/pengine/test10/ticket-master-9.exp +@@ -10,9 +10,6 @@ + + + +- +- +- + + + diff --git a/SOURCES/bz1078078-pcmk-pe_fix_comments_and_formatting_for_pe_ordering_enum.patch b/SOURCES/bz1078078-pcmk-pe_fix_comments_and_formatting_for_pe_ordering_enum.patch new file mode 100644 index 0000000..f9b3516 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_fix_comments_and_formatting_for_pe_ordering_enum.patch @@ -0,0 +1,55 @@ +commit 75a1451ef00947f7689d42d59efa92503485b8a3 +Author: Andrew Beekhof +Date: Tue Apr 1 19:31:28 2014 +1100 + + Refactor: PE: Fix comments and formatting for pe_ordering enum + + (cherry picked from commit 43596dcacadbdd1199e1238052105914311b151c) + + Conflicts: + include/crm/pengine/status.h + +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index 158a562..c81566a 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -319,28 +319,28 @@ enum pe_link_state { + + /* *INDENT-OFF* */ + enum pe_ordering { +- pe_order_none = 0x0, /* deleted */ +- pe_order_optional = 0x1, /* pure ordering, nothing implied */ ++ pe_order_none = 0x0, /* deleted */ ++ pe_order_optional = 0x1, /* pure ordering, nothing implied */ + +- pe_order_implies_first = 0x10, /* If 'first' is required, ensure 'then' is too */ +- pe_order_implies_then = 0x20, /* If 'then' is required, ensure 'first' is too */ ++ pe_order_implies_first = 0x10, /* If 'then' is required, ensure 'first' is too */ ++ pe_order_implies_then = 0x20, /* If 'first' is required, ensure 'then' is too */ + pe_order_implies_first_master = 0x40, /* Imply 'first' is required when 'then' is required and then's rsc holds Master role. */ + + pe_order_runnable_left = 0x100, /* 'then' requires 'first' to be runnable */ + + pe_order_restart = 0x1000, /* 'then' is runnable if 'first' is optional or runnable */ +- pe_order_stonith_stop = 0x2000, /* only applies if the action is non-pseudo */ +- pe_order_serialize_only = 0x4000, /* serialize */ ++ pe_order_stonith_stop = 0x2000, /* only applies if the action is non-pseudo */ ++ pe_order_serialize_only = 0x4000, /* serialize */ + + pe_order_implies_first_printed = 0x10000, /* Like ..implies_first but only ensures 'first' is printed, not manditory */ +- pe_order_implies_then_printed = 0x20000, /* Like ..implies_then but only ensures 'then' is printed, not manditory */ ++ pe_order_implies_then_printed = 0x20000, /* Like ..implies_then but only ensures 'then' is printed, not manditory */ + +- pe_order_asymmetrical = 0x100000, /* Indicates asymmetrical one way ordering constraint. */ +- pe_order_load = 0x200000, /* Only relevant if... */ +- pe_order_one_or_more = 0x400000, /* 'then' is only runnable if one or more of it's dependancies are too */ ++ pe_order_asymmetrical = 0x100000, /* Indicates asymmetrical one way ordering constraint. */ ++ pe_order_load = 0x200000, /* Only relevant if... */ ++ pe_order_one_or_more = 0x400000, /* 'then' is only runnable if one or more of it's dependancies are too */ + pe_order_anti_colocation = 0x800000, + +- pe_order_trace = 0x4000000 /* test marker */ ++ pe_order_trace = 0x4000000, /* test marker */ + }; + /* *INDENT-ON* */ + diff --git a/SOURCES/bz1078078-pcmk-pe_include_stonith_operation_in_dot_graph_regression_test_results.patch b/SOURCES/bz1078078-pcmk-pe_include_stonith_operation_in_dot_graph_regression_test_results.patch new file mode 100644 index 0000000..fbc2956 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_include_stonith_operation_in_dot_graph_regression_test_results.patch @@ -0,0 +1,1092 @@ +commit dfcfb5f354e6df601878ce6fbcc4ef7baaf8a969 +Author: Andrew Beekhof +Date: Wed Apr 2 15:30:45 2014 +1100 + + Test: PE: Include stonith operation in .dot graph regression test results + + (cherry picked from commit c1fb74e4a3fd39ee94182bfd6f26634cc7bdcdca) + + Conflicts: + pengine/test10/bug-5186-partial-migrate.dot + pengine/test10/remote-fence-unclean.dot + +diff --git a/pengine/test10/594.dot b/pengine/test10/594.dot +index 0e7a507..9e38bc6 100644 +--- a/pengine/test10/594.dot ++++ b/pengine/test10/594.dot +@@ -45,8 +45,8 @@ digraph "g" { + "rsc_hadev2_stop_0 hadev2" [ style=bold color="green" fontcolor="black" ] + "rsc_hadev3_monitor_0 hadev2" -> "probe_complete hadev2" [ style = bold] + "rsc_hadev3_monitor_0 hadev2" [ style=bold color="green" fontcolor="black" ] +-"stonith hadev3" -> "stonith_complete" [ style = bold] +-"stonith hadev3" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hadev3" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hadev3" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "DcIPaddr_start_0 hadev1" [ style = bold] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc_hadev2_start_0 hadev1" [ style = bold] +diff --git a/pengine/test10/829.dot b/pengine/test10/829.dot +index cb41eda..85873cf 100644 +--- a/pengine/test10/829.dot ++++ b/pengine/test10/829.dot +@@ -57,11 +57,11 @@ digraph "g" { + "rsc_c001n08_monitor_0 c001n01" [ style=bold color="green" fontcolor="black" ] + "rsc_c001n08_monitor_0 c001n03" -> "probe_complete c001n03" [ style = bold] + "rsc_c001n08_monitor_0 c001n03" [ style=bold color="green" fontcolor="black" ] +-"stonith c001n02" -> "DoFencing_stop_0" [ style = bold] +-"stonith c001n02" -> "child_DoFencing:0_stop_0 c001n02" [ style = bold] +-"stonith c001n02" -> "rsc_c001n02_stop_0 c001n02" [ style = bold] +-"stonith c001n02" -> "stonith_complete" [ style = bold] +-"stonith c001n02" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' c001n02" -> "DoFencing_stop_0" [ style = bold] ++"stonith 'reboot' c001n02" -> "child_DoFencing:0_stop_0 c001n02" [ style = bold] ++"stonith 'reboot' c001n02" -> "rsc_c001n02_stop_0 c001n02" [ style = bold] ++"stonith 'reboot' c001n02" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' c001n02" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc_c001n02_start_0 c001n01" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] +diff --git a/pengine/test10/bug-lf-2508.dot b/pengine/test10/bug-lf-2508.dot +index 918e4fa..6bef312 100644 +--- a/pengine/test10/bug-lf-2508.dot ++++ b/pengine/test10/bug-lf-2508.dot +@@ -91,19 +91,19 @@ digraph "g" { + "prmStonith4-3:1_stop_0 srv02" -> "grpStonith4:1_stopped_0" [ style = bold] + "prmStonith4-3:1_stop_0 srv02" -> "prmStonith4-1:1_stop_0 srv02" [ style = bold] + "prmStonith4-3:1_stop_0 srv02" [ style=bold color="green" fontcolor="orange"] +-"stonith srv02" -> "Dummy02_stop_0 srv02" [ style = bold] +-"stonith srv02" -> "Group02_stop_0" [ style = bold] +-"stonith srv02" -> "clnStonith1_stop_0" [ style = bold] +-"stonith srv02" -> "clnStonith3_stop_0" [ style = bold] +-"stonith srv02" -> "clnStonith4_stop_0" [ style = bold] +-"stonith srv02" -> "prmStonith1-1:1_stop_0 srv02" [ style = bold] +-"stonith srv02" -> "prmStonith1-3:1_stop_0 srv02" [ style = bold] +-"stonith srv02" -> "prmStonith3-1:1_stop_0 srv02" [ style = bold] +-"stonith srv02" -> "prmStonith3-3:1_stop_0 srv02" [ style = bold] +-"stonith srv02" -> "prmStonith4-1:1_stop_0 srv02" [ style = bold] +-"stonith srv02" -> "prmStonith4-3:1_stop_0 srv02" [ style = bold] +-"stonith srv02" -> "stonith_complete" [ style = bold] +-"stonith srv02" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' srv02" -> "Dummy02_stop_0 srv02" [ style = bold] ++"stonith 'reboot' srv02" -> "Group02_stop_0" [ style = bold] ++"stonith 'reboot' srv02" -> "clnStonith1_stop_0" [ style = bold] ++"stonith 'reboot' srv02" -> "clnStonith3_stop_0" [ style = bold] ++"stonith 'reboot' srv02" -> "clnStonith4_stop_0" [ style = bold] ++"stonith 'reboot' srv02" -> "prmStonith1-1:1_stop_0 srv02" [ style = bold] ++"stonith 'reboot' srv02" -> "prmStonith1-3:1_stop_0 srv02" [ style = bold] ++"stonith 'reboot' srv02" -> "prmStonith3-1:1_stop_0 srv02" [ style = bold] ++"stonith 'reboot' srv02" -> "prmStonith3-3:1_stop_0 srv02" [ style = bold] ++"stonith 'reboot' srv02" -> "prmStonith4-1:1_stop_0 srv02" [ style = bold] ++"stonith 'reboot' srv02" -> "prmStonith4-3:1_stop_0 srv02" [ style = bold] ++"stonith 'reboot' srv02" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' srv02" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "Dummy01_start_0 srv01" [ style = bold] + "stonith_complete" -> "Dummy02_start_0 srv04" [ style = bold] + "stonith_complete" -> "all_stopped" [ style = bold] +diff --git a/pengine/test10/bug-lf-2551.dot b/pengine/test10/bug-lf-2551.dot +index b4a69db..3225e00 100644 +--- a/pengine/test10/bug-lf-2551.dot ++++ b/pengine/test10/bug-lf-2551.dot +@@ -51,33 +51,33 @@ digraph "g" { + "ocfs2-1:3_stop_0 hex-9" -> "base-group:3_stopped_0" [ style = bold] + "ocfs2-1:3_stop_0 hex-9" -> "vg1:3_stop_0 hex-9" [ style = bold] + "ocfs2-1:3_stop_0 hex-9" [ style=bold color="green" fontcolor="orange" ] +-"stonith hex-9" -> "base-clone_stop_0" [ style = bold] +-"stonith hex-9" -> "clvm:3_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "cmirrord:3_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "dlm:3_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "dummy1_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "fencing-sbd_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "o2cb:3_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "ocfs2-1:3_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "stonith_complete" [ style = bold] +-"stonith hex-9" -> "vg1:3_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-03_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-06_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-09_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-13_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-17_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-21_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-25_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-29_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-33_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-37_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-41_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-45_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-49_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-53_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-57_stop_0 hex-9" [ style = bold] +-"stonith hex-9" -> "vm-61_stop_0 hex-9" [ style = bold] +-"stonith hex-9" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hex-9" -> "base-clone_stop_0" [ style = bold] ++"stonith 'reboot' hex-9" -> "clvm:3_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "cmirrord:3_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "dlm:3_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "dummy1_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "fencing-sbd_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "o2cb:3_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "ocfs2-1:3_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hex-9" -> "vg1:3_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-03_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-06_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-09_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-13_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-17_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-21_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-25_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-29_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-33_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-37_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-41_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-45_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-49_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-53_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-57_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" -> "vm-61_stop_0 hex-9" [ style = bold] ++"stonith 'reboot' hex-9" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "dummy1_start_0 hex-0" [ style = bold] + "stonith_complete" -> "fencing-sbd_start_0 hex-0" [ style = bold] +diff --git a/pengine/test10/bug-lf-2606.dot b/pengine/test10/bug-lf-2606.dot +index 403dc1e..612bd02 100644 +--- a/pengine/test10/bug-lf-2606.dot ++++ b/pengine/test10/bug-lf-2606.dot +@@ -24,13 +24,13 @@ digraph "g" { + "rsc3:1_stop_0 node2" -> "all_stopped" [ style = bold] + "rsc3:1_stop_0 node2" -> "ms3_stopped_0" [ style = bold] + "rsc3:1_stop_0 node2" [ style=bold color="green" fontcolor="orange"] +-"stonith node2" -> "ms3_stop_0" [ style = bold] +-"stonith node2" -> "rsc1_stop_0 node2" [ style = bold] +-"stonith node2" -> "rsc2_stop_0 node2" [ style = bold] +-"stonith node2" -> "rsc3:1_demote_0 node2" [ style = bold] +-"stonith node2" -> "rsc3:1_stop_0 node2" [ style = bold] +-"stonith node2" -> "stonith_complete" [ style = bold] +-"stonith node2" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' node2" -> "ms3_stop_0" [ style = bold] ++"stonith 'reboot' node2" -> "rsc1_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "rsc2_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "rsc3:1_demote_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "rsc3:1_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node2" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc2_start_0 node1" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/interleave-pseudo-stop.dot b/pengine/test10/interleave-pseudo-stop.dot +index 8c3ddd3..5b3d272 100644 +--- a/pengine/test10/interleave-pseudo-stop.dot ++++ b/pengine/test10/interleave-pseudo-stop.dot +@@ -91,19 +91,19 @@ digraph "g" { + "imagestorecloneset_stop_0" [ style=bold color="green" fontcolor="orange" ] + "imagestorecloneset_stopped_0" -> "imagestorecloneset_post_notify_stopped_0" [ style = bold] + "imagestorecloneset_stopped_0" [ style=bold color="green" fontcolor="orange" ] +-"stonith node1" -> "configstoreclone:0_post_notify_stonith_0" [ style = bold] +-"stonith node1" -> "configstoreclone:0_stop_0 node1" [ style = bold] +-"stonith node1" -> "configstorecloneset_stop_0" [ style = bold] +-"stonith node1" -> "evmsclone:0_post_notify_stonith_0" [ style = bold] +-"stonith node1" -> "evmsclone:0_stop_0 node1" [ style = bold] +-"stonith node1" -> "evmscloneset_stop_0" [ style = bold] +-"stonith node1" -> "imagestoreclone:0_post_notify_stonith_0" [ style = bold] +-"stonith node1" -> "imagestoreclone:0_stop_0 node1" [ style = bold] +-"stonith node1" -> "imagestorecloneset_stop_0" [ style = bold] +-"stonith node1" -> "stonith_complete" [ style = bold] +-"stonith node1" -> "stonithclone:0_stop_0 node1" [ style = bold] +-"stonith node1" -> "stonithcloneset_stop_0" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node1" -> "configstoreclone:0_post_notify_stonith_0" [ style = bold] ++"stonith 'reboot' node1" -> "configstoreclone:0_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "configstorecloneset_stop_0" [ style = bold] ++"stonith 'reboot' node1" -> "evmsclone:0_post_notify_stonith_0" [ style = bold] ++"stonith 'reboot' node1" -> "evmsclone:0_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "evmscloneset_stop_0" [ style = bold] ++"stonith 'reboot' node1" -> "imagestoreclone:0_post_notify_stonith_0" [ style = bold] ++"stonith 'reboot' node1" -> "imagestoreclone:0_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "imagestorecloneset_stop_0" [ style = bold] ++"stonith 'reboot' node1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node1" -> "stonithclone:0_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "stonithcloneset_stop_0" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + "stonithclone:0_stop_0 node1" -> "all_stopped" [ style = bold] +diff --git a/pengine/test10/master-7.dot b/pengine/test10/master-7.dot +index ae5e08e..61b0f1c 100644 +--- a/pengine/test10/master-7.dot ++++ b/pengine/test10/master-7.dot +@@ -127,16 +127,16 @@ digraph "g" { + "rsc_c001n01_stop_0 c001n01" -> "all_stopped" [ style = bold] + "rsc_c001n01_stop_0 c001n01" -> "rsc_c001n01_start_0 c001n03" [ style = bold] + "rsc_c001n01_stop_0 c001n01" [ style=bold color="green" fontcolor="orange" ] +-"stonith c001n01" -> "DcIPaddr_stop_0 c001n01" [ style = bold] +-"stonith c001n01" -> "DoFencing_stop_0" [ style = bold] +-"stonith c001n01" -> "child_DoFencing:0_stop_0 c001n01" [ style = bold] +-"stonith c001n01" -> "master_rsc_1_stop_0" [ style = bold] +-"stonith c001n01" -> "ocf_msdummy:0_demote_0 c001n01" [ style = bold] +-"stonith c001n01" -> "ocf_msdummy:0_stop_0 c001n01" [ style = bold] +-"stonith c001n01" -> "ocf_msdummy:4_stop_0 c001n01" [ style = bold] +-"stonith c001n01" -> "rsc_c001n01_stop_0 c001n01" [ style = bold] +-"stonith c001n01" -> "stonith_complete" [ style = bold] +-"stonith c001n01" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' c001n01" -> "DcIPaddr_stop_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "DoFencing_stop_0" [ style = bold] ++"stonith 'reboot' c001n01" -> "child_DoFencing:0_stop_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "master_rsc_1_stop_0" [ style = bold] ++"stonith 'reboot' c001n01" -> "ocf_msdummy:0_demote_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "ocf_msdummy:0_stop_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "ocf_msdummy:4_stop_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "rsc_c001n01_stop_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' c001n01" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "DcIPaddr_start_0 c001n03" [ style = bold] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "heartbeat_192.168.100.182_start_0 c001n02" [ style = bold] +diff --git a/pengine/test10/master-8.dot b/pengine/test10/master-8.dot +index a6ed1f6..70be8f1 100644 +--- a/pengine/test10/master-8.dot ++++ b/pengine/test10/master-8.dot +@@ -137,15 +137,15 @@ digraph "g" { + "rsc_c001n01_stop_0 c001n01" -> "all_stopped" [ style = bold] + "rsc_c001n01_stop_0 c001n01" -> "rsc_c001n01_start_0 c001n03" [ style = bold] + "rsc_c001n01_stop_0 c001n01" [ style=bold color="green" fontcolor="orange" ] +-"stonith c001n01" -> "DcIPaddr_stop_0 c001n01" [ style = bold] +-"stonith c001n01" -> "DoFencing_stop_0" [ style = bold] +-"stonith c001n01" -> "child_DoFencing:0_stop_0 c001n01" [ style = bold] +-"stonith c001n01" -> "master_rsc_1_stop_0" [ style = bold] +-"stonith c001n01" -> "ocf_msdummy:0_demote_0 c001n01" [ style = bold] +-"stonith c001n01" -> "ocf_msdummy:0_stop_0 c001n01" [ style = bold] +-"stonith c001n01" -> "rsc_c001n01_stop_0 c001n01" [ style = bold] +-"stonith c001n01" -> "stonith_complete" [ style = bold] +-"stonith c001n01" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' c001n01" -> "DcIPaddr_stop_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "DoFencing_stop_0" [ style = bold] ++"stonith 'reboot' c001n01" -> "child_DoFencing:0_stop_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "master_rsc_1_stop_0" [ style = bold] ++"stonith 'reboot' c001n01" -> "ocf_msdummy:0_demote_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "ocf_msdummy:0_stop_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "rsc_c001n01_stop_0 c001n01" [ style = bold] ++"stonith 'reboot' c001n01" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' c001n01" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "DcIPaddr_start_0 c001n03" [ style = bold] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "heartbeat_192.168.100.182_start_0 c001n02" [ style = bold] +diff --git a/pengine/test10/migrate-fencing.dot b/pengine/test10/migrate-fencing.dot +index 8dd3e3a..1aa8d3c 100644 +--- a/pengine/test10/migrate-fencing.dot ++++ b/pengine/test10/migrate-fencing.dot +@@ -107,21 +107,21 @@ digraph "g" { + "stateful-1:1_promote_0 pcmk-1" [ style=bold color="green" fontcolor="black" ] + "stateful-1:2_monitor_15000 pcmk-2" [ style=bold color="green" fontcolor="black" ] + "stateful-1:3_monitor_15000 pcmk-3" [ style=bold color="green" fontcolor="black" ] +-"stonith pcmk-4" -> "Connectivity_stop_0" [ style = bold] +-"stonith pcmk-4" -> "FencingChild:0_stop_0 pcmk-4" [ style = bold] +-"stonith pcmk-4" -> "Fencing_stop_0" [ style = bold] +-"stonith pcmk-4" -> "group-1_stop_0" [ style = bold] +-"stonith pcmk-4" -> "lsb-dummy_stop_0 pcmk-4" [ style = bold] +-"stonith pcmk-4" -> "master-1_stop_0" [ style = bold] +-"stonith pcmk-4" -> "ping-1:0_stop_0 pcmk-4" [ style = bold] +-"stonith pcmk-4" -> "r192.168.101.181_stop_0 pcmk-4" [ style = bold] +-"stonith pcmk-4" -> "r192.168.101.182_stop_0 pcmk-4" [ style = bold] +-"stonith pcmk-4" -> "r192.168.101.183_stop_0 pcmk-4" [ style = bold] +-"stonith pcmk-4" -> "rsc_pcmk-4_stop_0 pcmk-4" [ style = bold] +-"stonith pcmk-4" -> "stateful-1:0_demote_0 pcmk-4" [ style = bold] +-"stonith pcmk-4" -> "stateful-1:0_stop_0 pcmk-4" [ style = bold] +-"stonith pcmk-4" -> "stonith_complete" [ style = bold] +-"stonith pcmk-4" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' pcmk-4" -> "Connectivity_stop_0" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "FencingChild:0_stop_0 pcmk-4" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "Fencing_stop_0" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "group-1_stop_0" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "lsb-dummy_stop_0 pcmk-4" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "master-1_stop_0" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "ping-1:0_stop_0 pcmk-4" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "r192.168.101.181_stop_0 pcmk-4" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "r192.168.101.182_stop_0 pcmk-4" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "r192.168.101.183_stop_0 pcmk-4" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "rsc_pcmk-4_stop_0 pcmk-4" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "stateful-1:0_demote_0 pcmk-4" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "stateful-1:0_stop_0 pcmk-4" [ style = bold] ++"stonith 'reboot' pcmk-4" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' pcmk-4" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "lsb-dummy_start_0 pcmk-1" [ style = bold] + "stonith_complete" -> "migrator_migrate_to_0 pcmk-1" [ style = bold] +diff --git a/pengine/test10/rec-node-11.dot b/pengine/test10/rec-node-11.dot +index d78cf6b..2865f41 100644 +--- a/pengine/test10/rec-node-11.dot ++++ b/pengine/test10/rec-node-11.dot +@@ -44,11 +44,11 @@ digraph "g" { + "rsc3_stop_0 node2" -> "all_stopped" [ style = bold] + "rsc3_stop_0 node2" -> "rsc3_start_0 node2" [ style = bold] + "rsc3_stop_0 node2" [ style=bold color="green" fontcolor="black" ] +-"stonith node1" -> "group1_stop_0" [ style = bold] +-"stonith node1" -> "rsc1_stop_0 node1" [ style = bold] +-"stonith node1" -> "rsc2_stop_0 node1" [ style = bold] +-"stonith node1" -> "stonith_complete" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node1" -> "group1_stop_0" [ style = bold] ++"stonith 'reboot' node1" -> "rsc1_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc2_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 node2" -> "probe_complete node2" [ style = bold] + "stonith-1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 node2" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/rec-node-12.dot b/pengine/test10/rec-node-12.dot +index 5f79a9a..8275a2e 100644 +--- a/pengine/test10/rec-node-12.dot ++++ b/pengine/test10/rec-node-12.dot +@@ -105,8 +105,8 @@ digraph "g" { + "rsc_c001n08_monitor_5000 c001n08" [ style=bold color="green" fontcolor="black" ] + "rsc_c001n08_start_0 c001n08" -> "rsc_c001n08_monitor_5000 c001n08" [ style = bold] + "rsc_c001n08_start_0 c001n08" [ style=bold color="green" fontcolor="black" ] +-"stonith c001n02" -> "stonith_complete" [ style = bold] +-"stonith c001n02" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' c001n02" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' c001n02" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "DcIPaddr_start_0 c001n08" [ style = bold] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc_c001n01_start_0 c001n01" [ style = bold] +diff --git a/pengine/test10/rec-node-13.dot b/pengine/test10/rec-node-13.dot +index 9a55e66..094c241 100644 +--- a/pengine/test10/rec-node-13.dot ++++ b/pengine/test10/rec-node-13.dot +@@ -7,10 +7,10 @@ + "ocf_msdummy:6_stop_0 c001n04" -> "all_stopped" [ style = bold] + "ocf_msdummy:6_stop_0 c001n04" -> "master_rsc_1_stopped_0" [ style = bold] + "ocf_msdummy:6_stop_0 c001n04" [ style=bold color="green" fontcolor="orange" ] +-"stonith c001n04" -> "master_rsc_1_stop_0" [ style = bold] +-"stonith c001n04" -> "ocf_msdummy:6_stop_0 c001n04" [ style = bold] +-"stonith c001n04" -> "stonith_complete" [ style = bold] +-"stonith c001n04" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' c001n04" -> "master_rsc_1_stop_0" [ style = bold] ++"stonith 'reboot' c001n04" -> "ocf_msdummy:6_stop_0 c001n04" [ style = bold] ++"stonith 'reboot' c001n04" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' c001n04" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/rec-node-14.dot b/pengine/test10/rec-node-14.dot +index 6f5956c..395fa89 100644 +--- a/pengine/test10/rec-node-14.dot ++++ b/pengine/test10/rec-node-14.dot +@@ -1,11 +1,11 @@ + digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange" ] +-"stonith node1" -> "stonith node3" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black" ] +-"stonith node2" [ style=bold color="green" fontcolor="black" ] +-"stonith node3" -> "stonith node2" [ style = bold] +-"stonith node3" -> "stonith_complete" [ style = bold] +-"stonith node3" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node1" -> "stonith 'reboot' node3" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' node2" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' node3" -> "stonith 'reboot' node2" [ style = bold] ++"stonith 'reboot' node3" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node3" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/rec-node-15.dot b/pengine/test10/rec-node-15.dot +index a01ffd2..31d0929 100644 +--- a/pengine/test10/rec-node-15.dot ++++ b/pengine/test10/rec-node-15.dot +@@ -82,8 +82,8 @@ digraph "g" { + "probe_complete" -> "LVM_12_stop_0 sapcl02" [ style = bold] + "probe_complete" -> "stonith-1_start_0 sapcl01" [ style = bold] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith sapcl03" -> "stonith_complete" [ style = bold] +-"stonith sapcl03" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' sapcl03" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' sapcl03" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 sapcl01" -> "probe_complete sapcl01" [ style = bold] + "stonith-1_monitor_0 sapcl01" [ style=bold color="green" fontcolor="black" ] + "stonith-1_monitor_0 sapcl02" -> "probe_complete sapcl02" [ style = bold] +diff --git a/pengine/test10/rec-node-2.dot b/pengine/test10/rec-node-2.dot +index 23b3f2f..f8ba887 100644 +--- a/pengine/test10/rec-node-2.dot ++++ b/pengine/test10/rec-node-2.dot +@@ -46,8 +46,8 @@ digraph "g" { + "rsc6_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "rsc6_start_0 node2" -> "group2_running_0" [ style = bold] + "rsc6_start_0 node2" [ style=bold color="green" fontcolor="black" ] +-"stonith node1" -> "stonith_complete" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 node2" -> "probe_complete node2" [ style = bold] + "stonith-1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 node2" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/rec-node-4.dot b/pengine/test10/rec-node-4.dot +index be10460..e5b31ef 100644 +--- a/pengine/test10/rec-node-4.dot ++++ b/pengine/test10/rec-node-4.dot +@@ -20,10 +20,10 @@ digraph "g" { + "rsc2_stop_0 node1" -> "all_stopped" [ style = bold] + "rsc2_stop_0 node1" -> "rsc2_start_0 node2" [ style = bold] + "rsc2_stop_0 node1" [ style=bold color="green" fontcolor="orange" ] +-"stonith node1" -> "rsc1_stop_0 node1" [ style = bold] +-"stonith node1" -> "rsc2_stop_0 node1" [ style = bold] +-"stonith node1" -> "stonith_complete" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node1" -> "rsc1_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc2_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 node2" -> "probe_complete node2" [ style = bold] + "stonith-1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 node2" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/rec-node-6.dot b/pengine/test10/rec-node-6.dot +index be10460..e5b31ef 100644 +--- a/pengine/test10/rec-node-6.dot ++++ b/pengine/test10/rec-node-6.dot +@@ -20,10 +20,10 @@ digraph "g" { + "rsc2_stop_0 node1" -> "all_stopped" [ style = bold] + "rsc2_stop_0 node1" -> "rsc2_start_0 node2" [ style = bold] + "rsc2_stop_0 node1" [ style=bold color="green" fontcolor="orange" ] +-"stonith node1" -> "rsc1_stop_0 node1" [ style = bold] +-"stonith node1" -> "rsc2_stop_0 node1" [ style = bold] +-"stonith node1" -> "stonith_complete" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node1" -> "rsc1_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc2_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 node2" -> "probe_complete node2" [ style = bold] + "stonith-1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 node2" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/rec-node-7.dot b/pengine/test10/rec-node-7.dot +index be10460..e5b31ef 100644 +--- a/pengine/test10/rec-node-7.dot ++++ b/pengine/test10/rec-node-7.dot +@@ -20,10 +20,10 @@ digraph "g" { + "rsc2_stop_0 node1" -> "all_stopped" [ style = bold] + "rsc2_stop_0 node1" -> "rsc2_start_0 node2" [ style = bold] + "rsc2_stop_0 node1" [ style=bold color="green" fontcolor="orange" ] +-"stonith node1" -> "rsc1_stop_0 node1" [ style = bold] +-"stonith node1" -> "rsc2_stop_0 node1" [ style = bold] +-"stonith node1" -> "stonith_complete" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node1" -> "rsc1_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc2_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 node2" -> "probe_complete node2" [ style = bold] + "stonith-1_monitor_0 node2" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 node2" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/rec-rsc-5.dot b/pengine/test10/rec-rsc-5.dot +index 48bad88..53b9521 100644 +--- a/pengine/test10/rec-rsc-5.dot ++++ b/pengine/test10/rec-rsc-5.dot +@@ -20,10 +20,10 @@ digraph "g" { + "rsc2_stop_0 node2" -> "all_stopped" [ style = bold] + "rsc2_stop_0 node2" -> "rsc2_start_0 node1" [ style = bold] + "rsc2_stop_0 node2" [ style=bold color="green" fontcolor="orange" ] +-"stonith node2" -> "rsc1_stop_0 node2" [ style = bold] +-"stonith node2" -> "rsc2_stop_0 node2" [ style = bold] +-"stonith node2" -> "stonith_complete" [ style = bold] +-"stonith node2" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node2" -> "rsc1_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "rsc2_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node2" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 node1" -> "probe_complete node1" [ style = bold] + "stonith-1_monitor_0 node1" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 node1" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/stonith-0.dot b/pengine/test10/stonith-0.dot +index db60eb9..29cdd59 100644 +--- a/pengine/test10/stonith-0.dot ++++ b/pengine/test10/stonith-0.dot +@@ -66,19 +66,19 @@ digraph "g" { + "rsc_c001n07_stop_0 c001n03" -> "all_stopped" [ style = bold] + "rsc_c001n07_stop_0 c001n03" -> "rsc_c001n07_start_0 c001n07" [ style = bold] + "rsc_c001n07_stop_0 c001n03" [ style=bold color="green" fontcolor="orange" ] +-"stonith c001n03" -> "group-1_stop_0" [ style = bold] +-"stonith c001n03" -> "heartbeat_192.168.100.182_stop_0 c001n03" [ style = bold] +-"stonith c001n03" -> "ocf_192.168.100.181_stop_0 c001n03" [ style = bold] +-"stonith c001n03" -> "ocf_192.168.100.183_stop_0 c001n03" [ style = bold] +-"stonith c001n03" -> "rsc_c001n07_stop_0 c001n03" [ style = bold] +-"stonith c001n03" [ style=bold color="green" fontcolor="black" ] +-"stonith c001n05" -> "group-1_stop_0" [ style = bold] +-"stonith c001n05" -> "ocf_192.168.100.181_stop_0 c001n05" [ style = bold] +-"stonith c001n05" -> "ocf_192.168.100.183_stop_0 c001n05" [ style = bold] +-"stonith c001n05" -> "rsc_c001n05_stop_0 c001n05" [ style = bold] +-"stonith c001n05" -> "stonith c001n03" [ style = bold] +-"stonith c001n05" -> "stonith_complete" [ style = bold] +-"stonith c001n05" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' c001n03" -> "group-1_stop_0" [ style = bold] ++"stonith 'reboot' c001n03" -> "heartbeat_192.168.100.182_stop_0 c001n03" [ style = bold] ++"stonith 'reboot' c001n03" -> "ocf_192.168.100.181_stop_0 c001n03" [ style = bold] ++"stonith 'reboot' c001n03" -> "ocf_192.168.100.183_stop_0 c001n03" [ style = bold] ++"stonith 'reboot' c001n03" -> "rsc_c001n07_stop_0 c001n03" [ style = bold] ++"stonith 'reboot' c001n03" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' c001n05" -> "group-1_stop_0" [ style = bold] ++"stonith 'reboot' c001n05" -> "ocf_192.168.100.181_stop_0 c001n05" [ style = bold] ++"stonith 'reboot' c001n05" -> "ocf_192.168.100.183_stop_0 c001n05" [ style = bold] ++"stonith 'reboot' c001n05" -> "rsc_c001n05_stop_0 c001n05" [ style = bold] ++"stonith 'reboot' c001n05" -> "stonith 'reboot' c001n03" [ style = bold] ++"stonith 'reboot' c001n05" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' c001n05" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "heartbeat_192.168.100.182_start_0 c001n02" [ style = bold] + "stonith_complete" -> "ocf_192.168.100.181_start_0 c001n02" [ style = bold] +diff --git a/pengine/test10/stonith-1.dot b/pengine/test10/stonith-1.dot +index 5ee6c1b..d64edcf 100644 +--- a/pengine/test10/stonith-1.dot ++++ b/pengine/test10/stonith-1.dot +@@ -88,15 +88,15 @@ digraph "g" { + "rsc_sles-3_stop_0 sles-3" -> "rsc_sles-3_start_0 sles-4" [ style = bold] + "rsc_sles-3_stop_0 sles-3" [ style=bold color="green" fontcolor="orange" ] + "rsc_sles-4_monitor_5000 sles-4" [ style=bold color="green" fontcolor="black" ] +-"stonith sles-3" -> "DoFencing_stop_0" [ style = bold] +-"stonith sles-3" -> "child_DoFencing:2_stop_0 sles-3" [ style = bold] +-"stonith sles-3" -> "master_rsc_1_stop_0" [ style = bold] +-"stonith sles-3" -> "migrator_stop_0 sles-3" [ style = bold] +-"stonith sles-3" -> "ocf_msdummy:2_stop_0 sles-3" [ style = bold] +-"stonith sles-3" -> "ocf_msdummy:5_stop_0 sles-3" [ style = bold] +-"stonith sles-3" -> "rsc_sles-3_stop_0 sles-3" [ style = bold] +-"stonith sles-3" -> "stonith_complete" [ style = bold] +-"stonith sles-3" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' sles-3" -> "DoFencing_stop_0" [ style = bold] ++"stonith 'reboot' sles-3" -> "child_DoFencing:2_stop_0 sles-3" [ style = bold] ++"stonith 'reboot' sles-3" -> "master_rsc_1_stop_0" [ style = bold] ++"stonith 'reboot' sles-3" -> "migrator_stop_0 sles-3" [ style = bold] ++"stonith 'reboot' sles-3" -> "ocf_msdummy:2_stop_0 sles-3" [ style = bold] ++"stonith 'reboot' sles-3" -> "ocf_msdummy:5_stop_0 sles-3" [ style = bold] ++"stonith 'reboot' sles-3" -> "rsc_sles-3_stop_0 sles-3" [ style = bold] ++"stonith 'reboot' sles-3" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' sles-3" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "migrator_start_0 sles-4" [ style = bold] + "stonith_complete" -> "ocf_msdummy:0_start_0 sles-4" [ style = bold] +diff --git a/pengine/test10/stonith-2.dot b/pengine/test10/stonith-2.dot +index c5626c7..2e9420b 100644 +--- a/pengine/test10/stonith-2.dot ++++ b/pengine/test10/stonith-2.dot +@@ -3,8 +3,8 @@ digraph "g" { + "rsc_sles-5_monitor_5000 sles-6" [ style=bold color="green" fontcolor="black" ] + "rsc_sles-5_start_0 sles-6" -> "rsc_sles-5_monitor_5000 sles-6" [ style = bold] + "rsc_sles-5_start_0 sles-6" [ style=bold color="green" fontcolor="black" ] +-"stonith sles-5" -> "stonith_complete" [ style = bold] +-"stonith sles-5" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' sles-5" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' sles-5" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc_sles-5_start_0 sles-6" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] +diff --git a/pengine/test10/stonith-3.dot b/pengine/test10/stonith-3.dot +index f93ef79..df5f607 100644 +--- a/pengine/test10/stonith-3.dot ++++ b/pengine/test10/stonith-3.dot +@@ -23,8 +23,8 @@ digraph "g" { + "probe_complete" -> "prmIpPostgreSQLDB_start_0 rh5node2" [ style = bold] + "probe_complete" -> "prmStonith:0_start_0 rh5node2" [ style = bold] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith rh5node1" -> "stonith_complete" [ style = bold] +-"stonith rh5node1" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' rh5node1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' rh5node1" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "prmIpPostgreSQLDB_start_0 rh5node2" [ style = bold] + "stonith_complete" -> "prmStonith:0_start_0 rh5node2" [ style = bold] +diff --git a/pengine/test10/stonith-4.dot b/pengine/test10/stonith-4.dot +index 1508fc8..24243d8 100644 +--- a/pengine/test10/stonith-4.dot ++++ b/pengine/test10/stonith-4.dot +@@ -15,14 +15,14 @@ digraph "g" { + "probe_complete pcmk-3" [ style=dashed color="red" fontcolor="black"] + "probe_complete" -> "Fencing_start_0 pcmk-1" [ style = bold] + "probe_complete" [ style=bold color="green" fontcolor="orange"] +-"stonith pcmk-10" -> "stonith pcmk-5" [ style = bold] +-"stonith pcmk-10" [ style=bold color="green" fontcolor="black"] +-"stonith pcmk-5" -> "stonith pcmk-7" [ style = bold] +-"stonith pcmk-5" [ style=bold color="green" fontcolor="black"] +-"stonith pcmk-7" -> "stonith pcmk-8" [ style = bold] +-"stonith pcmk-7" [ style=bold color="green" fontcolor="black"] +-"stonith pcmk-8" -> "stonith_complete" [ style = bold] +-"stonith pcmk-8" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' pcmk-10" -> "stonith 'reboot' pcmk-5" [ style = bold] ++"stonith 'reboot' pcmk-10" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' pcmk-5" -> "stonith 'reboot' pcmk-7" [ style = bold] ++"stonith 'reboot' pcmk-5" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' pcmk-7" -> "stonith 'reboot' pcmk-8" [ style = bold] ++"stonith 'reboot' pcmk-7" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' pcmk-8" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' pcmk-8" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "Fencing_start_0 pcmk-1" [ style = bold] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/stop-failure-no-quorum.dot b/pengine/test10/stop-failure-no-quorum.dot +index 55faa45..e00258d 100644 +--- a/pengine/test10/stop-failure-no-quorum.dot ++++ b/pengine/test10/stop-failure-no-quorum.dot +@@ -10,10 +10,10 @@ digraph "g" { + "clvm_stop_0 pcmk-2" -> "clvm-clone_stopped_0" [ style = bold] + "clvm_stop_0 pcmk-2" [ style=bold color="green" fontcolor="orange"] + "dlm_monitor_60000 pcmk-1" [ style=dashed color="red" fontcolor="black"] +-"stonith pcmk-2" -> "clvm-clone_stop_0" [ style = bold] +-"stonith pcmk-2" -> "clvm_stop_0 pcmk-2" [ style = bold] +-"stonith pcmk-2" -> "stonith_complete" [ style = bold] +-"stonith pcmk-2" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' pcmk-2" -> "clvm-clone_stop_0" [ style = bold] ++"stonith 'reboot' pcmk-2" -> "clvm_stop_0 pcmk-2" [ style = bold] ++"stonith 'reboot' pcmk-2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' pcmk-2" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/stop-failure-with-fencing.dot b/pengine/test10/stop-failure-with-fencing.dot +index a4dd5f8..cd5c089 100644 +--- a/pengine/test10/stop-failure-with-fencing.dot ++++ b/pengine/test10/stop-failure-with-fencing.dot +@@ -15,10 +15,10 @@ digraph "g" { + "probe_complete pcmk-1" -> "probe_complete" [ style = bold] + "probe_complete pcmk-1" [ style=bold color="green" fontcolor="black"] + "probe_complete" [ style=bold color="green" fontcolor="orange"] +-"stonith pcmk-2" -> "clvm-clone_stop_0" [ style = bold] +-"stonith pcmk-2" -> "clvm_stop_0 pcmk-2" [ style = bold] +-"stonith pcmk-2" -> "stonith_complete" [ style = bold] +-"stonith pcmk-2" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' pcmk-2" -> "clvm-clone_stop_0" [ style = bold] ++"stonith 'reboot' pcmk-2" -> "clvm_stop_0 pcmk-2" [ style = bold] ++"stonith 'reboot' pcmk-2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' pcmk-2" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/systemhealth1.dot b/pengine/test10/systemhealth1.dot +index c47ce24..28841b7 100644 +--- a/pengine/test10/systemhealth1.dot ++++ b/pengine/test10/systemhealth1.dot +@@ -1,9 +1,9 @@ + digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21c" [ style=bold color="green" fontcolor="black" ] +-"stonith hs21d" -> "stonith hs21c" [ style = bold] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21c" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' hs21d" -> "stonith 'reboot' hs21c" [ style = bold] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealth2.dot b/pengine/test10/systemhealth2.dot +index a3d05c0..dcf49eb 100644 +--- a/pengine/test10/systemhealth2.dot ++++ b/pengine/test10/systemhealth2.dot +@@ -16,8 +16,8 @@ digraph "g" { + "probe_complete" -> "nfs_1_start_0 hs21c" [ style = bold] + "probe_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 hs21c" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/systemhealth3.dot b/pengine/test10/systemhealth3.dot +index a3d05c0..dcf49eb 100644 +--- a/pengine/test10/systemhealth3.dot ++++ b/pengine/test10/systemhealth3.dot +@@ -16,8 +16,8 @@ digraph "g" { + "probe_complete" -> "nfs_1_start_0 hs21c" [ style = bold] + "probe_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 hs21c" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/systemhealthm1.dot b/pengine/test10/systemhealthm1.dot +index c47ce24..28841b7 100644 +--- a/pengine/test10/systemhealthm1.dot ++++ b/pengine/test10/systemhealthm1.dot +@@ -1,9 +1,9 @@ + digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21c" [ style=bold color="green" fontcolor="black" ] +-"stonith hs21d" -> "stonith hs21c" [ style = bold] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21c" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' hs21d" -> "stonith 'reboot' hs21c" [ style = bold] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealthm2.dot b/pengine/test10/systemhealthm2.dot +index a3d05c0..dcf49eb 100644 +--- a/pengine/test10/systemhealthm2.dot ++++ b/pengine/test10/systemhealthm2.dot +@@ -16,8 +16,8 @@ digraph "g" { + "probe_complete" -> "nfs_1_start_0 hs21c" [ style = bold] + "probe_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 hs21c" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/systemhealthm3.dot b/pengine/test10/systemhealthm3.dot +index 987de92..2980e3f 100644 +--- a/pengine/test10/systemhealthm3.dot ++++ b/pengine/test10/systemhealthm3.dot +@@ -7,8 +7,8 @@ digraph "g" { + "probe_complete hs21c" -> "probe_complete" [ style = bold] + "probe_complete hs21c" [ style=bold color="green" fontcolor="black" ] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith_complete" -> "all_stopped" [ style = bold] +diff --git a/pengine/test10/systemhealthn1.dot b/pengine/test10/systemhealthn1.dot +index c47ce24..28841b7 100644 +--- a/pengine/test10/systemhealthn1.dot ++++ b/pengine/test10/systemhealthn1.dot +@@ -1,9 +1,9 @@ + digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21c" [ style=bold color="green" fontcolor="black" ] +-"stonith hs21d" -> "stonith hs21c" [ style = bold] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21c" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' hs21d" -> "stonith 'reboot' hs21c" [ style = bold] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealthn2.dot b/pengine/test10/systemhealthn2.dot +index a3d05c0..dcf49eb 100644 +--- a/pengine/test10/systemhealthn2.dot ++++ b/pengine/test10/systemhealthn2.dot +@@ -16,8 +16,8 @@ digraph "g" { + "probe_complete" -> "nfs_1_start_0 hs21c" [ style = bold] + "probe_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 hs21c" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/systemhealthn3.dot b/pengine/test10/systemhealthn3.dot +index a3d05c0..dcf49eb 100644 +--- a/pengine/test10/systemhealthn3.dot ++++ b/pengine/test10/systemhealthn3.dot +@@ -16,8 +16,8 @@ digraph "g" { + "probe_complete" -> "nfs_1_start_0 hs21c" [ style = bold] + "probe_complete" -> "stonith-1_start_0 hs21c" [ style = bold] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith-1_start_0 hs21c" [ style=bold color="green" fontcolor="black" ] +diff --git a/pengine/test10/systemhealtho1.dot b/pengine/test10/systemhealtho1.dot +index c47ce24..28841b7 100644 +--- a/pengine/test10/systemhealtho1.dot ++++ b/pengine/test10/systemhealtho1.dot +@@ -1,9 +1,9 @@ + digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21c" [ style=bold color="green" fontcolor="black" ] +-"stonith hs21d" -> "stonith hs21c" [ style = bold] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21c" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' hs21d" -> "stonith 'reboot' hs21c" [ style = bold] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealtho2.dot b/pengine/test10/systemhealtho2.dot +index 987de92..2980e3f 100644 +--- a/pengine/test10/systemhealtho2.dot ++++ b/pengine/test10/systemhealtho2.dot +@@ -7,8 +7,8 @@ digraph "g" { + "probe_complete hs21c" -> "probe_complete" [ style = bold] + "probe_complete hs21c" [ style=bold color="green" fontcolor="black" ] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith_complete" -> "all_stopped" [ style = bold] +diff --git a/pengine/test10/systemhealtho3.dot b/pengine/test10/systemhealtho3.dot +index 987de92..2980e3f 100644 +--- a/pengine/test10/systemhealtho3.dot ++++ b/pengine/test10/systemhealtho3.dot +@@ -7,8 +7,8 @@ digraph "g" { + "probe_complete hs21c" -> "probe_complete" [ style = bold] + "probe_complete hs21c" [ style=bold color="green" fontcolor="black" ] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith_complete" -> "all_stopped" [ style = bold] +diff --git a/pengine/test10/systemhealthp1.dot b/pengine/test10/systemhealthp1.dot +index c47ce24..28841b7 100644 +--- a/pengine/test10/systemhealthp1.dot ++++ b/pengine/test10/systemhealthp1.dot +@@ -1,9 +1,9 @@ + digraph "g" { + "all_stopped" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21c" [ style=bold color="green" fontcolor="black" ] +-"stonith hs21d" -> "stonith hs21c" [ style = bold] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21c" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' hs21d" -> "stonith 'reboot' hs21c" [ style = bold] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/systemhealthp2.dot b/pengine/test10/systemhealthp2.dot +index 8100587..bb9d914 100644 +--- a/pengine/test10/systemhealthp2.dot ++++ b/pengine/test10/systemhealthp2.dot +@@ -15,8 +15,8 @@ digraph "g" { + "probe_complete" -> "apache_1_start_0 hs21c" [ style = bold] + "probe_complete" -> "nfs_1_start_0 hs21c" [ style = bold] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith_complete" -> "all_stopped" [ style = bold] +diff --git a/pengine/test10/systemhealthp3.dot b/pengine/test10/systemhealthp3.dot +index 987de92..2980e3f 100644 +--- a/pengine/test10/systemhealthp3.dot ++++ b/pengine/test10/systemhealthp3.dot +@@ -7,8 +7,8 @@ digraph "g" { + "probe_complete hs21c" -> "probe_complete" [ style = bold] + "probe_complete hs21c" [ style=bold color="green" fontcolor="black" ] + "probe_complete" [ style=bold color="green" fontcolor="orange" ] +-"stonith hs21d" -> "stonith_complete" [ style = bold] +-"stonith hs21d" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' hs21d" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' hs21d" [ style=bold color="green" fontcolor="black"] + "stonith-1_monitor_0 hs21c" -> "probe_complete hs21c" [ style = bold] + "stonith-1_monitor_0 hs21c" [ style=bold color="green" fontcolor="black" ] + "stonith_complete" -> "all_stopped" [ style = bold] +diff --git a/pengine/test10/ticket-clone-21.dot b/pengine/test10/ticket-clone-21.dot +index 44a702e..c26817e 100644 +--- a/pengine/test10/ticket-clone-21.dot ++++ b/pengine/test10/ticket-clone-21.dot +@@ -13,15 +13,15 @@ digraph "g" { + "rsc1:1_stop_0 node1" [ style=bold color="green" fontcolor="orange"] + "rsc_stonith_stop_0 node1" -> "all_stopped" [ style = bold] + "rsc_stonith_stop_0 node1" [ style=bold color="green" fontcolor="orange"] +-"stonith node1" -> "clone1_stop_0" [ style = bold] +-"stonith node1" -> "rsc1:1_stop_0 node1" [ style = bold] +-"stonith node1" -> "rsc_stonith_stop_0 node1" [ style = bold] +-"stonith node1" -> "stonith node2" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black"] +-"stonith node2" -> "clone1_stop_0" [ style = bold] +-"stonith node2" -> "rsc1:0_stop_0 node2" [ style = bold] +-"stonith node2" -> "stonith_complete" [ style = bold] +-"stonith node2" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' node1" -> "clone1_stop_0" [ style = bold] ++"stonith 'reboot' node1" -> "rsc1:1_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc_stonith_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "stonith 'reboot' node2" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' node2" -> "clone1_stop_0" [ style = bold] ++"stonith 'reboot' node2" -> "rsc1:0_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node2" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/ticket-clone-9.dot b/pengine/test10/ticket-clone-9.dot +index 20066c8..8abddd4 100644 +--- a/pengine/test10/ticket-clone-9.dot ++++ b/pengine/test10/ticket-clone-9.dot +@@ -13,15 +13,15 @@ digraph "g" { + "rsc1:1_stop_0 node1" [ style=bold color="green" fontcolor="orange" ] + "rsc_stonith_stop_0 node1" -> "all_stopped" [ style = bold] + "rsc_stonith_stop_0 node1" [ style=bold color="green" fontcolor="orange"] +-"stonith node1" -> "clone1_stop_0" [ style = bold] +-"stonith node1" -> "rsc1:1_stop_0 node1" [ style = bold] +-"stonith node1" -> "rsc_stonith_stop_0 node1" [ style = bold] +-"stonith node1" -> "stonith node2" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black" ] +-"stonith node2" -> "clone1_stop_0" [ style = bold] +-"stonith node2" -> "rsc1:0_stop_0 node2" [ style = bold] +-"stonith node2" -> "stonith_complete" [ style = bold] +-"stonith node2" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node1" -> "clone1_stop_0" [ style = bold] ++"stonith 'reboot' node1" -> "rsc1:1_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc_stonith_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "stonith 'reboot' node2" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' node2" -> "clone1_stop_0" [ style = bold] ++"stonith 'reboot' node2" -> "rsc1:0_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node2" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/ticket-group-21.dot b/pengine/test10/ticket-group-21.dot +index aeebada..cac1fc7 100644 +--- a/pengine/test10/ticket-group-21.dot ++++ b/pengine/test10/ticket-group-21.dot +@@ -13,11 +13,11 @@ digraph "g" { + "rsc2_stop_0 node2" -> "group1_stopped_0" [ style = bold] + "rsc2_stop_0 node2" -> "rsc1_stop_0 node2" [ style = bold] + "rsc2_stop_0 node2" [ style=bold color="green" fontcolor="orange"] +-"stonith node2" -> "group1_stop_0" [ style = bold] +-"stonith node2" -> "rsc1_stop_0 node2" [ style = bold] +-"stonith node2" -> "rsc2_stop_0 node2" [ style = bold] +-"stonith node2" -> "stonith_complete" [ style = bold] +-"stonith node2" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' node2" -> "group1_stop_0" [ style = bold] ++"stonith 'reboot' node2" -> "rsc1_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "rsc2_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node2" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/ticket-group-9.dot b/pengine/test10/ticket-group-9.dot +index 9991946..1649e62 100644 +--- a/pengine/test10/ticket-group-9.dot ++++ b/pengine/test10/ticket-group-9.dot +@@ -13,11 +13,11 @@ digraph "g" { + "rsc2_stop_0 node2" -> "group1_stopped_0" [ style = bold] + "rsc2_stop_0 node2" -> "rsc1_stop_0 node2" [ style = bold] + "rsc2_stop_0 node2" [ style=bold color="green" fontcolor="orange" ] +-"stonith node2" -> "group1_stop_0" [ style = bold] +-"stonith node2" -> "rsc1_stop_0 node2" [ style = bold] +-"stonith node2" -> "rsc2_stop_0 node2" [ style = bold] +-"stonith node2" -> "stonith_complete" [ style = bold] +-"stonith node2" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node2" -> "group1_stop_0" [ style = bold] ++"stonith 'reboot' node2" -> "rsc1_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "rsc2_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node2" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/pengine/test10/ticket-master-21.dot b/pengine/test10/ticket-master-21.dot +index 9f04de9..e1abd95 100644 +--- a/pengine/test10/ticket-master-21.dot ++++ b/pengine/test10/ticket-master-21.dot +@@ -20,12 +20,12 @@ digraph "g" { + "rsc_stonith_stop_0 node1" -> "all_stopped" [ style = bold] + "rsc_stonith_stop_0 node1" -> "rsc_stonith_start_0 node2" [ style = bold] + "rsc_stonith_stop_0 node1" [ style=bold color="green" fontcolor="orange"] +-"stonith node1" -> "ms1_stop_0" [ style = bold] +-"stonith node1" -> "rsc1:1_demote_0 node1" [ style = bold] +-"stonith node1" -> "rsc1:1_stop_0 node1" [ style = bold] +-"stonith node1" -> "rsc_stonith_stop_0 node1" [ style = bold] +-"stonith node1" -> "stonith_complete" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' node1" -> "ms1_stop_0" [ style = bold] ++"stonith 'reboot' node1" -> "rsc1:1_demote_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc1:1_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc_stonith_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc_stonith_start_0 node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] +diff --git a/pengine/test10/ticket-master-9.dot b/pengine/test10/ticket-master-9.dot +index 91a833a..40c2e7f 100644 +--- a/pengine/test10/ticket-master-9.dot ++++ b/pengine/test10/ticket-master-9.dot +@@ -20,12 +20,12 @@ digraph "g" { + "rsc_stonith_stop_0 node1" -> "all_stopped" [ style = bold] + "rsc_stonith_stop_0 node1" -> "rsc_stonith_start_0 node2" [ style = bold] + "rsc_stonith_stop_0 node1" [ style=bold color="green" fontcolor="orange"] +-"stonith node1" -> "ms1_stop_0" [ style = bold] +-"stonith node1" -> "rsc1:1_demote_0 node1" [ style = bold] +-"stonith node1" -> "rsc1:1_stop_0 node1" [ style = bold] +-"stonith node1" -> "rsc_stonith_stop_0 node1" [ style = bold] +-"stonith node1" -> "stonith_complete" [ style = bold] +-"stonith node1" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node1" -> "ms1_stop_0" [ style = bold] ++"stonith 'reboot' node1" -> "rsc1:1_demote_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc1:1_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "rsc_stonith_stop_0 node1" [ style = bold] ++"stonith 'reboot' node1" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node1" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" -> "rsc_stonith_start_0 node2" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] +diff --git a/pengine/test10/ticket-primitive-21.dot b/pengine/test10/ticket-primitive-21.dot +index f54e0f0..ba07df3 100644 +--- a/pengine/test10/ticket-primitive-21.dot ++++ b/pengine/test10/ticket-primitive-21.dot +@@ -3,9 +3,9 @@ digraph "g" { + "probe_complete node1" [ style=bold color="green" fontcolor="black"] + "rsc1_stop_0 node2" -> "all_stopped" [ style = bold] + "rsc1_stop_0 node2" [ style=bold color="green" fontcolor="orange"] +-"stonith node2" -> "rsc1_stop_0 node2" [ style = bold] +-"stonith node2" -> "stonith_complete" [ style = bold] +-"stonith node2" [ style=bold color="green" fontcolor="black"] ++"stonith 'reboot' node2" -> "rsc1_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node2" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange"] + } +diff --git a/pengine/test10/ticket-primitive-9.dot b/pengine/test10/ticket-primitive-9.dot +index 40a69d7..0036a23 100644 +--- a/pengine/test10/ticket-primitive-9.dot ++++ b/pengine/test10/ticket-primitive-9.dot +@@ -3,9 +3,9 @@ digraph "g" { + "probe_complete node1" [ style=bold color="green" fontcolor="black" ] + "rsc1_stop_0 node2" -> "all_stopped" [ style = bold] + "rsc1_stop_0 node2" [ style=bold color="green" fontcolor="orange" ] +-"stonith node2" -> "rsc1_stop_0 node2" [ style = bold] +-"stonith node2" -> "stonith_complete" [ style = bold] +-"stonith node2" [ style=bold color="green" fontcolor="black" ] ++"stonith 'reboot' node2" -> "rsc1_stop_0 node2" [ style = bold] ++"stonith 'reboot' node2" -> "stonith_complete" [ style = bold] ++"stonith 'reboot' node2" [ style=bold color="green" fontcolor="black"] + "stonith_complete" -> "all_stopped" [ style = bold] + "stonith_complete" [ style=bold color="green" fontcolor="orange" ] + } +diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c +index 2eb68f1..9d20cdd 100644 +--- a/tools/crm_simulate.c ++++ b/tools/crm_simulate.c +@@ -758,7 +758,9 @@ create_action_name(action_t * action) + free(key); + + } else if (safe_str_eq(action->task, CRM_OP_FENCE)) { +- action_name = g_strdup_printf("%s%s %s", prefix ? prefix : "", action->task, action_host); ++ const char *op = g_hash_table_lookup(action->meta, "stonith_action"); ++ ++ action_name = g_strdup_printf("%s%s '%s' %s", prefix ? prefix : "", action->task, op, action_host); + + } else if (action->rsc && action_host) { + action_name = g_strdup_printf("%s%s %s", prefix ? prefix : "", action->uuid, action_host); diff --git a/SOURCES/bz1078078-pcmk-pe_include_stonith_operation_in_the_summary_results_and_handle_unfencing.patch b/SOURCES/bz1078078-pcmk-pe_include_stonith_operation_in_the_summary_results_and_handle_unfencing.patch new file mode 100644 index 0000000..a964e70 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_include_stonith_operation_in_the_summary_results_and_handle_unfencing.patch @@ -0,0 +1,734 @@ +commit 54c57b981935172a28eea3c1e070c408642846b1 +Author: Andrew Beekhof +Date: Wed Apr 2 15:34:26 2014 +1100 + + Test: PE: Include stonith operation in the summary results and handle unfencing + + (cherry picked from commit 8d8de0a3ec77738e5d508b7e746c1d493c9617e1) + + Conflicts: + pengine/test10/bug-5186-partial-migrate.summary + pengine/test10/remote-fence-unclean.summary + +diff --git a/pengine/test10/594.summary b/pengine/test10/594.summary +index 865b386..08de5cc 100644 +--- a/pengine/test10/594.summary ++++ b/pengine/test10/594.summary +@@ -27,7 +27,7 @@ Executing cluster transition: + * Resource action: child_DoFencing:0 monitor on hadev1 + * Resource action: child_DoFencing:2 monitor on hadev2 + * Pseudo action: DoFencing_stop_0 +- * Fencing hadev3 ++ * Fencing hadev3 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Resource action: DcIPaddr start on hadev1 +diff --git a/pengine/test10/829.summary b/pengine/test10/829.summary +index f08a30a..cf23cf0 100644 +--- a/pengine/test10/829.summary ++++ b/pengine/test10/829.summary +@@ -36,7 +36,7 @@ Executing cluster transition: + * Resource action: child_DoFencing:2 monitor on c001n03 + * Resource action: child_DoFencing:3 monitor on c001n03 + * Resource action: child_DoFencing:3 monitor on c001n01 +- * Fencing c001n02 ++ * Fencing c001n02 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: rsc_c001n02_stop_0 +diff --git a/pengine/test10/bug-lf-2508.summary b/pengine/test10/bug-lf-2508.summary +index 4435ced..7f5aed1 100644 +--- a/pengine/test10/bug-lf-2508.summary ++++ b/pengine/test10/bug-lf-2508.summary +@@ -39,7 +39,7 @@ Transition Summary: + Executing cluster transition: + * Pseudo action: Group01_start_0 + * Resource action: prmStonith3-1:1 monitor=3600000 on srv01 +- * Fencing srv02 ++ * Fencing srv02 (reboot) + * Pseudo action: stonith_complete + * Resource action: Dummy01 start on srv01 + * Pseudo action: Group02_stop_0 +diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary +index d299ee9..d362a23 100644 +--- a/pengine/test10/bug-lf-2551.summary ++++ b/pengine/test10/bug-lf-2551.summary +@@ -102,7 +102,7 @@ Transition Summary: + Executing cluster transition: + * Resource action: dummy1 monitor=300000 on hex-8 + * Resource action: dummy1 monitor=300000 on hex-7 +- * Fencing hex-9 ++ * Fencing hex-9 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: load_stopped_hex-8 + * Pseudo action: load_stopped_hex-7 +diff --git a/pengine/test10/bug-lf-2606.summary b/pengine/test10/bug-lf-2606.summary +index c03ed6a..baa0f3b 100644 +--- a/pengine/test10/bug-lf-2606.summary ++++ b/pengine/test10/bug-lf-2606.summary +@@ -17,7 +17,7 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: ms3_demote_0 +- * Fencing node2 ++ * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: rsc2_stop_0 +diff --git a/pengine/test10/interleave-pseudo-stop.summary b/pengine/test10/interleave-pseudo-stop.summary +index 7ac882d..ea86c4f 100644 +--- a/pengine/test10/interleave-pseudo-stop.summary ++++ b/pengine/test10/interleave-pseudo-stop.summary +@@ -22,7 +22,7 @@ Executing cluster transition: + * Pseudo action: evmscloneset_pre_notify_stop_0 + * Pseudo action: imagestorecloneset_pre_notify_stop_0 + * Pseudo action: configstorecloneset_pre_notify_stop_0 +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: stonithcloneset_stop_0 + * Resource action: evmsclone:1 notify on node2 +diff --git a/pengine/test10/master-7.summary b/pengine/test10/master-7.summary +index 5102290..81aeed7 100644 +--- a/pengine/test10/master-7.summary ++++ b/pengine/test10/master-7.summary +@@ -55,7 +55,7 @@ Executing cluster transition: + * Resource action: ocf_msdummy:7 monitor on c001n03 + * Resource action: ocf_msdummy:7 monitor on c001n02 + * Pseudo action: master_rsc_1_demote_0 +- * Fencing c001n01 ++ * Fencing c001n01 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: DcIPaddr_stop_0 +diff --git a/pengine/test10/master-8.summary b/pengine/test10/master-8.summary +index 8828bfe..e12f791 100644 +--- a/pengine/test10/master-8.summary ++++ b/pengine/test10/master-8.summary +@@ -56,7 +56,7 @@ Executing cluster transition: + * Resource action: ocf_msdummy:7 monitor on c001n03 + * Resource action: ocf_msdummy:7 monitor on c001n02 + * Pseudo action: master_rsc_1_demote_0 +- * Fencing c001n01 ++ * Fencing c001n01 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: DcIPaddr_stop_0 +diff --git a/pengine/test10/migrate-fencing.summary b/pengine/test10/migrate-fencing.summary +index 920e754..e8bcd45 100644 +--- a/pengine/test10/migrate-fencing.summary ++++ b/pengine/test10/migrate-fencing.summary +@@ -36,7 +36,7 @@ Transition Summary: + Executing cluster transition: + * Resource action: stateful-1:3 monitor=15000 on pcmk-3 + * Resource action: stateful-1:2 monitor=15000 on pcmk-2 +- * Fencing pcmk-4 ++ * Fencing pcmk-4 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: Fencing_stop_0 + * Pseudo action: rsc_pcmk-4_stop_0 +diff --git a/pengine/test10/rec-node-11.summary b/pengine/test10/rec-node-11.summary +index 5ba0c0d..d2ad78a 100644 +--- a/pengine/test10/rec-node-11.summary ++++ b/pengine/test10/rec-node-11.summary +@@ -19,7 +19,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on node2 + * Resource action: rsc1 monitor on node2 + * Resource action: rsc2 monitor on node2 +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Resource action: stonith-1 start on node2 +diff --git a/pengine/test10/rec-node-12.summary b/pengine/test10/rec-node-12.summary +index eeccb53..4fd63b4 100644 +--- a/pengine/test10/rec-node-12.summary ++++ b/pengine/test10/rec-node-12.summary +@@ -53,7 +53,7 @@ Executing cluster transition: + * Resource action: child_DoFencing:3 monitor on c001n03 + * Resource action: child_DoFencing:3 monitor on c001n01 + * Pseudo action: DoFencing_start_0 +- * Fencing c001n02 ++ * Fencing c001n02 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/rec-node-13.summary b/pengine/test10/rec-node-13.summary +index 356b67d..82f210a 100644 +--- a/pengine/test10/rec-node-13.summary ++++ b/pengine/test10/rec-node-13.summary +@@ -37,7 +37,7 @@ Transition Summary: + * Stop ocf_msdummy:6 (c001n04) + + Executing cluster transition: +- * Fencing c001n04 ++ * Fencing c001n04 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: master_rsc_1_stop_0 + * Pseudo action: ocf_msdummy:6_stop_0 +diff --git a/pengine/test10/rec-node-14.summary b/pengine/test10/rec-node-14.summary +index 231decd..d97d943 100644 +--- a/pengine/test10/rec-node-14.summary ++++ b/pengine/test10/rec-node-14.summary +@@ -10,9 +10,9 @@ Node node3 (uuid3): UNCLEAN (offline) + Transition Summary: + + Executing cluster transition: +- * Fencing node1 +- * Fencing node3 +- * Fencing node2 ++ * Fencing node1 (reboot) ++ * Fencing node3 (reboot) ++ * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped + +diff --git a/pengine/test10/rec-node-15.summary b/pengine/test10/rec-node-15.summary +index 570433c..85bfed5 100644 +--- a/pengine/test10/rec-node-15.summary ++++ b/pengine/test10/rec-node-15.summary +@@ -36,7 +36,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on sapcl01 + * Pseudo action: app02_stop_0 + * Pseudo action: oracle_start_0 +- * Fencing sapcl03 ++ * Fencing sapcl03 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Resource action: stonith-1 start on sapcl01 +diff --git a/pengine/test10/rec-node-2.summary b/pengine/test10/rec-node-2.summary +index 645ef46..daa2879 100644 +--- a/pengine/test10/rec-node-2.summary ++++ b/pengine/test10/rec-node-2.summary +@@ -32,7 +32,7 @@ Executing cluster transition: + * Pseudo action: group2_start_0 + * Resource action: rsc5 monitor on node2 + * Resource action: rsc6 monitor on node2 +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/rec-node-4.summary b/pengine/test10/rec-node-4.summary +index 782c377..295da1d 100644 +--- a/pengine/test10/rec-node-4.summary ++++ b/pengine/test10/rec-node-4.summary +@@ -16,7 +16,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on node2 + * Resource action: rsc1 monitor on node2 + * Resource action: rsc2 monitor on node2 +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Resource action: stonith-1 start on node2 +diff --git a/pengine/test10/rec-node-6.summary b/pengine/test10/rec-node-6.summary +index 2bd563b..300d61b 100644 +--- a/pengine/test10/rec-node-6.summary ++++ b/pengine/test10/rec-node-6.summary +@@ -16,7 +16,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on node2 + * Resource action: rsc1 monitor on node2 + * Resource action: rsc2 monitor on node2 +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Resource action: stonith-1 start on node2 +diff --git a/pengine/test10/rec-node-7.summary b/pengine/test10/rec-node-7.summary +index 782c377..295da1d 100644 +--- a/pengine/test10/rec-node-7.summary ++++ b/pengine/test10/rec-node-7.summary +@@ -16,7 +16,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on node2 + * Resource action: rsc1 monitor on node2 + * Resource action: rsc2 monitor on node2 +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Resource action: stonith-1 start on node2 +diff --git a/pengine/test10/rec-rsc-5.summary b/pengine/test10/rec-rsc-5.summary +index 9c9239b..549abe6 100644 +--- a/pengine/test10/rec-rsc-5.summary ++++ b/pengine/test10/rec-rsc-5.summary +@@ -16,7 +16,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on node1 + * Resource action: rsc1 monitor on node1 + * Resource action: rsc2 monitor on node1 +- * Fencing node2 ++ * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Resource action: stonith-1 start on node1 +diff --git a/pengine/test10/stonith-0.summary b/pengine/test10/stonith-0.summary +index 9330291..4ec1ff7 100644 +--- a/pengine/test10/stonith-0.summary ++++ b/pengine/test10/stonith-0.summary +@@ -45,8 +45,8 @@ Transition Summary: + + Executing cluster transition: + * Resource action: child_DoFencing:4 monitor=20000 on c001n08 +- * Fencing c001n05 +- * Fencing c001n03 ++ * Fencing c001n05 (reboot) ++ * Fencing c001n03 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: group-1_stop_0 + * Pseudo action: ocf_192.168.100.183_stop_0 +diff --git a/pengine/test10/stonith-1.summary b/pengine/test10/stonith-1.summary +index d3e81db..ed83ad7 100644 +--- a/pengine/test10/stonith-1.summary ++++ b/pengine/test10/stonith-1.summary +@@ -44,7 +44,7 @@ Executing cluster transition: + * Resource action: lsb_dummy monitor=5000 on sles-2 + * Resource action: rsc_sles-2 monitor=5000 on sles-2 + * Resource action: rsc_sles-4 monitor=5000 on sles-4 +- * Fencing sles-3 ++ * Fencing sles-3 (reboot) + * Pseudo action: stonith_complete + * Resource action: r192.168.100.183 start on sles-1 + * Pseudo action: migrator_stop_0 +diff --git a/pengine/test10/stonith-2.summary b/pengine/test10/stonith-2.summary +index f02dd74..169f44f 100644 +--- a/pengine/test10/stonith-2.summary ++++ b/pengine/test10/stonith-2.summary +@@ -36,7 +36,7 @@ Transition Summary: + * Start rsc_sles-5 (sles-6) + + Executing cluster transition: +- * Fencing sles-5 ++ * Fencing sles-5 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped + * Resource action: rsc_sles-5 start on sles-6 +diff --git a/pengine/test10/stonith-3.summary b/pengine/test10/stonith-3.summary +index b5b6d8a..095ce73 100644 +--- a/pengine/test10/stonith-3.summary ++++ b/pengine/test10/stonith-3.summary +@@ -15,7 +15,7 @@ Executing cluster transition: + * Resource action: prmIpPostgreSQLDB monitor on rh5node2 + * Resource action: prmStonith:0 monitor on rh5node2 + * Pseudo action: clnStonith_start_0 +- * Fencing rh5node1 ++ * Fencing rh5node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/stonith-4.summary b/pengine/test10/stonith-4.summary +index 1f87015..2e12deb 100644 +--- a/pengine/test10/stonith-4.summary ++++ b/pengine/test10/stonith-4.summary +@@ -17,10 +17,10 @@ Transition Summary: + * Start Fencing (pcmk-1) + + Executing cluster transition: +- * Fencing pcmk-10 +- * Fencing pcmk-5 +- * Fencing pcmk-7 +- * Fencing pcmk-8 ++ * Fencing pcmk-10 (reboot) ++ * Fencing pcmk-5 (reboot) ++ * Fencing pcmk-7 (reboot) ++ * Fencing pcmk-8 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped + Transition failed: terminated +diff --git a/pengine/test10/stop-failure-no-quorum.summary b/pengine/test10/stop-failure-no-quorum.summary +index 49a0f54..d57bdc6 100644 +--- a/pengine/test10/stop-failure-no-quorum.summary ++++ b/pengine/test10/stop-failure-no-quorum.summary +@@ -22,7 +22,7 @@ Transition Summary: + * Start Fencing (pcmk-1 - blocked) + + Executing cluster transition: +- * Fencing pcmk-2 ++ * Fencing pcmk-2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: clvm-clone_stop_0 + * Pseudo action: clvm_stop_0 +diff --git a/pengine/test10/stop-failure-with-fencing.summary b/pengine/test10/stop-failure-with-fencing.summary +index 0473b80..de74dbc 100644 +--- a/pengine/test10/stop-failure-with-fencing.summary ++++ b/pengine/test10/stop-failure-with-fencing.summary +@@ -22,7 +22,7 @@ Transition Summary: + + Executing cluster transition: + * Resource action: Fencing monitor on pcmk-1 +- * Fencing pcmk-2 ++ * Fencing pcmk-2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: clvm-clone_stop_0 +diff --git a/pengine/test10/systemhealth1.summary b/pengine/test10/systemhealth1.summary +index 49c6579..bbdbcaf 100644 +--- a/pengine/test10/systemhealth1.summary ++++ b/pengine/test10/systemhealth1.summary +@@ -10,8 +10,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + Transition Summary: + + Executing cluster transition: +- * Fencing hs21d +- * Fencing hs21c ++ * Fencing hs21d (reboot) ++ * Fencing hs21c (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped + +diff --git a/pengine/test10/systemhealth2.summary b/pengine/test10/systemhealth2.summary +index dfdfa02..0ca09d0 100644 +--- a/pengine/test10/systemhealth2.summary ++++ b/pengine/test10/systemhealth2.summary +@@ -16,7 +16,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealth3.summary b/pengine/test10/systemhealth3.summary +index dfdfa02..0ca09d0 100644 +--- a/pengine/test10/systemhealth3.summary ++++ b/pengine/test10/systemhealth3.summary +@@ -16,7 +16,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealthm1.summary b/pengine/test10/systemhealthm1.summary +index 49c6579..bbdbcaf 100644 +--- a/pengine/test10/systemhealthm1.summary ++++ b/pengine/test10/systemhealthm1.summary +@@ -10,8 +10,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + Transition Summary: + + Executing cluster transition: +- * Fencing hs21d +- * Fencing hs21c ++ * Fencing hs21d (reboot) ++ * Fencing hs21c (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped + +diff --git a/pengine/test10/systemhealthm2.summary b/pengine/test10/systemhealthm2.summary +index dfdfa02..0ca09d0 100644 +--- a/pengine/test10/systemhealthm2.summary ++++ b/pengine/test10/systemhealthm2.summary +@@ -16,7 +16,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealthm3.summary b/pengine/test10/systemhealthm3.summary +index 6964943..7fcab05 100644 +--- a/pengine/test10/systemhealthm3.summary ++++ b/pengine/test10/systemhealthm3.summary +@@ -13,7 +13,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealthn1.summary b/pengine/test10/systemhealthn1.summary +index 49c6579..bbdbcaf 100644 +--- a/pengine/test10/systemhealthn1.summary ++++ b/pengine/test10/systemhealthn1.summary +@@ -10,8 +10,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + Transition Summary: + + Executing cluster transition: +- * Fencing hs21d +- * Fencing hs21c ++ * Fencing hs21d (reboot) ++ * Fencing hs21c (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped + +diff --git a/pengine/test10/systemhealthn2.summary b/pengine/test10/systemhealthn2.summary +index dfdfa02..0ca09d0 100644 +--- a/pengine/test10/systemhealthn2.summary ++++ b/pengine/test10/systemhealthn2.summary +@@ -16,7 +16,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealthn3.summary b/pengine/test10/systemhealthn3.summary +index dfdfa02..0ca09d0 100644 +--- a/pengine/test10/systemhealthn3.summary ++++ b/pengine/test10/systemhealthn3.summary +@@ -16,7 +16,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealtho1.summary b/pengine/test10/systemhealtho1.summary +index 49c6579..bbdbcaf 100644 +--- a/pengine/test10/systemhealtho1.summary ++++ b/pengine/test10/systemhealtho1.summary +@@ -10,8 +10,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + Transition Summary: + + Executing cluster transition: +- * Fencing hs21d +- * Fencing hs21c ++ * Fencing hs21d (reboot) ++ * Fencing hs21c (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped + +diff --git a/pengine/test10/systemhealtho2.summary b/pengine/test10/systemhealtho2.summary +index 6964943..7fcab05 100644 +--- a/pengine/test10/systemhealtho2.summary ++++ b/pengine/test10/systemhealtho2.summary +@@ -13,7 +13,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealtho3.summary b/pengine/test10/systemhealtho3.summary +index 6964943..7fcab05 100644 +--- a/pengine/test10/systemhealtho3.summary ++++ b/pengine/test10/systemhealtho3.summary +@@ -13,7 +13,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealthp1.summary b/pengine/test10/systemhealthp1.summary +index 49c6579..bbdbcaf 100644 +--- a/pengine/test10/systemhealthp1.summary ++++ b/pengine/test10/systemhealthp1.summary +@@ -10,8 +10,8 @@ Node hs21d (737318c6-0f92-4592-9754-45967d45aff7): UNCLEAN (offline) + Transition Summary: + + Executing cluster transition: +- * Fencing hs21d +- * Fencing hs21c ++ * Fencing hs21d (reboot) ++ * Fencing hs21c (reboot) + * Pseudo action: stonith_complete + * Pseudo action: all_stopped + +diff --git a/pengine/test10/systemhealthp2.summary b/pengine/test10/systemhealthp2.summary +index f01150f..12a888c 100644 +--- a/pengine/test10/systemhealthp2.summary ++++ b/pengine/test10/systemhealthp2.summary +@@ -15,7 +15,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/systemhealthp3.summary b/pengine/test10/systemhealthp3.summary +index 6964943..7fcab05 100644 +--- a/pengine/test10/systemhealthp3.summary ++++ b/pengine/test10/systemhealthp3.summary +@@ -13,7 +13,7 @@ Executing cluster transition: + * Resource action: stonith-1 monitor on hs21c + * Resource action: apache_1 monitor on hs21c + * Resource action: nfs_1 monitor on hs21c +- * Fencing hs21d ++ * Fencing hs21d (reboot) + * Pseudo action: stonith_complete + * Pseudo action: probe_complete + * Pseudo action: all_stopped +diff --git a/pengine/test10/ticket-clone-21.summary b/pengine/test10/ticket-clone-21.summary +index 7973159..5ae23ec 100644 +--- a/pengine/test10/ticket-clone-21.summary ++++ b/pengine/test10/ticket-clone-21.summary +@@ -12,9 +12,9 @@ Transition Summary: + * Stop rsc1:1 (node2) + + Executing cluster transition: +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: rsc_stonith_stop_0 +- * Fencing node2 ++ * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: clone1_stop_0 + * Pseudo action: rsc1:1_stop_0 +diff --git a/pengine/test10/ticket-clone-9.summary b/pengine/test10/ticket-clone-9.summary +index 7973159..5ae23ec 100644 +--- a/pengine/test10/ticket-clone-9.summary ++++ b/pengine/test10/ticket-clone-9.summary +@@ -12,9 +12,9 @@ Transition Summary: + * Stop rsc1:1 (node2) + + Executing cluster transition: +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: rsc_stonith_stop_0 +- * Fencing node2 ++ * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: clone1_stop_0 + * Pseudo action: rsc1:1_stop_0 +diff --git a/pengine/test10/ticket-group-21.summary b/pengine/test10/ticket-group-21.summary +index c0783e0..9027d2c 100644 +--- a/pengine/test10/ticket-group-21.summary ++++ b/pengine/test10/ticket-group-21.summary +@@ -12,7 +12,7 @@ Transition Summary: + * Stop rsc2 (node2) + + Executing cluster transition: +- * Fencing node2 ++ * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: group1_stop_0 + * Pseudo action: rsc2_stop_0 +diff --git a/pengine/test10/ticket-group-9.summary b/pengine/test10/ticket-group-9.summary +index c0783e0..9027d2c 100644 +--- a/pengine/test10/ticket-group-9.summary ++++ b/pengine/test10/ticket-group-9.summary +@@ -12,7 +12,7 @@ Transition Summary: + * Stop rsc2 (node2) + + Executing cluster transition: +- * Fencing node2 ++ * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: group1_stop_0 + * Pseudo action: rsc2_stop_0 +diff --git a/pengine/test10/ticket-master-21.summary b/pengine/test10/ticket-master-21.summary +index ca5bf84..64a9cbe 100644 +--- a/pengine/test10/ticket-master-21.summary ++++ b/pengine/test10/ticket-master-21.summary +@@ -13,7 +13,7 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: rsc_stonith_stop_0 + * Pseudo action: rsc1:1_demote_0 +diff --git a/pengine/test10/ticket-master-9.summary b/pengine/test10/ticket-master-9.summary +index ca5bf84..64a9cbe 100644 +--- a/pengine/test10/ticket-master-9.summary ++++ b/pengine/test10/ticket-master-9.summary +@@ -13,7 +13,7 @@ Transition Summary: + + Executing cluster transition: + * Pseudo action: ms1_demote_0 +- * Fencing node1 ++ * Fencing node1 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: rsc_stonith_stop_0 + * Pseudo action: rsc1:1_demote_0 +diff --git a/pengine/test10/ticket-primitive-21.summary b/pengine/test10/ticket-primitive-21.summary +index 3afd279..6c523fe 100644 +--- a/pengine/test10/ticket-primitive-21.summary ++++ b/pengine/test10/ticket-primitive-21.summary +@@ -9,7 +9,7 @@ Transition Summary: + * Stop rsc1 (node2) + + Executing cluster transition: +- * Fencing node2 ++ * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: all_stopped +diff --git a/pengine/test10/ticket-primitive-9.summary b/pengine/test10/ticket-primitive-9.summary +index 3afd279..6c523fe 100644 +--- a/pengine/test10/ticket-primitive-9.summary ++++ b/pengine/test10/ticket-primitive-9.summary +@@ -9,7 +9,7 @@ Transition Summary: + * Stop rsc1 (node2) + + Executing cluster transition: +- * Fencing node2 ++ * Fencing node2 (reboot) + * Pseudo action: stonith_complete + * Pseudo action: rsc1_stop_0 + * Pseudo action: all_stopped +diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c +index 9d20cdd..e81d4ee 100644 +--- a/tools/crm_simulate.c ++++ b/tools/crm_simulate.c +@@ -497,31 +497,36 @@ exec_crmd_action(crm_graph_t * graph, crm_action_t * action) + static gboolean + exec_stonith_action(crm_graph_t * graph, crm_action_t * action) + { +- int rc = 0; +- char xpath[STATUS_PATH_MAX]; ++ const char *op = crm_meta_value(action->params, "stonith_action"); + char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET); +- xmlNode *cib_node = modify_node(global_cib, target, FALSE); + +- crm_xml_add(cib_node, XML_ATTR_ORIGIN, __FUNCTION__); +- CRM_ASSERT(cib_node != NULL); ++ quiet_log(" * Fencing %s (%s)\n", target, op); ++ if(safe_str_neq(op, "on")) { ++ int rc = 0; ++ char xpath[STATUS_PATH_MAX]; ++ xmlNode *cib_node = modify_node(global_cib, target, FALSE); + +- quiet_log(" * Fencing %s\n", target); +- rc = global_cib->cmds->replace(global_cib, XML_CIB_TAG_STATUS, cib_node, ++ crm_xml_add(cib_node, XML_ATTR_ORIGIN, __FUNCTION__); ++ CRM_ASSERT(cib_node != NULL); ++ ++ rc = global_cib->cmds->replace(global_cib, XML_CIB_TAG_STATUS, cib_node, + cib_sync_call | cib_scope_local); +- CRM_ASSERT(rc == pcmk_ok); ++ CRM_ASSERT(rc == pcmk_ok); + +- snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM); +- rc = global_cib->cmds->delete(global_cib, xpath, NULL, +- cib_xpath | cib_sync_call | cib_scope_local); ++ snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, XML_CIB_TAG_LRM); ++ rc = global_cib->cmds->delete(global_cib, xpath, NULL, ++ cib_xpath | cib_sync_call | cib_scope_local); + +- snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, +- XML_TAG_TRANSIENT_NODEATTRS); +- rc = global_cib->cmds->delete(global_cib, xpath, NULL, +- cib_xpath | cib_sync_call | cib_scope_local); ++ snprintf(xpath, STATUS_PATH_MAX, "//node_state[@uname='%s']/%s", target, ++ XML_TAG_TRANSIENT_NODEATTRS); ++ rc = global_cib->cmds->delete(global_cib, xpath, NULL, ++ cib_xpath | cib_sync_call | cib_scope_local); ++ ++ free_xml(cib_node); ++ } + + action->confirmed = TRUE; + update_graph(graph, action); +- free_xml(cib_node); + free(target); + return TRUE; + } diff --git a/SOURCES/bz1078078-pcmk-pe_indicate_the_type_of_fencing_in_the_fencing_operation_key.patch b/SOURCES/bz1078078-pcmk-pe_indicate_the_type_of_fencing_in_the_fencing_operation_key.patch new file mode 100644 index 0000000..af405dc --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_indicate_the_type_of_fencing_in_the_fencing_operation_key.patch @@ -0,0 +1,3230 @@ +commit cce2536efbed5b3bed24e6ad13532dc4db55d4ae +Author: Andrew Beekhof +Date: Tue Apr 1 15:13:21 2014 +1100 + + Refactor: PE: Indicate the type of fencing in the fencing operation key + + (cherry picked from commit 22dbe7cdda718563d02f65b0009b1984497a0947) + + Conflicts: + pengine/test10/594.exp + pengine/test10/829.exp + pengine/test10/bug-5186-partial-migrate.exp + pengine/test10/bug-lf-2508.exp + pengine/test10/bug-lf-2551.exp + pengine/test10/bug-lf-2606.exp + pengine/test10/interleave-pseudo-stop.exp + pengine/test10/master-7.exp + pengine/test10/master-8.exp + pengine/test10/migrate-fencing.exp + pengine/test10/rec-node-11.exp + pengine/test10/rec-node-12.exp + pengine/test10/rec-node-13.exp + pengine/test10/rec-node-14.exp + pengine/test10/rec-node-15.exp + pengine/test10/rec-node-2.exp + pengine/test10/rec-node-4.exp + pengine/test10/rec-node-6.exp + pengine/test10/rec-node-7.exp + pengine/test10/rec-rsc-5.exp + pengine/test10/remote-fence-unclean.exp + pengine/test10/stonith-0.exp + pengine/test10/stonith-1.exp + pengine/test10/stonith-2.exp + pengine/test10/stonith-3.exp + pengine/test10/stonith-4.exp + pengine/test10/stop-failure-no-quorum.exp + pengine/test10/stop-failure-with-fencing.exp + pengine/test10/systemhealth1.exp + pengine/test10/systemhealth2.exp + pengine/test10/systemhealth3.exp + pengine/test10/systemhealthm1.exp + pengine/test10/systemhealthm2.exp + pengine/test10/systemhealthm3.exp + pengine/test10/systemhealthn1.exp + pengine/test10/systemhealthn2.exp + pengine/test10/systemhealthn3.exp + pengine/test10/systemhealtho1.exp + pengine/test10/systemhealtho2.exp + pengine/test10/systemhealtho3.exp + pengine/test10/systemhealthp1.exp + pengine/test10/systemhealthp2.exp + pengine/test10/systemhealthp3.exp + pengine/test10/ticket-clone-21.exp + pengine/test10/ticket-clone-9.exp + pengine/test10/ticket-group-21.exp + pengine/test10/ticket-group-9.exp + pengine/test10/ticket-master-21.exp + pengine/test10/ticket-master-9.exp + pengine/test10/ticket-primitive-21.exp + pengine/test10/ticket-primitive-9.exp + +diff --git a/pengine/allocate.c b/pengine/allocate.c +index e535b84..a8e5412 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1265,15 +1265,21 @@ any_managed_resources(pe_working_set_t * data_set) + action_t * + pe_fence_op(node_t * node, const char *op, pe_working_set_t * data_set) + { +- action_t *stonith_op = +- custom_action(NULL, g_strdup_printf("%s-%s", CRM_OP_FENCE, node->details->uname), +- CRM_OP_FENCE, node, FALSE, TRUE, data_set); ++ action_t *stonith_op = NULL; ++ ++ if(op == NULL) { ++ op = data_set->stonith_action; ++ } ++ ++ stonith_op = custom_action( ++ NULL, g_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op), ++ CRM_OP_FENCE, node, FALSE, TRUE, data_set); + + add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname); + + add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id); + +- add_hash_param(stonith_op->meta, "stonith_action", op ? op : data_set->stonith_action); ++ add_hash_param(stonith_op->meta, "stonith_action", op); + + return stonith_op; + } +diff --git a/pengine/test10/594.exp b/pengine/test10/594.exp +index 5cb6931..f1e77f8 100644 +--- a/pengine/test10/594.exp ++++ b/pengine/test10/594.exp +@@ -183,7 +183,7 @@ + + + +- ++ + + + +@@ -215,7 +215,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/829.exp b/pengine/test10/829.exp +index 4562aed..749014f 100644 +--- a/pengine/test10/829.exp ++++ b/pengine/test10/829.exp +@@ -78,7 +78,7 @@ + + + +- ++ + + + +@@ -159,7 +159,7 @@ + + + +- ++ + + + +@@ -240,13 +240,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -260,7 +260,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-5186-partial-migrate.exp b/pengine/test10/bug-5186-partial-migrate.exp +new file mode 100644 +index 0000000..7e10e3a +--- /dev/null ++++ b/pengine/test10/bug-5186-partial-migrate.exp +@@ -0,0 +1,425 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/bug-lf-2508.exp b/pengine/test10/bug-lf-2508.exp +index 3e54b30..d56002e 100644 +--- a/pengine/test10/bug-lf-2508.exp ++++ b/pengine/test10/bug-lf-2508.exp +@@ -74,7 +74,7 @@ + + + +- ++ + + + +@@ -148,7 +148,7 @@ + + + +- ++ + + + +@@ -196,7 +196,7 @@ + + + +- ++ + + + +@@ -211,7 +211,7 @@ + + + +- ++ + + + +@@ -238,7 +238,7 @@ + + + +- ++ + + + +@@ -286,7 +286,7 @@ + + + +- ++ + + + +@@ -301,7 +301,7 @@ + + + +- ++ + + + +@@ -393,7 +393,7 @@ + + + +- ++ + + + +@@ -468,7 +468,7 @@ + + + +- ++ + + + +@@ -483,7 +483,7 @@ + + + +- ++ + + + +@@ -510,13 +510,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -530,7 +530,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2551.exp b/pengine/test10/bug-lf-2551.exp +index 5c2fd65..7790199 100644 +--- a/pengine/test10/bug-lf-2551.exp ++++ b/pengine/test10/bug-lf-2551.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -82,7 +82,7 @@ + + + +- ++ + + + +@@ -142,7 +142,7 @@ + + + +- ++ + + + +@@ -160,7 +160,7 @@ + + + +- ++ + + + +@@ -178,7 +178,7 @@ + + + +- ++ + + + +@@ -196,7 +196,7 @@ + + + +- ++ + + + +@@ -214,7 +214,7 @@ + + + +- ++ + + + +@@ -229,7 +229,7 @@ + + + +- ++ + + + +@@ -304,7 +304,7 @@ + + + +- ++ + + + +@@ -316,7 +316,7 @@ + + + +- ++ + + + +@@ -328,7 +328,7 @@ + + + +- ++ + + + +@@ -340,7 +340,7 @@ + + + +- ++ + + + +@@ -352,7 +352,7 @@ + + + +- ++ + + + +@@ -364,7 +364,7 @@ + + + +- ++ + + + +@@ -376,7 +376,7 @@ + + + +- ++ + + + +@@ -388,7 +388,7 @@ + + + +- ++ + + + +@@ -400,7 +400,7 @@ + + + +- ++ + + + +@@ -412,7 +412,7 @@ + + + +- ++ + + + +@@ -424,7 +424,7 @@ + + + +- ++ + + + +@@ -436,7 +436,7 @@ + + + +- ++ + + + +@@ -448,7 +448,7 @@ + + + +- ++ + + + +@@ -460,7 +460,7 @@ + + + +- ++ + + + +@@ -472,7 +472,7 @@ + + + +- ++ + + + +@@ -484,7 +484,7 @@ + + + +- ++ + + + +@@ -496,13 +496,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -516,7 +516,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/bug-lf-2606.exp b/pengine/test10/bug-lf-2606.exp +index ae490e6..270131a 100644 +--- a/pengine/test10/bug-lf-2606.exp ++++ b/pengine/test10/bug-lf-2606.exp +@@ -7,7 +7,7 @@ + + + +- ++ + + + +@@ -48,7 +48,7 @@ + + + +- ++ + + + +@@ -66,7 +66,7 @@ + + + +- ++ + + + +@@ -81,7 +81,7 @@ + + + +- ++ + + + +@@ -134,13 +134,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -154,7 +154,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/interleave-pseudo-stop.exp b/pengine/test10/interleave-pseudo-stop.exp +index ae5c9a6..a5e8880 100644 +--- a/pengine/test10/interleave-pseudo-stop.exp ++++ b/pengine/test10/interleave-pseudo-stop.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -93,7 +93,7 @@ + + + +- ++ + + + +@@ -114,7 +114,7 @@ + + + +- ++ + + + +@@ -203,7 +203,7 @@ + + + +- ++ + + + +@@ -259,7 +259,7 @@ + + + +- ++ + + + +@@ -274,7 +274,7 @@ + + + +- ++ + + + +@@ -357,7 +357,7 @@ + + + +- ++ + + + +@@ -413,7 +413,7 @@ + + + +- ++ + + + +@@ -428,7 +428,7 @@ + + + +- ++ + + + +@@ -511,13 +511,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -531,7 +531,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/master-7.exp b/pengine/test10/master-7.exp +index b9b246c..4fba172 100644 +--- a/pengine/test10/master-7.exp ++++ b/pengine/test10/master-7.exp +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -349,7 +349,7 @@ + + + +- ++ + + + +@@ -367,7 +367,7 @@ + + + +- ++ + + + +@@ -430,7 +430,7 @@ + + + +- ++ + + + +@@ -451,7 +451,7 @@ + + + +- ++ + + + +@@ -466,7 +466,7 @@ + + + +- ++ + + + +@@ -484,7 +484,7 @@ + + + +- ++ + + + +@@ -621,13 +621,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -641,7 +641,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/master-8.exp b/pengine/test10/master-8.exp +index e6c73a0..c0cd853 100644 +--- a/pengine/test10/master-8.exp ++++ b/pengine/test10/master-8.exp +@@ -39,7 +39,7 @@ + + + +- ++ + + + +@@ -349,7 +349,7 @@ + + + +- ++ + + + +@@ -367,7 +367,7 @@ + + + +- ++ + + + +@@ -430,7 +430,7 @@ + + + +- ++ + + + +@@ -489,7 +489,7 @@ + + + +- ++ + + + +@@ -504,7 +504,7 @@ + + + +- ++ + + + +@@ -647,7 +647,7 @@ + + + +- ++ + + + +@@ -683,7 +683,7 @@ + + + +- ++ + + + +@@ -697,7 +697,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/migrate-fencing.exp b/pengine/test10/migrate-fencing.exp +index 0fba419..02b70c8 100644 +--- a/pengine/test10/migrate-fencing.exp ++++ b/pengine/test10/migrate-fencing.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -159,7 +159,7 @@ + + + +- ++ + + + +@@ -212,7 +212,7 @@ + + + +- ++ + + + +@@ -262,7 +262,7 @@ + + + +- ++ + + + +@@ -303,7 +303,7 @@ + + + +- ++ + + + +@@ -347,7 +347,7 @@ + + + +- ++ + + + +@@ -429,7 +429,7 @@ + + + +- ++ + + + +@@ -456,7 +456,7 @@ + + + +- ++ + + + +@@ -474,7 +474,7 @@ + + + +- ++ + + + +@@ -489,7 +489,7 @@ + + + +- ++ + + + +@@ -620,13 +620,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -640,7 +640,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-11.exp b/pengine/test10/rec-node-11.exp +index 09aaddf..173df21 100644 +--- a/pengine/test10/rec-node-11.exp ++++ b/pengine/test10/rec-node-11.exp +@@ -50,7 +50,7 @@ + + + +- ++ + + + +@@ -126,7 +126,7 @@ + + + +- ++ + + + +@@ -178,7 +178,7 @@ + + + +- ++ + + + +@@ -225,7 +225,7 @@ + + + +- ++ + + + +@@ -239,7 +239,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-12.exp b/pengine/test10/rec-node-12.exp +index 07c0027..9732907 100644 +--- a/pengine/test10/rec-node-12.exp ++++ b/pengine/test10/rec-node-12.exp +@@ -514,7 +514,7 @@ + + + +- ++ + + + +@@ -528,7 +528,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-13.exp b/pengine/test10/rec-node-13.exp +index 63abd08..ae23ea5 100644 +--- a/pengine/test10/rec-node-13.exp ++++ b/pengine/test10/rec-node-13.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -37,13 +37,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -57,7 +57,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-14.exp b/pengine/test10/rec-node-14.exp +index 2ca969a..e473479 100644 +--- a/pengine/test10/rec-node-14.exp ++++ b/pengine/test10/rec-node-14.exp +@@ -1,31 +1,31 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -39,7 +39,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-15.exp b/pengine/test10/rec-node-15.exp +index a48d8c5..3c79d65 100644 +--- a/pengine/test10/rec-node-15.exp ++++ b/pengine/test10/rec-node-15.exp +@@ -445,7 +445,7 @@ + + + +- ++ + + + +@@ -459,7 +459,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-2.exp b/pengine/test10/rec-node-2.exp +index 2785c57..901eded 100644 +--- a/pengine/test10/rec-node-2.exp ++++ b/pengine/test10/rec-node-2.exp +@@ -246,7 +246,7 @@ + + + +- ++ + + + +@@ -260,7 +260,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-4.exp b/pengine/test10/rec-node-4.exp +index bdaaa38..1e98f4d 100644 +--- a/pengine/test10/rec-node-4.exp ++++ b/pengine/test10/rec-node-4.exp +@@ -54,7 +54,7 @@ + + + +- ++ + + + +@@ -97,7 +97,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +@@ -126,7 +126,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-6.exp b/pengine/test10/rec-node-6.exp +index bdaaa38..1e98f4d 100644 +--- a/pengine/test10/rec-node-6.exp ++++ b/pengine/test10/rec-node-6.exp +@@ -54,7 +54,7 @@ + + + +- ++ + + + +@@ -97,7 +97,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +@@ -126,7 +126,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-node-7.exp b/pengine/test10/rec-node-7.exp +index bdaaa38..1e98f4d 100644 +--- a/pengine/test10/rec-node-7.exp ++++ b/pengine/test10/rec-node-7.exp +@@ -54,7 +54,7 @@ + + + +- ++ + + + +@@ -97,7 +97,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +@@ -126,7 +126,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/rec-rsc-5.exp b/pengine/test10/rec-rsc-5.exp +index 897d5f6..69a85d1 100644 +--- a/pengine/test10/rec-rsc-5.exp ++++ b/pengine/test10/rec-rsc-5.exp +@@ -54,7 +54,7 @@ + + + +- ++ + + + +@@ -97,7 +97,7 @@ + + + +- ++ + + + +@@ -112,7 +112,7 @@ + + + +- ++ + + + +@@ -126,7 +126,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/remote-fence-unclean.exp b/pengine/test10/remote-fence-unclean.exp +new file mode 100644 +index 0000000..9b1e327 +--- /dev/null ++++ b/pengine/test10/remote-fence-unclean.exp +@@ -0,0 +1,201 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/stonith-0.exp b/pengine/test10/stonith-0.exp +index f4aed36..d664a12 100644 +--- a/pengine/test10/stonith-0.exp ++++ b/pengine/test10/stonith-0.exp +@@ -34,10 +34,10 @@ + + + +- ++ + + +- ++ + + + +@@ -123,7 +123,7 @@ + + + +- ++ + + + +@@ -141,7 +141,7 @@ + + + +- ++ + + + +@@ -203,7 +203,7 @@ + + + +- ++ + + + +@@ -256,7 +256,7 @@ + + + +- ++ + + + +@@ -271,7 +271,7 @@ + + + +- ++ + + + +@@ -312,7 +312,7 @@ + + + +- ++ + + + +@@ -353,7 +353,7 @@ + + + +- ++ + + + +@@ -368,7 +368,7 @@ + + + +- ++ + + + +@@ -376,13 +376,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -394,7 +394,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stonith-1.exp b/pengine/test10/stonith-1.exp +index 12a2839..b9798e3 100644 +--- a/pengine/test10/stonith-1.exp ++++ b/pengine/test10/stonith-1.exp +@@ -108,7 +108,7 @@ + + + +- ++ + + + +@@ -158,7 +158,7 @@ + + + +- ++ + + + +@@ -211,7 +211,7 @@ + + + +- ++ + + + +@@ -238,7 +238,7 @@ + + + +- ++ + + + +@@ -370,7 +370,7 @@ + + + +- ++ + + + +@@ -475,7 +475,7 @@ + + + +- ++ + + + +@@ -505,7 +505,7 @@ + + + +- ++ + + + +@@ -553,7 +553,7 @@ + + + +- ++ + + + +@@ -567,7 +567,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stonith-2.exp b/pengine/test10/stonith-2.exp +index dfab333..84a7fb8 100644 +--- a/pengine/test10/stonith-2.exp ++++ b/pengine/test10/stonith-2.exp +@@ -27,7 +27,7 @@ + + + +- ++ + + + +@@ -41,7 +41,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stonith-3.exp b/pengine/test10/stonith-3.exp +index bb15555..49e6e81 100644 +--- a/pengine/test10/stonith-3.exp ++++ b/pengine/test10/stonith-3.exp +@@ -117,7 +117,7 @@ + + + +- ++ + + + +@@ -131,7 +131,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stonith-4.exp b/pengine/test10/stonith-4.exp +index 8e5ee2c..2cfffc0 100644 +--- a/pengine/test10/stonith-4.exp ++++ b/pengine/test10/stonith-4.exp +@@ -17,43 +17,43 @@ + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -67,7 +67,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stop-failure-no-quorum.exp b/pengine/test10/stop-failure-no-quorum.exp +index b2260c7..895d43c 100644 +--- a/pengine/test10/stop-failure-no-quorum.exp ++++ b/pengine/test10/stop-failure-no-quorum.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -37,13 +37,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -57,7 +57,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/stop-failure-with-fencing.exp b/pengine/test10/stop-failure-with-fencing.exp +index 1c22e38..335ac37 100644 +--- a/pengine/test10/stop-failure-with-fencing.exp ++++ b/pengine/test10/stop-failure-with-fencing.exp +@@ -10,7 +10,7 @@ + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -52,7 +52,7 @@ + + + +- ++ + + + +@@ -66,7 +66,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealth1.exp b/pengine/test10/systemhealth1.exp +index f8c698e..b090f6b 100644 +--- a/pengine/test10/systemhealth1.exp ++++ b/pengine/test10/systemhealth1.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,13 +9,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealth2.exp b/pengine/test10/systemhealth2.exp +index 9232b32..68b5078 100644 +--- a/pengine/test10/systemhealth2.exp ++++ b/pengine/test10/systemhealth2.exp +@@ -102,7 +102,7 @@ + + + +- ++ + + + +@@ -116,7 +116,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealth3.exp b/pengine/test10/systemhealth3.exp +index 9232b32..68b5078 100644 +--- a/pengine/test10/systemhealth3.exp ++++ b/pengine/test10/systemhealth3.exp +@@ -102,7 +102,7 @@ + + + +- ++ + + + +@@ -116,7 +116,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthm1.exp b/pengine/test10/systemhealthm1.exp +index f8c698e..b090f6b 100644 +--- a/pengine/test10/systemhealthm1.exp ++++ b/pengine/test10/systemhealthm1.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,13 +9,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthm2.exp b/pengine/test10/systemhealthm2.exp +index 9232b32..68b5078 100644 +--- a/pengine/test10/systemhealthm2.exp ++++ b/pengine/test10/systemhealthm2.exp +@@ -102,7 +102,7 @@ + + + +- ++ + + + +@@ -116,7 +116,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthm3.exp b/pengine/test10/systemhealthm3.exp +index 15b37be..7246b5b 100644 +--- a/pengine/test10/systemhealthm3.exp ++++ b/pengine/test10/systemhealthm3.exp +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthn1.exp b/pengine/test10/systemhealthn1.exp +index f8c698e..b090f6b 100644 +--- a/pengine/test10/systemhealthn1.exp ++++ b/pengine/test10/systemhealthn1.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,13 +9,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthn2.exp b/pengine/test10/systemhealthn2.exp +index 9232b32..68b5078 100644 +--- a/pengine/test10/systemhealthn2.exp ++++ b/pengine/test10/systemhealthn2.exp +@@ -102,7 +102,7 @@ + + + +- ++ + + + +@@ -116,7 +116,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthn3.exp b/pengine/test10/systemhealthn3.exp +index 9232b32..68b5078 100644 +--- a/pengine/test10/systemhealthn3.exp ++++ b/pengine/test10/systemhealthn3.exp +@@ -102,7 +102,7 @@ + + + +- ++ + + + +@@ -116,7 +116,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealtho1.exp b/pengine/test10/systemhealtho1.exp +index f8c698e..b090f6b 100644 +--- a/pengine/test10/systemhealtho1.exp ++++ b/pengine/test10/systemhealtho1.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,13 +9,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealtho2.exp b/pengine/test10/systemhealtho2.exp +index 15b37be..7246b5b 100644 +--- a/pengine/test10/systemhealtho2.exp ++++ b/pengine/test10/systemhealtho2.exp +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealtho3.exp b/pengine/test10/systemhealtho3.exp +index 15b37be..7246b5b 100644 +--- a/pengine/test10/systemhealtho3.exp ++++ b/pengine/test10/systemhealtho3.exp +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthp1.exp b/pengine/test10/systemhealthp1.exp +index f8c698e..b090f6b 100644 +--- a/pengine/test10/systemhealthp1.exp ++++ b/pengine/test10/systemhealthp1.exp +@@ -1,7 +1,7 @@ + + + +- ++ + + + +@@ -9,13 +9,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthp2.exp b/pengine/test10/systemhealthp2.exp +index f333166..e3ba25d 100644 +--- a/pengine/test10/systemhealthp2.exp ++++ b/pengine/test10/systemhealthp2.exp +@@ -86,7 +86,7 @@ + + + +- ++ + + + +@@ -100,7 +100,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/systemhealthp3.exp b/pengine/test10/systemhealthp3.exp +index 15b37be..7246b5b 100644 +--- a/pengine/test10/systemhealthp3.exp ++++ b/pengine/test10/systemhealthp3.exp +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -42,7 +42,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-21.exp b/pengine/test10/ticket-clone-21.exp +index ee3a383..00f136d 100644 +--- a/pengine/test10/ticket-clone-21.exp ++++ b/pengine/test10/ticket-clone-21.exp +@@ -7,7 +7,7 @@ + + + +- ++ + + + +@@ -22,7 +22,7 @@ + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -67,28 +67,28 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -102,7 +102,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-clone-9.exp b/pengine/test10/ticket-clone-9.exp +index ee3a383..00f136d 100644 +--- a/pengine/test10/ticket-clone-9.exp ++++ b/pengine/test10/ticket-clone-9.exp +@@ -7,7 +7,7 @@ + + + +- ++ + + + +@@ -22,7 +22,7 @@ + + + +- ++ + + + +@@ -37,7 +37,7 @@ + + + +- ++ + + + +@@ -67,28 +67,28 @@ + + + +- ++ + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + + + +- ++ + + + +@@ -102,7 +102,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-21.exp b/pengine/test10/ticket-group-21.exp +index 53e30f0..d85b683 100644 +--- a/pengine/test10/ticket-group-21.exp ++++ b/pengine/test10/ticket-group-21.exp +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -58,13 +58,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-group-9.exp b/pengine/test10/ticket-group-9.exp +index 53e30f0..d85b683 100644 +--- a/pengine/test10/ticket-group-9.exp ++++ b/pengine/test10/ticket-group-9.exp +@@ -25,7 +25,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -58,13 +58,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -78,7 +78,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-21.exp b/pengine/test10/ticket-master-21.exp +index 47f4257..1ed731c 100644 +--- a/pengine/test10/ticket-master-21.exp ++++ b/pengine/test10/ticket-master-21.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -41,7 +41,7 @@ + + + +- ++ + + + +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -109,13 +109,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -129,7 +129,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-master-9.exp b/pengine/test10/ticket-master-9.exp +index 47f4257..1ed731c 100644 +--- a/pengine/test10/ticket-master-9.exp ++++ b/pengine/test10/ticket-master-9.exp +@@ -23,7 +23,7 @@ + + + +- ++ + + + +@@ -41,7 +41,7 @@ + + + +- ++ + + + +@@ -56,7 +56,7 @@ + + + +- ++ + + + +@@ -109,13 +109,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -129,7 +129,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-21.exp b/pengine/test10/ticket-primitive-21.exp +index 8e8b128..c88cf38 100644 +--- a/pengine/test10/ticket-primitive-21.exp ++++ b/pengine/test10/ticket-primitive-21.exp +@@ -7,13 +7,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/ticket-primitive-9.exp b/pengine/test10/ticket-primitive-9.exp +index 8e8b128..c88cf38 100644 +--- a/pengine/test10/ticket-primitive-9.exp ++++ b/pengine/test10/ticket-primitive-9.exp +@@ -7,13 +7,13 @@ + + + +- ++ + + + + + +- ++ + + + +@@ -27,7 +27,7 @@ + + + +- ++ + + + diff --git a/SOURCES/bz1078078-pcmk-pe_potential_memory_leak.patch b/SOURCES/bz1078078-pcmk-pe_potential_memory_leak.patch new file mode 100644 index 0000000..3848308 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_potential_memory_leak.patch @@ -0,0 +1,21 @@ +commit 566a6901e5badb712809473527f42678267460a8 +Author: Andrew Beekhof +Date: Tue Apr 1 16:00:40 2014 +1100 + + Fix: PE: Potential memory leak + + (cherry picked from commit b1474d8feb1d975f7a87b4008aabc6ff338d2349) + +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 1e827e0..410acfc 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -358,7 +358,7 @@ custom_action(resource_t * rsc, char *key, const char *task, + GListPtr possible_matches = NULL; + + CRM_CHECK(key != NULL, return NULL); +- CRM_CHECK(task != NULL, return NULL); ++ CRM_CHECK(task != NULL, free(key); return NULL); + + if (save_action && rsc != NULL) { + possible_matches = find_actions(rsc->actions, key, on_node); diff --git a/SOURCES/bz1078078-pcmk-pe_repair_regression_tests.patch b/SOURCES/bz1078078-pcmk-pe_repair_regression_tests.patch new file mode 100644 index 0000000..fae99f7 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_repair_regression_tests.patch @@ -0,0 +1,1309 @@ +commit fe3e8bf6ece8eeb90c8c1c738b7961e6413762fc +Author: Andrew Beekhof +Date: Fri Apr 11 08:16:33 2014 +1000 + + PE: Repair regression tests + +diff --git a/pengine/test10/unfence-definition.exp b/pengine/test10/unfence-definition.exp +index f023a0b..392c286 100644 +--- a/pengine/test10/unfence-definition.exp ++++ b/pengine/test10/unfence-definition.exp +@@ -1,14 +1,14 @@ +- ++ + + + + +- ++ + + + + +- ++ + + + +@@ -28,7 +28,7 @@ + + + +- ++ + + + +@@ -41,7 +41,7 @@ + + + +- ++ + + + +@@ -54,7 +54,7 @@ + + + +- ++ + + + +@@ -67,7 +67,7 @@ + + + +- ++ + + + +@@ -89,7 +89,7 @@ + + + +- ++ + + + +@@ -108,7 +108,7 @@ + + + +- ++ + + + +@@ -127,7 +127,7 @@ + + + +- ++ + + + +@@ -135,7 +135,7 @@ + + + +- ++ + + + +@@ -150,7 +150,7 @@ + + + +- ++ + + + +@@ -162,7 +162,7 @@ + + + +- ++ + + + +@@ -180,12 +180,12 @@ + + + +- ++ + + + + +- ++ + + + +@@ -199,7 +199,7 @@ + + + +- ++ + + + +@@ -224,7 +224,7 @@ + + + +- ++ + + + +@@ -240,7 +240,7 @@ + + + +- ++ + + + +@@ -262,7 +262,7 @@ + + + +- ++ + + + +@@ -271,7 +271,7 @@ + + + +- ++ + + + +@@ -296,7 +296,7 @@ + + + +- ++ + + + +@@ -304,7 +304,7 @@ + + + +- ++ + + + +@@ -319,7 +319,7 @@ + + + +- ++ + + + +@@ -327,7 +327,7 @@ + + + +- ++ + + + +@@ -348,12 +348,12 @@ + + + +- ++ + + + + +- ++ + + + +@@ -369,7 +369,7 @@ + + + +- ++ + + + +@@ -377,7 +377,7 @@ + + + +- ++ + + + +@@ -389,7 +389,7 @@ + + + +- ++ + + + +@@ -404,7 +404,7 @@ + + + +- ++ + + + +@@ -416,7 +416,7 @@ + + + +- ++ + + + +@@ -424,7 +424,7 @@ + + + +- ++ + + + +@@ -442,7 +442,7 @@ + + + +- ++ + + + +@@ -462,37 +462,37 @@ + + + +- +- +- ++ ++ ++ + + + +- +- +- +- +- +- +- ++ + + +- ++ + + + + + +- +- +- ++ ++ ++ + + + +- ++ + + +- ++ ++ ++ ++ ++ ++ ++ + + + +diff --git a/pengine/test10/unfence-definition.scores b/pengine/test10/unfence-definition.scores +new file mode 100644 +index 0000000..a4b5fb1 +--- /dev/null ++++ b/pengine/test10/unfence-definition.scores +@@ -0,0 +1,77 @@ ++Allocation scores: ++clone_color: clvmd-clone allocation score on virt-1: 0 ++clone_color: clvmd-clone allocation score on virt-2: 0 ++clone_color: clvmd-clone allocation score on virt-3: 0 ++clone_color: clvmd-clone allocation score on virt-4: 0 ++clone_color: clvmd:0 allocation score on virt-1: 1 ++clone_color: clvmd:0 allocation score on virt-2: 0 ++clone_color: clvmd:0 allocation score on virt-3: 0 ++clone_color: clvmd:0 allocation score on virt-4: 0 ++clone_color: clvmd:1 allocation score on virt-1: 0 ++clone_color: clvmd:1 allocation score on virt-2: 0 ++clone_color: clvmd:1 allocation score on virt-3: 0 ++clone_color: clvmd:1 allocation score on virt-4: 0 ++clone_color: clvmd:2 allocation score on virt-1: 0 ++clone_color: clvmd:2 allocation score on virt-2: 0 ++clone_color: clvmd:2 allocation score on virt-3: 0 ++clone_color: clvmd:2 allocation score on virt-4: 0 ++clone_color: clvmd:3 allocation score on virt-1: 0 ++clone_color: clvmd:3 allocation score on virt-2: 0 ++clone_color: clvmd:3 allocation score on virt-3: 0 ++clone_color: clvmd:3 allocation score on virt-4: 0 ++clone_color: dlm-clone allocation score on virt-1: 0 ++clone_color: dlm-clone allocation score on virt-2: 0 ++clone_color: dlm-clone allocation score on virt-3: 0 ++clone_color: dlm-clone allocation score on virt-4: 0 ++clone_color: dlm:0 allocation score on virt-1: 1 ++clone_color: dlm:0 allocation score on virt-2: 0 ++clone_color: dlm:0 allocation score on virt-3: 0 ++clone_color: dlm:0 allocation score on virt-4: 0 ++clone_color: dlm:1 allocation score on virt-1: 0 ++clone_color: dlm:1 allocation score on virt-2: 1 ++clone_color: dlm:1 allocation score on virt-3: 0 ++clone_color: dlm:1 allocation score on virt-4: 0 ++clone_color: dlm:2 allocation score on virt-1: 0 ++clone_color: dlm:2 allocation score on virt-2: 0 ++clone_color: dlm:2 allocation score on virt-3: 0 ++clone_color: dlm:2 allocation score on virt-4: 0 ++clone_color: dlm:3 allocation score on virt-1: 0 ++clone_color: dlm:3 allocation score on virt-2: 0 ++clone_color: dlm:3 allocation score on virt-3: 0 ++clone_color: dlm:3 allocation score on virt-4: 0 ++native_color: clvmd:0 allocation score on virt-1: 1 ++native_color: clvmd:0 allocation score on virt-2: -INFINITY ++native_color: clvmd:0 allocation score on virt-3: -INFINITY ++native_color: clvmd:0 allocation score on virt-4: -INFINITY ++native_color: clvmd:1 allocation score on virt-1: -INFINITY ++native_color: clvmd:1 allocation score on virt-2: 0 ++native_color: clvmd:1 allocation score on virt-3: -INFINITY ++native_color: clvmd:1 allocation score on virt-4: -INFINITY ++native_color: clvmd:2 allocation score on virt-1: -INFINITY ++native_color: clvmd:2 allocation score on virt-2: -INFINITY ++native_color: clvmd:2 allocation score on virt-3: 0 ++native_color: clvmd:2 allocation score on virt-4: -INFINITY ++native_color: clvmd:3 allocation score on virt-1: -INFINITY ++native_color: clvmd:3 allocation score on virt-2: -INFINITY ++native_color: clvmd:3 allocation score on virt-3: -INFINITY ++native_color: clvmd:3 allocation score on virt-4: -INFINITY ++native_color: dlm:0 allocation score on virt-1: 1 ++native_color: dlm:0 allocation score on virt-2: -INFINITY ++native_color: dlm:0 allocation score on virt-3: 0 ++native_color: dlm:0 allocation score on virt-4: -INFINITY ++native_color: dlm:1 allocation score on virt-1: 0 ++native_color: dlm:1 allocation score on virt-2: 1 ++native_color: dlm:1 allocation score on virt-3: 0 ++native_color: dlm:1 allocation score on virt-4: -INFINITY ++native_color: dlm:2 allocation score on virt-1: -INFINITY ++native_color: dlm:2 allocation score on virt-2: -INFINITY ++native_color: dlm:2 allocation score on virt-3: 0 ++native_color: dlm:2 allocation score on virt-4: -INFINITY ++native_color: dlm:3 allocation score on virt-1: -INFINITY ++native_color: dlm:3 allocation score on virt-2: -INFINITY ++native_color: dlm:3 allocation score on virt-3: -INFINITY ++native_color: dlm:3 allocation score on virt-4: -INFINITY ++native_color: fencing allocation score on virt-1: 0 ++native_color: fencing allocation score on virt-2: 0 ++native_color: fencing allocation score on virt-3: 0 ++native_color: fencing allocation score on virt-4: 0 +diff --git a/pengine/test10/unfence-definition.summary b/pengine/test10/unfence-definition.summary +index 3e4b66b..3d65029 100644 +--- a/pengine/test10/unfence-definition.summary ++++ b/pengine/test10/unfence-definition.summary +@@ -35,8 +35,8 @@ Executing cluster transition: + * Resource action: dlm stop on virt-1 + * Pseudo action: dlm-clone_stopped_0 + * Fencing virt-3 (on) +- * Pseudo action: all_stopped + * Fencing virt-1 (on) ++ * Pseudo action: all_stopped + * Resource action: fencing monitor on virt-3 + * Pseudo action: dlm-clone_start_0 + * Resource action: fencing start on virt-1 +diff --git a/pengine/test10/unfence-definition.xml b/pengine/test10/unfence-definition.xml +new file mode 100644 +index 0000000..5d77931 +--- /dev/null ++++ b/pengine/test10/unfence-definition.xml +@@ -0,0 +1,73 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/unfence-parameters.exp b/pengine/test10/unfence-parameters.exp +index 8ff3016..31b0371 100644 +--- a/pengine/test10/unfence-parameters.exp ++++ b/pengine/test10/unfence-parameters.exp +@@ -1,9 +1,9 @@ +- ++ + + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -29,12 +29,12 @@ + + + +- ++ + + + + +- ++ + + + +@@ -51,7 +51,7 @@ + + + +- ++ + + + +@@ -73,7 +73,7 @@ + + + +- ++ + + + +@@ -95,7 +95,7 @@ + + + +- ++ + + + +@@ -120,7 +120,7 @@ + + + +- ++ + + + +@@ -136,7 +136,7 @@ + + + +- ++ + + + +@@ -158,7 +158,7 @@ + + + +- ++ + + + +@@ -166,7 +166,7 @@ + + + +- ++ + + + +@@ -184,7 +184,7 @@ + + + +- ++ + + + +@@ -196,7 +196,7 @@ + + + +- ++ + + + +@@ -217,12 +217,12 @@ + + + +- ++ + + + + +- ++ + + + +@@ -239,7 +239,7 @@ + + + +- ++ + + + +@@ -264,7 +264,7 @@ + + + +- ++ + + + +@@ -280,7 +280,7 @@ + + + +- ++ + + + +@@ -305,7 +305,7 @@ + + + +- ++ + + + +@@ -314,7 +314,7 @@ + + + +- ++ + + + +@@ -339,7 +339,7 @@ + + + +- ++ + + + +@@ -347,7 +347,7 @@ + + + +- ++ + + + +@@ -362,7 +362,7 @@ + + + +- ++ + + + +@@ -370,7 +370,7 @@ + + + +- ++ + + + +@@ -391,12 +391,12 @@ + + + +- ++ + + + + +- ++ + + + +@@ -415,7 +415,7 @@ + + + +- ++ + + + +@@ -423,7 +423,7 @@ + + + +- ++ + + + +@@ -435,7 +435,7 @@ + + + +- ++ + + + +@@ -450,7 +450,7 @@ + + + +- ++ + + + +@@ -462,7 +462,7 @@ + + + +- ++ + + + +@@ -470,7 +470,7 @@ + + + +- ++ + + + +@@ -488,7 +488,7 @@ + + + +- ++ + + + +@@ -509,7 +509,7 @@ + + + +- ++ + + + +@@ -526,8 +526,23 @@ + + + +- +- ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ + + + +@@ -548,19 +563,4 @@ + + + +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- + +diff --git a/pengine/test10/unfence-parameters.scores b/pengine/test10/unfence-parameters.scores +new file mode 100644 +index 0000000..a4b5fb1 +--- /dev/null ++++ b/pengine/test10/unfence-parameters.scores +@@ -0,0 +1,77 @@ ++Allocation scores: ++clone_color: clvmd-clone allocation score on virt-1: 0 ++clone_color: clvmd-clone allocation score on virt-2: 0 ++clone_color: clvmd-clone allocation score on virt-3: 0 ++clone_color: clvmd-clone allocation score on virt-4: 0 ++clone_color: clvmd:0 allocation score on virt-1: 1 ++clone_color: clvmd:0 allocation score on virt-2: 0 ++clone_color: clvmd:0 allocation score on virt-3: 0 ++clone_color: clvmd:0 allocation score on virt-4: 0 ++clone_color: clvmd:1 allocation score on virt-1: 0 ++clone_color: clvmd:1 allocation score on virt-2: 0 ++clone_color: clvmd:1 allocation score on virt-3: 0 ++clone_color: clvmd:1 allocation score on virt-4: 0 ++clone_color: clvmd:2 allocation score on virt-1: 0 ++clone_color: clvmd:2 allocation score on virt-2: 0 ++clone_color: clvmd:2 allocation score on virt-3: 0 ++clone_color: clvmd:2 allocation score on virt-4: 0 ++clone_color: clvmd:3 allocation score on virt-1: 0 ++clone_color: clvmd:3 allocation score on virt-2: 0 ++clone_color: clvmd:3 allocation score on virt-3: 0 ++clone_color: clvmd:3 allocation score on virt-4: 0 ++clone_color: dlm-clone allocation score on virt-1: 0 ++clone_color: dlm-clone allocation score on virt-2: 0 ++clone_color: dlm-clone allocation score on virt-3: 0 ++clone_color: dlm-clone allocation score on virt-4: 0 ++clone_color: dlm:0 allocation score on virt-1: 1 ++clone_color: dlm:0 allocation score on virt-2: 0 ++clone_color: dlm:0 allocation score on virt-3: 0 ++clone_color: dlm:0 allocation score on virt-4: 0 ++clone_color: dlm:1 allocation score on virt-1: 0 ++clone_color: dlm:1 allocation score on virt-2: 1 ++clone_color: dlm:1 allocation score on virt-3: 0 ++clone_color: dlm:1 allocation score on virt-4: 0 ++clone_color: dlm:2 allocation score on virt-1: 0 ++clone_color: dlm:2 allocation score on virt-2: 0 ++clone_color: dlm:2 allocation score on virt-3: 0 ++clone_color: dlm:2 allocation score on virt-4: 0 ++clone_color: dlm:3 allocation score on virt-1: 0 ++clone_color: dlm:3 allocation score on virt-2: 0 ++clone_color: dlm:3 allocation score on virt-3: 0 ++clone_color: dlm:3 allocation score on virt-4: 0 ++native_color: clvmd:0 allocation score on virt-1: 1 ++native_color: clvmd:0 allocation score on virt-2: -INFINITY ++native_color: clvmd:0 allocation score on virt-3: -INFINITY ++native_color: clvmd:0 allocation score on virt-4: -INFINITY ++native_color: clvmd:1 allocation score on virt-1: -INFINITY ++native_color: clvmd:1 allocation score on virt-2: 0 ++native_color: clvmd:1 allocation score on virt-3: -INFINITY ++native_color: clvmd:1 allocation score on virt-4: -INFINITY ++native_color: clvmd:2 allocation score on virt-1: -INFINITY ++native_color: clvmd:2 allocation score on virt-2: -INFINITY ++native_color: clvmd:2 allocation score on virt-3: 0 ++native_color: clvmd:2 allocation score on virt-4: -INFINITY ++native_color: clvmd:3 allocation score on virt-1: -INFINITY ++native_color: clvmd:3 allocation score on virt-2: -INFINITY ++native_color: clvmd:3 allocation score on virt-3: -INFINITY ++native_color: clvmd:3 allocation score on virt-4: -INFINITY ++native_color: dlm:0 allocation score on virt-1: 1 ++native_color: dlm:0 allocation score on virt-2: -INFINITY ++native_color: dlm:0 allocation score on virt-3: 0 ++native_color: dlm:0 allocation score on virt-4: -INFINITY ++native_color: dlm:1 allocation score on virt-1: 0 ++native_color: dlm:1 allocation score on virt-2: 1 ++native_color: dlm:1 allocation score on virt-3: 0 ++native_color: dlm:1 allocation score on virt-4: -INFINITY ++native_color: dlm:2 allocation score on virt-1: -INFINITY ++native_color: dlm:2 allocation score on virt-2: -INFINITY ++native_color: dlm:2 allocation score on virt-3: 0 ++native_color: dlm:2 allocation score on virt-4: -INFINITY ++native_color: dlm:3 allocation score on virt-1: -INFINITY ++native_color: dlm:3 allocation score on virt-2: -INFINITY ++native_color: dlm:3 allocation score on virt-3: -INFINITY ++native_color: dlm:3 allocation score on virt-4: -INFINITY ++native_color: fencing allocation score on virt-1: 0 ++native_color: fencing allocation score on virt-2: 0 ++native_color: fencing allocation score on virt-3: 0 ++native_color: fencing allocation score on virt-4: 0 +diff --git a/pengine/test10/unfence-parameters.summary b/pengine/test10/unfence-parameters.summary +index 5276cba..2bd499c 100644 +--- a/pengine/test10/unfence-parameters.summary ++++ b/pengine/test10/unfence-parameters.summary +@@ -37,8 +37,8 @@ Executing cluster transition: + * Pseudo action: dlm-clone_stopped_0 + * Fencing virt-3 (on) + * Fencing virt-2 (on) +- * Pseudo action: all_stopped + * Fencing virt-1 (on) ++ * Pseudo action: all_stopped + * Resource action: fencing monitor on virt-3 + * Resource action: fencing start on virt-1 + * Pseudo action: dlm-clone_start_0 +diff --git a/pengine/test10/unfence-parameters.xml b/pengine/test10/unfence-parameters.xml +new file mode 100644 +index 0000000..f660100 +--- /dev/null ++++ b/pengine/test10/unfence-parameters.xml +@@ -0,0 +1,73 @@ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ +diff --git a/pengine/test10/unfence-startup.exp b/pengine/test10/unfence-startup.exp +index a0661e7..6319b13 100644 +--- a/pengine/test10/unfence-startup.exp ++++ b/pengine/test10/unfence-startup.exp +@@ -1,9 +1,9 @@ +- ++ + + + + +- ++ + + + +@@ -16,7 +16,7 @@ + + + +- ++ + + + +@@ -35,7 +35,7 @@ + + + +- ++ + + + +@@ -43,7 +43,7 @@ + + + +- ++ + + + +@@ -58,7 +58,7 @@ + + + +- ++ + + + +@@ -71,7 +71,7 @@ + + + +- ++ + + + +@@ -90,7 +90,7 @@ + + + +- ++ + + + +@@ -99,7 +99,7 @@ + + + +- ++ + + + +@@ -124,7 +124,7 @@ + + + +- ++ + + + +@@ -132,7 +132,7 @@ + + + +- ++ + + + +@@ -150,7 +150,7 @@ + + + +- ++ + + + +@@ -165,7 +165,7 @@ + + + +- ++ + + + +@@ -173,7 +173,7 @@ + + + +- ++ + + + +@@ -185,7 +185,7 @@ + + + +- ++ + + + +@@ -200,7 +200,7 @@ + + + +- ++ + + + +@@ -212,7 +212,7 @@ + + + +- ++ + + + +@@ -220,7 +220,7 @@ + + + +- ++ + + + +@@ -238,7 +238,7 @@ + + + +- ++ + + + +@@ -253,7 +253,7 @@ + + + +- ++ + + + +diff --git a/pengine/test10/unfence-startup.scores b/pengine/test10/unfence-startup.scores +new file mode 100644 +index 0000000..a4b5fb1 +--- /dev/null ++++ b/pengine/test10/unfence-startup.scores +@@ -0,0 +1,77 @@ ++Allocation scores: ++clone_color: clvmd-clone allocation score on virt-1: 0 ++clone_color: clvmd-clone allocation score on virt-2: 0 ++clone_color: clvmd-clone allocation score on virt-3: 0 ++clone_color: clvmd-clone allocation score on virt-4: 0 ++clone_color: clvmd:0 allocation score on virt-1: 1 ++clone_color: clvmd:0 allocation score on virt-2: 0 ++clone_color: clvmd:0 allocation score on virt-3: 0 ++clone_color: clvmd:0 allocation score on virt-4: 0 ++clone_color: clvmd:1 allocation score on virt-1: 0 ++clone_color: clvmd:1 allocation score on virt-2: 0 ++clone_color: clvmd:1 allocation score on virt-3: 0 ++clone_color: clvmd:1 allocation score on virt-4: 0 ++clone_color: clvmd:2 allocation score on virt-1: 0 ++clone_color: clvmd:2 allocation score on virt-2: 0 ++clone_color: clvmd:2 allocation score on virt-3: 0 ++clone_color: clvmd:2 allocation score on virt-4: 0 ++clone_color: clvmd:3 allocation score on virt-1: 0 ++clone_color: clvmd:3 allocation score on virt-2: 0 ++clone_color: clvmd:3 allocation score on virt-3: 0 ++clone_color: clvmd:3 allocation score on virt-4: 0 ++clone_color: dlm-clone allocation score on virt-1: 0 ++clone_color: dlm-clone allocation score on virt-2: 0 ++clone_color: dlm-clone allocation score on virt-3: 0 ++clone_color: dlm-clone allocation score on virt-4: 0 ++clone_color: dlm:0 allocation score on virt-1: 1 ++clone_color: dlm:0 allocation score on virt-2: 0 ++clone_color: dlm:0 allocation score on virt-3: 0 ++clone_color: dlm:0 allocation score on virt-4: 0 ++clone_color: dlm:1 allocation score on virt-1: 0 ++clone_color: dlm:1 allocation score on virt-2: 1 ++clone_color: dlm:1 allocation score on virt-3: 0 ++clone_color: dlm:1 allocation score on virt-4: 0 ++clone_color: dlm:2 allocation score on virt-1: 0 ++clone_color: dlm:2 allocation score on virt-2: 0 ++clone_color: dlm:2 allocation score on virt-3: 0 ++clone_color: dlm:2 allocation score on virt-4: 0 ++clone_color: dlm:3 allocation score on virt-1: 0 ++clone_color: dlm:3 allocation score on virt-2: 0 ++clone_color: dlm:3 allocation score on virt-3: 0 ++clone_color: dlm:3 allocation score on virt-4: 0 ++native_color: clvmd:0 allocation score on virt-1: 1 ++native_color: clvmd:0 allocation score on virt-2: -INFINITY ++native_color: clvmd:0 allocation score on virt-3: -INFINITY ++native_color: clvmd:0 allocation score on virt-4: -INFINITY ++native_color: clvmd:1 allocation score on virt-1: -INFINITY ++native_color: clvmd:1 allocation score on virt-2: 0 ++native_color: clvmd:1 allocation score on virt-3: -INFINITY ++native_color: clvmd:1 allocation score on virt-4: -INFINITY ++native_color: clvmd:2 allocation score on virt-1: -INFINITY ++native_color: clvmd:2 allocation score on virt-2: -INFINITY ++native_color: clvmd:2 allocation score on virt-3: 0 ++native_color: clvmd:2 allocation score on virt-4: -INFINITY ++native_color: clvmd:3 allocation score on virt-1: -INFINITY ++native_color: clvmd:3 allocation score on virt-2: -INFINITY ++native_color: clvmd:3 allocation score on virt-3: -INFINITY ++native_color: clvmd:3 allocation score on virt-4: -INFINITY ++native_color: dlm:0 allocation score on virt-1: 1 ++native_color: dlm:0 allocation score on virt-2: -INFINITY ++native_color: dlm:0 allocation score on virt-3: 0 ++native_color: dlm:0 allocation score on virt-4: -INFINITY ++native_color: dlm:1 allocation score on virt-1: 0 ++native_color: dlm:1 allocation score on virt-2: 1 ++native_color: dlm:1 allocation score on virt-3: 0 ++native_color: dlm:1 allocation score on virt-4: -INFINITY ++native_color: dlm:2 allocation score on virt-1: -INFINITY ++native_color: dlm:2 allocation score on virt-2: -INFINITY ++native_color: dlm:2 allocation score on virt-3: 0 ++native_color: dlm:2 allocation score on virt-4: -INFINITY ++native_color: dlm:3 allocation score on virt-1: -INFINITY ++native_color: dlm:3 allocation score on virt-2: -INFINITY ++native_color: dlm:3 allocation score on virt-3: -INFINITY ++native_color: dlm:3 allocation score on virt-4: -INFINITY ++native_color: fencing allocation score on virt-1: 0 ++native_color: fencing allocation score on virt-2: 0 ++native_color: fencing allocation score on virt-3: 0 ++native_color: fencing allocation score on virt-4: 0 diff --git a/SOURCES/bz1078078-pcmk-pe_store_non_resource_actions_in_a_hashtable_where_we_can_easily_and_cheaply_find_them_again.patch b/SOURCES/bz1078078-pcmk-pe_store_non_resource_actions_in_a_hashtable_where_we_can_easily_and_cheaply_find_them_again.patch new file mode 100644 index 0000000..c9c10a6 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-pe_store_non_resource_actions_in_a_hashtable_where_we_can_easily_and_cheaply_find_them_again.patch @@ -0,0 +1,118 @@ +commit 3544fb711731cb6ebcd48c2af808e4ec25e5eaed +Author: Andrew Beekhof +Date: Wed Apr 2 15:44:31 2014 +1100 + + Refactor: PE: Store non-resource actions in a hashtable where we can easily (and cheaply) find them again + + (cherry picked from commit a65979cbbec616f978154e95b49e8f7bcd064a14) + +diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h +index c81566a..b74bf83 100644 +--- a/include/crm/pengine/status.h ++++ b/include/crm/pengine/status.h +@@ -93,6 +93,7 @@ typedef struct pe_working_set_s { + GHashTable *config_hash; + GHashTable *domains; + GHashTable *tickets; ++ GHashTable *singletons; /* Actions for which there can be only one - ie. fence nodeX */ + + GListPtr nodes; + GListPtr resources; +diff --git a/lib/pengine/status.c b/lib/pengine/status.c +index bb9dfcb..8695dae 100644 +--- a/lib/pengine/status.c ++++ b/lib/pengine/status.c +@@ -191,6 +191,10 @@ cleanup_calculations(pe_working_set_t * data_set) + g_hash_table_destroy(data_set->config_hash); + } + ++ if (data_set->singletons != NULL) { ++ g_hash_table_destroy(data_set->singletons); ++ } ++ + if (data_set->tickets) { + g_hash_table_destroy(data_set->tickets); + } +diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c +index 410acfc..2520eeb 100644 +--- a/lib/pengine/utils.c ++++ b/lib/pengine/utils.c +@@ -362,6 +362,17 @@ custom_action(resource_t * rsc, char *key, const char *task, + + if (save_action && rsc != NULL) { + possible_matches = find_actions(rsc->actions, key, on_node); ++ } else if(save_action) { ++#if 0 ++ action = g_hash_table_lookup(data_set->singletons, key); ++#else ++ /* More expensive but takes 'node' into account */ ++ possible_matches = find_actions(data_set->actions, key, on_node); ++#endif ++ } ++ ++ if(data_set->singletons == NULL) { ++ data_set->singletons = g_hash_table_new_full(crm_str_hash, g_str_equal, NULL, NULL); + } + + if (possible_matches != NULL) { +@@ -424,6 +435,9 @@ custom_action(resource_t * rsc, char *key, const char *task, + + if (save_action) { + data_set->actions = g_list_prepend(data_set->actions, action); ++ if(rsc == NULL) { ++ g_hash_table_insert(data_set->singletons, action->uuid, action); ++ } + } + + if (rsc != NULL) { +@@ -1588,6 +1602,9 @@ order_actions(action_t * lh_action, action_t * rh_action, enum pe_ordering order + + crm_trace("Ordering Action %s before %s", lh_action->uuid, rh_action->uuid); + ++ /* Ensure we never create a dependancy on ourselves... its happened */ ++ CRM_ASSERT(lh_action != rh_action); ++ + /* Filter dups, otherwise update_action_states() has too much work to do */ + gIter = lh_action->actions_after; + for (; gIter != NULL; gIter = gIter->next) { +@@ -1624,20 +1641,12 @@ action_t * + get_pseudo_op(const char *name, pe_working_set_t * data_set) + { + action_t *op = NULL; +- const char *op_s = name; +- GListPtr possible_matches = NULL; + +- possible_matches = find_actions(data_set->actions, name, NULL); +- if (possible_matches != NULL) { +- if (g_list_length(possible_matches) > 1) { +- pe_warn("Action %s exists %d times", name, g_list_length(possible_matches)); +- } +- +- op = g_list_nth_data(possible_matches, 0); +- g_list_free(possible_matches); +- +- } else { +- op = custom_action(NULL, strdup(op_s), op_s, NULL, TRUE, TRUE, data_set); ++ if(data_set->singletons) { ++ op = g_hash_table_lookup(data_set->singletons, name); ++ } ++ if (op == NULL) { ++ op = custom_action(NULL, strdup(name), name, NULL, TRUE, TRUE, data_set); + set_bit(op->flags, pe_action_pseudo); + set_bit(op->flags, pe_action_runnable); + } +diff --git a/pengine/allocate.c b/pengine/allocate.c +index a07fdad..b9ce069 100644 +--- a/pengine/allocate.c ++++ b/pengine/allocate.c +@@ -1274,6 +1274,10 @@ pe_fence_op(node_t * node, const char *op, pe_working_set_t * data_set) + + key = g_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op); + ++ if(data_set->singletons) { ++ stonith_op = g_hash_table_lookup(data_set->singletons, key); ++ } ++ + if(stonith_op == NULL) { + stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, FALSE, TRUE, data_set); + diff --git a/SOURCES/bz1078078-pcmk-unfencing_is_based_on_device_probes_there_is_no_need_to_unfence_when_normal_resources_are_found_active.patch b/SOURCES/bz1078078-pcmk-unfencing_is_based_on_device_probes_there_is_no_need_to_unfence_when_normal_resources_are_found_active.patch new file mode 100644 index 0000000..c3e1164 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-unfencing_is_based_on_device_probes_there_is_no_need_to_unfence_when_normal_resources_are_found_active.patch @@ -0,0 +1,32 @@ +commit a5bc09d20ea41afe42bbda037d5e8ecb74f2bb15 +Author: Andrew Beekhof +Date: Fri Apr 11 07:39:59 2014 +1000 + + Fix: PE: Unfencing is based on device probes, there is no need to unfence when normal resources are found active + + (cherry picked from commit 7aad7cb9a5a1778dbc13ed56f976086e0393a7e3) + +diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c +index eb10bd8..363e2bc 100644 +--- a/lib/pengine/unpack.c ++++ b/lib/pengine/unpack.c +@@ -2329,17 +2329,8 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op, + if (is_probe && target_rc == 7) { + task_status_i = PCMK_LRM_OP_DONE; + set_bit(rsc->flags, pe_rsc_unexpectedly_running); +- +- if(is_set(rsc->flags, pe_rsc_needs_unfencing)) { +- /* _Require_ unfencing after probing resources that need unfencing and was found active */ +- crm_warn("Operation %s found resource %s active on %s: triggering unfencing", +- task, rsc->id, node->details->uname); +- trigger_unfencing(NULL, node, "Unexpected state", NULL, data_set); +- +- } else { +- pe_rsc_info(rsc, "Operation %s found resource %s active on %s", +- task, rsc->id, node->details->uname); +- } ++ pe_rsc_info(rsc, "Operation %s found resource %s active on %s", ++ task, rsc->id, node->details->uname); + + /* legacy code for pre-0.6.5 operations */ + } else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) { diff --git a/SOURCES/bz1078078-pcmk-warn_about_fencing_when_fencing_is_disabled.patch b/SOURCES/bz1078078-pcmk-warn_about_fencing_when_fencing_is_disabled.patch new file mode 100644 index 0000000..22440d2 --- /dev/null +++ b/SOURCES/bz1078078-pcmk-warn_about_fencing_when_fencing_is_disabled.patch @@ -0,0 +1,19 @@ +commit 583197a0748aaef3dca09fe45cdac30f75304947 +Author: Andrew Beekhof +Date: Thu Apr 10 19:55:26 2014 +1000 + + Correctly warn about fencing when fencing is disabled + +diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c +index a3133e7..225b269 100644 +--- a/lib/pengine/complex.c ++++ b/lib/pengine/complex.c +@@ -574,7 +574,7 @@ common_unpack(xmlNode * xml_obj, resource_t ** rsc, + + } else if (safe_str_eq(value, "fencing")) { + set_bit((*rsc)->flags, pe_rsc_needs_fencing); +- if (is_set(data_set->flags, pe_flag_stonith_enabled)) { ++ if (is_not_set(data_set->flags, pe_flag_stonith_enabled)) { + crm_config_warn("%s requires fencing but fencing is disabled", (*rsc)->id); + } + diff --git a/SPECS/pacemaker.spec b/SPECS/pacemaker.spec index ca0820d..b3df2d7 100644 --- a/SPECS/pacemaker.spec +++ b/SPECS/pacemaker.spec @@ -2,7 +2,7 @@ %global uname hacluster %global pcmk_docdir %{_docdir}/%{name} -%global specversion 29 +%global specversion 31 %global upstream_prefix pacemaker %global upstream_version Pacemaker-1.1.10 @@ -148,6 +148,39 @@ Patch85: bz1070916-pcmk-systemd_prevent_use_of_null_when_determining_if_a Patch86: bz1070916-pcmk-fencing_remove_shadow_definition_and_use_of_variable_progress.patch Patch87: bz1078504-pcmk-pengine_fixes_invalid_transition_caused_by_clones_with_more_than_10_instances.patch Patch88: bz1064519-pcmk-remove_q_0_from_default_dlm_controld_arguments.patch +Patch89: bz1078078-pcmk-crmd_correctly_react_to_successful_unfencing_operations.patch +Patch90: bz1078078-pcmk-pe_indicate_the_type_of_fencing_in_the_fencing_operation_key.patch +Patch91: bz1078078-pcmk-pe_potential_memory_leak.patch +Patch92: bz1078078-pcmk-pe_fix_comments_and_formatting_for_pe_ordering_enum.patch +Patch93: bz1078078-pcmk-pe_ensure_actions_like_shutdown_and_fence_have_a_unique_key.patch +Patch94: bz1078078-pcmk-pe_include_stonith_operation_in_dot_graph_regression_test_results.patch +Patch95: bz1078078-pcmk-pe_include_stonith_operation_in_the_summary_results_and_handle_unfencing.patch +Patch96: bz1078078-pcmk-pe_store_non_resource_actions_in_a_hashtable_where_we_can_easily_and_cheaply_find_them_again.patch +Patch97: bz1078078-pcmk-pe_automatically_re_unfence_a_node_if_the_fencing_device_definition_changes.patch +Patch98: bz1078078-pcmk-pe_fencing_devices_default_to_only_requiring_quorum_in_order_to_start.patch +Patch99: bz1078078-pcmk-pe_delay_unfencing_until_after_we_know_the_state_of_all_resources_that_require_unfencing.patch +Patch100: bz1078078-pcmk-lrmd_provide_stderr_output_from_agents_if_available_otherwise_fall_back_to_stdout.patch +Patch101: bz1078078-pcmk-crmd_report_unsuccessful_unfencing_operations.patch +Patch102: bz1078078-pcmk-fencing_automatically_switch_from_list_to_status_to_static_list_if_those_actions_are_not_advertised_in_the_metadata.patch +Patch103: bz1078078-pcmk-filter_self_fencing_at_the_peers_to_allow_unfencing_to_work_correctly.patch +Patch104: bz1078078-pcmk-pe_ensure_unfencing_occurs_before_fencing_devices_are_re_probed.patch +Patch105: bz1078078-pcmk-logging_daemons_always_get_a_log_file_unless_explicitly_set_to_configured_none.patch +Patch106: bz1078078-pcmk-logging_simplify_processing_of_logging_directives.patch +Patch107: bz1078078-pcmk-lrmd_expose_logging_variables_expected_by_ocf_agents.patch +Patch108: bz1078078-pcmk-fencing_correctly_record_which_peer_performed_the_fencing_operation.patch +Patch109: bz1078078-pcmk-crmd_do_not_erase_the_status_section_for_unfenced_nodes.patch +Patch110: bz1078078-pcmk-crmd_device_isn_t_hooked_up_yet_don_t_bother_logging_it.patch +Patch111: bz1078078-pcmk-pe_ensure_unfencing_only_happens_once_even_if_the_transition_is_interrupted.patch +Patch112: bz1078078-pcmk-pe_do_not_unfence_nodes_that_are_offline_unclean_or_shutting_down.patch +Patch113: bz1078078-pcmk-warn_about_fencing_when_fencing_is_disabled.patch +Patch114: bz1078078-pcmk-crmd_do_not_overwrite_existing_node_state_when_fencing_completes.patch +Patch115: bz1078078-pcmk-unfencing_is_based_on_device_probes_there_is_no_need_to_unfence_when_normal_resources_are_found_active.patch +Patch116: bz1078078-pcmk-pe_repair_regression_tests.patch +Patch117: bz1078078-pcmk-crm_resource_wait_for_the_correct_number_of_replies_when_cleaning_up_resources.patch +Patch118: bz1078078-pcmk-crm_report_suppress_logging_errors_after_the_target_directory_has_been_compressed.patch +Patch119: bz1078078-pcmk-fence_using_all_required_devices.patch +Patch120: bz1078078-pcmk-execute_all_required_fencing_devices_regardless_of_what_topology_level_they_are_at.patch +Patch121: bz1078078-pcmk-default_to_off_when_agent_does_not_advertise_reboot_in_metadata.patch BuildRoot: %(mktemp -ud %{_tmppath}/%{name}-%{version}-%{release}-XXXXXX) AutoReqProv: on @@ -335,95 +368,9 @@ manager for Corosync, CMAN and/or Linux-HA. %prep -%setup -q -n %{upstream_prefix}-%{upstream_version} -%patch1 -p1 -%patch2 -p1 -%patch3 -p1 -%patch4 -p1 -%patch5 -p1 -%patch6 -p1 -%patch7 -p1 -%patch8 -p1 -%patch9 -p1 -%patch10 -p1 -%patch11 -p1 -%patch12 -p1 -%patch13 -p1 -%patch14 -p1 -%patch15 -p1 -%patch16 -p1 -%patch17 -p1 -%patch18 -p1 -%patch19 -p1 -%patch20 -p1 -%patch21 -p1 -%patch22 -p1 -%patch23 -p1 -%patch24 -p1 -%patch25 -p1 -%patch26 -p1 -%patch27 -p1 -%patch28 -p1 -%patch29 -p1 -%patch30 -p1 -%patch31 -p1 -%patch32 -p1 -%patch33 -p1 -%patch34 -p1 -%patch35 -p1 -%patch36 -p1 -%patch37 -p1 -%patch38 -p1 -%patch39 -p1 - -%patch41 -p1 -%patch42 -p1 -%patch43 -p1 -%patch44 -p1 -%patch45 -p1 -%patch46 -p1 -%patch47 -p1 -%patch48 -p1 -%patch49 -p1 -%patch50 -p1 -%patch51 -p1 -%patch52 -p1 -%patch53 -p1 -%patch54 -p1 -%patch55 -p1 -%patch56 -p1 -%patch57 -p1 -%patch58 -p1 -%patch59 -p1 -%patch60 -p1 -%patch61 -p1 -%patch62 -p1 -%patch63 -p1 -%patch64 -p1 -%patch65 -p1 -%patch66 -p1 -%patch67 -p1 -%patch68 -p1 -%patch69 -p1 -%patch70 -p1 -%patch71 -p1 -%patch72 -p1 -%patch73 -p1 -%patch74 -p1 -%patch75 -p1 -%patch76 -p1 -%patch77 -p1 -%patch78 -p1 -%patch79 -p1 -%patch80 -p1 -%patch81 -p1 -%patch82 -p1 -%patch83 -p1 -%patch84 -p1 -%patch85 -p1 -%patch86 -p1 -%patch87 -p1 -%patch88 -p1 +%autosetup -n %{upstream_prefix}-%{upstream_version} -p1 + +# Add '-S git' to have RPM generate a git repo out of the patches # Force the local time # @@ -436,6 +383,9 @@ find . -exec touch \{\} \; %build ./autogen.sh +# For some reason, 'Apache Configuration' is no longer accepted by publican/Kate.pm on RHEL7 +sed -i 's/Apache Configuration/Bash/' doc/Clusters_from_Scratch/en-US/Ch-Apache.txt + # RHEL <= 5 does not support --docdir docdir=%{pcmk_docdir} %{configure} \ %{?with_profiling: --with-profiling} \ @@ -699,6 +649,35 @@ exit 0 %doc AUTHORS %changelog +* Thu Apr 17 2014 David Vossel - 1.1.10-31 + +- fencing: Fence using all required devices +- fencing: Execute all required fencing devices regardless of what topology level they are at +- fencing: default to 'off' when agent does not advertise 'reboot' in metadata + Resolves: rhbz#1078078 + +* Mon Apr 14 2014 Andrew Beekhof 1.1.10-30 + +- crmd: Do not erase the status section for unfenced nodes +- crmd: Correctly react to successful unfencing operations +- crmd: Report unsuccessful unfencing operations +- crmd: Do not overwrite existing node state when fencing completes +- fencing: Correctly record which peer performed the fencing operation +- fencing: Automatically switch from 'list' to 'status' to 'static-list' if those actions are not advertised in the metadata +- fencing: Filter self-fencing at the peers to allow unfencing to work correctly +- pengine: Automatically re-unfence a node if the fencing device definition changes +- pengine: Fencing devices default to only requiring quorum in order to start +- pengine: Delay unfencing until after we know the state of all resources that require unfencing +- pengine: Ensure unfencing occurs before fencing devices are (re-)probed +- pengine: Ensure unfencing only happens once, even if the transition is interrupted +- pengine: Do not unfence nodes that are offline, unclean or shutting down +- pengine: Unfencing is based on device probes, there is no need to unfence when normal resources are found active +- logging: daemons always get a log file, unless explicitly set to configured 'none' +- lrmd: Expose logging variables expected by OCF agents +- crm_report: Suppress logging errors after the target directory has been compressed +- crm_resource: Wait for the correct number of replies when cleaning up resources + Resolves: rhbz#1078078 + * Tue Mar 25 2014 David Vossel - 1.1.10-29 - Low: controld: Remove '-q 0' from default dlm_controld arguments @@ -706,7 +685,7 @@ exit 0 * Tue Mar 25 2014 David Vossel - 1.1.10-28 -- Fix: pengine: fixes invalid transition caused by clones with more than 10 instances +- pengine: fixes invalid transition caused by clones with more than 10 instances Resolves: rhbz#1078504 * Fri Feb 28 2014 Andrew Beekhof - 1.1.10-27