From a72ef2be2240aac04b784710ae81c4671a8189cd Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Sat, 19 Aug 2017 15:45:27 -0500 Subject: [PATCH 1/2] Fix: pengine: re-enable unrecoverable remote fencing The fence loop referenced by 00fed62 was due to get_remote_node_state() considering a failed remote connection resource unrecoverable, even if it was waiting for its reconnect interval to expire before attempting reconnection. --- pengine/allocate.c | 43 +++++++++++++++++---------- pengine/test10/remote-fence-unclean-3.dot | 5 ++++ pengine/test10/remote-fence-unclean-3.exp | 35 ++++++++++++++++++++-- pengine/test10/remote-fence-unclean-3.summary | 3 ++ 4 files changed, 68 insertions(+), 18 deletions(-) diff --git a/pengine/allocate.c b/pengine/allocate.c index 01b5b98..7956916 100644 --- a/pengine/allocate.c +++ b/pengine/allocate.c @@ -905,13 +905,10 @@ probe_resources(pe_working_set_t * data_set) continue; } else if (node->details->online == FALSE && node->details->remote_rsc) { - // TODO figure out why this results in fence loop - /* enum remote_connection_state state = get_remote_node_state(node); if(state == remote_state_failed) { pe_fence_node(data_set, node, "the connection is unrecoverable"); } - */ continue; } else if(node->details->online == FALSE) { @@ -1904,22 +1901,38 @@ get_remote_node_state(pe_node_t *node) * on that remote node until after it starts elsewhere. */ if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) { - /* There is nowhere left to run the connection resource, - * and the resource is in a failed state (either directly - * or because it is located on a failed node). - * - * If there are any resources known to be active on it (stop), - * or if there are resources in an unknown state (probe), we - * must assume the worst and fence it. - */ - if (is_set(remote_rsc->flags, pe_rsc_failed)) { - return remote_state_failed; - } else if(cluster_node && cluster_node->details->unclean) { + /* The connection resource is not going to run anywhere */ + + if (cluster_node && cluster_node->details->unclean) { + /* The remote connection is failed because its resource is on a + * failed node and can't be recovered elsewhere, so we must fence. + */ return remote_state_failed; - } else { + } + + if (is_not_set(remote_rsc->flags, pe_rsc_failed)) { + /* Connection resource is cleanly stopped */ return remote_state_stopped; } + /* Connection resource is failed */ + + if ((remote_rsc->next_role == RSC_ROLE_STOPPED) + && remote_rsc->remote_reconnect_interval + && node->details->remote_was_fenced) { + + /* We won't know whether the connection is recoverable until the + * reconnect interval expires and we reattempt connection. + */ + return remote_state_unknown; + } + + /* The remote connection is in a failed state. If there are any + * resources known to be active on it (stop) or in an unknown state + * (probe), we must assume the worst and fence it. + */ + return remote_state_failed; + } else if (cluster_node == NULL) { /* Connection is recoverable but not currently running anywhere, see if we can recover it first */ return remote_state_unknown; diff --git a/pengine/test10/remote-fence-unclean-3.dot b/pengine/test10/remote-fence-unclean-3.dot index 14adaef..b32b77e 100644 --- a/pengine/test10/remote-fence-unclean-3.dot +++ b/pengine/test10/remote-fence-unclean-3.dot @@ -1,4 +1,5 @@ digraph "g" { +"all_stopped" -> "fence1_start_0 overcloud-controller-0" [ style = bold] "all_stopped" [ style=bold color="green" fontcolor="orange"] "fence1_monitor_0 overcloud-controller-0" -> "fence1_start_0 overcloud-controller-0" [ style = bold] "fence1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] @@ -11,4 +12,8 @@ digraph "g" { "fence1_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] "overcloud-novacompute-0_stop_0 overcloud-controller-0" -> "all_stopped" [ style = bold] "overcloud-novacompute-0_stop_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"] +"stonith 'reboot' overcloud-novacompute-0" -> "stonith_complete" [ style = bold] +"stonith 'reboot' overcloud-novacompute-0" [ style=bold color="green" fontcolor="black"] +"stonith_complete" -> "all_stopped" [ style = bold] +"stonith_complete" [ style=bold color="green" fontcolor="orange"] } diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp index 64e5a62..b7bb735 100644 --- a/pengine/test10/remote-fence-unclean-3.exp +++ b/pengine/test10/remote-fence-unclean-3.exp @@ -21,6 +21,9 @@ + + + @@ -72,6 +75,29 @@ + + + + + + + + + + + + + + + + + + + + + + + @@ -80,6 +106,9 @@ + + + diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary index ec54d8e..6d15598 100644 --- a/pengine/test10/remote-fence-unclean-3.summary +++ b/pengine/test10/remote-fence-unclean-3.summary @@ -34,6 +34,7 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund openstack-cinder-backup-docker-0 (ocf::heartbeat:docker): Started overcloud-controller-1 Transition Summary: + * Fence (reboot) overcloud-novacompute-0 'the connection is unrecoverable' * Start fence1 (overcloud-controller-0) * Stop overcloud-novacompute-0 (overcloud-controller-0) @@ -42,6 +43,8 @@ Executing cluster transition: * Resource action: fence1 monitor on overcloud-controller-1 * Resource action: fence1 monitor on overcloud-controller-0 * Resource action: overcloud-novacompute-0 stop on overcloud-controller-0 + * Fencing overcloud-novacompute-0 (reboot) + * Pseudo action: stonith_complete * Pseudo action: all_stopped * Resource action: fence1 start on overcloud-controller-0 * Resource action: fence1 monitor=60000 on overcloud-controller-0 -- 1.8.3.1 From c700136f437ca74d3981a419dc07bbf9a7f692fb Mon Sep 17 00:00:00 2001 From: Ken Gaillot Date: Mon, 21 Aug 2017 15:23:04 -0500 Subject: [PATCH 2/2] Test: pengine: add regression test for remote connect interval wait --- pengine/regression.sh | 1 + pengine/test10/remote-reconnect-delay.dot | 9 + pengine/test10/remote-reconnect-delay.exp | 49 +++ pengine/test10/remote-reconnect-delay.scores | 207 +++++++++++ pengine/test10/remote-reconnect-delay.summary | 66 ++++ pengine/test10/remote-reconnect-delay.xml | 504 ++++++++++++++++++++++++++ 6 files changed, 836 insertions(+) create mode 100644 pengine/test10/remote-reconnect-delay.dot create mode 100644 pengine/test10/remote-reconnect-delay.exp create mode 100644 pengine/test10/remote-reconnect-delay.scores create mode 100644 pengine/test10/remote-reconnect-delay.summary create mode 100644 pengine/test10/remote-reconnect-delay.xml diff --git a/pengine/regression.sh b/pengine/regression.sh index d1a8a3f..7dc3a04 100755 --- a/pengine/regression.sh +++ b/pengine/regression.sh @@ -853,6 +853,7 @@ do_test remote-recover-connection "Optimistically recovery of only the connectio do_test remote-recover-all "Fencing when the connection has no home" do_test remote-recover-no-resources "Fencing when the connection has no home and no active resources" do_test remote-recover-unknown "Fencing when the connection has no home and the remote has no operation history" +do_test remote-reconnect-delay "Waiting for remote reconnect interval to expire" echo "" do_test resource-discovery "Exercises resource-discovery location constraint option." diff --git a/pengine/test10/remote-reconnect-delay.dot b/pengine/test10/remote-reconnect-delay.dot new file mode 100644 index 0000000..42c2cef --- /dev/null +++ b/pengine/test10/remote-reconnect-delay.dot @@ -0,0 +1,9 @@ +digraph "g" { +"Fencing_monitor_120000 rhel7-2" [ style=bold color="green" fontcolor="black"] +"Fencing_start_0 rhel7-2" -> "Fencing_monitor_120000 rhel7-2" [ style = bold] +"Fencing_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"] +"Fencing_stop_0 rhel7-2" -> "Fencing_start_0 rhel7-2" [ style = bold] +"Fencing_stop_0 rhel7-2" -> "all_stopped" [ style = bold] +"Fencing_stop_0 rhel7-2" [ style=bold color="green" fontcolor="black"] +"all_stopped" [ style=bold color="green" fontcolor="orange"] +} diff --git a/pengine/test10/remote-reconnect-delay.exp b/pengine/test10/remote-reconnect-delay.exp new file mode 100644 index 0000000..fc616c2 --- /dev/null +++ b/pengine/test10/remote-reconnect-delay.exp @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pengine/test10/remote-reconnect-delay.scores b/pengine/test10/remote-reconnect-delay.scores new file mode 100644 index 0000000..411af28 --- /dev/null +++ b/pengine/test10/remote-reconnect-delay.scores @@ -0,0 +1,207 @@ +Allocation scores: +Using the original execution date of: 2017-08-21 17:12:54Z +clone_color: Connectivity allocation score on remote-rhel7-3: 0 +clone_color: Connectivity allocation score on rhel7-1: 0 +clone_color: Connectivity allocation score on rhel7-2: 0 +clone_color: Connectivity allocation score on rhel7-4: 0 +clone_color: Connectivity allocation score on rhel7-5: 0 +clone_color: master-1 allocation score on remote-rhel7-3: -INFINITY +clone_color: master-1 allocation score on rhel7-1: 0 +clone_color: master-1 allocation score on rhel7-2: 0 +clone_color: master-1 allocation score on rhel7-4: 0 +clone_color: master-1 allocation score on rhel7-5: 0 +clone_color: ping-1:0 allocation score on remote-rhel7-3: 0 +clone_color: ping-1:0 allocation score on rhel7-1: 1 +clone_color: ping-1:0 allocation score on rhel7-2: 0 +clone_color: ping-1:0 allocation score on rhel7-4: 0 +clone_color: ping-1:0 allocation score on rhel7-5: 0 +clone_color: ping-1:1 allocation score on remote-rhel7-3: 0 +clone_color: ping-1:1 allocation score on rhel7-1: 0 +clone_color: ping-1:1 allocation score on rhel7-2: 1 +clone_color: ping-1:1 allocation score on rhel7-4: 0 +clone_color: ping-1:1 allocation score on rhel7-5: 0 +clone_color: ping-1:2 allocation score on remote-rhel7-3: 0 +clone_color: ping-1:2 allocation score on rhel7-1: 0 +clone_color: ping-1:2 allocation score on rhel7-2: 0 +clone_color: ping-1:2 allocation score on rhel7-4: 1 +clone_color: ping-1:2 allocation score on rhel7-5: 0 +clone_color: ping-1:3 allocation score on remote-rhel7-3: 0 +clone_color: ping-1:3 allocation score on rhel7-1: 0 +clone_color: ping-1:3 allocation score on rhel7-2: 0 +clone_color: ping-1:3 allocation score on rhel7-4: 0 +clone_color: ping-1:3 allocation score on rhel7-5: 1 +clone_color: ping-1:4 allocation score on remote-rhel7-3: 0 +clone_color: ping-1:4 allocation score on rhel7-1: 0 +clone_color: ping-1:4 allocation score on rhel7-2: 0 +clone_color: ping-1:4 allocation score on rhel7-4: 0 +clone_color: ping-1:4 allocation score on rhel7-5: 0 +clone_color: stateful-1:0 allocation score on remote-rhel7-3: -INFINITY +clone_color: stateful-1:0 allocation score on rhel7-1: 6 +clone_color: stateful-1:0 allocation score on rhel7-2: 0 +clone_color: stateful-1:0 allocation score on rhel7-4: 0 +clone_color: stateful-1:0 allocation score on rhel7-5: 0 +clone_color: stateful-1:1 allocation score on remote-rhel7-3: -INFINITY +clone_color: stateful-1:1 allocation score on rhel7-1: 0 +clone_color: stateful-1:1 allocation score on rhel7-2: 11 +clone_color: stateful-1:1 allocation score on rhel7-4: 0 +clone_color: stateful-1:1 allocation score on rhel7-5: 0 +clone_color: stateful-1:2 allocation score on remote-rhel7-3: -INFINITY +clone_color: stateful-1:2 allocation score on rhel7-1: 0 +clone_color: stateful-1:2 allocation score on rhel7-2: 0 +clone_color: stateful-1:2 allocation score on rhel7-4: 6 +clone_color: stateful-1:2 allocation score on rhel7-5: 0 +clone_color: stateful-1:3 allocation score on remote-rhel7-3: -INFINITY +clone_color: stateful-1:3 allocation score on rhel7-1: 0 +clone_color: stateful-1:3 allocation score on rhel7-2: 0 +clone_color: stateful-1:3 allocation score on rhel7-4: 0 +clone_color: stateful-1:3 allocation score on rhel7-5: 6 +clone_color: stateful-1:4 allocation score on remote-rhel7-3: -INFINITY +clone_color: stateful-1:4 allocation score on rhel7-1: 0 +clone_color: stateful-1:4 allocation score on rhel7-2: 0 +clone_color: stateful-1:4 allocation score on rhel7-4: 0 +clone_color: stateful-1:4 allocation score on rhel7-5: 0 +group_color: group-1 allocation score on remote-rhel7-3: 0 +group_color: group-1 allocation score on rhel7-1: 0 +group_color: group-1 allocation score on rhel7-2: 0 +group_color: group-1 allocation score on rhel7-4: 0 +group_color: group-1 allocation score on rhel7-5: 0 +group_color: petulant allocation score on remote-rhel7-3: 0 +group_color: petulant allocation score on rhel7-1: 0 +group_color: petulant allocation score on rhel7-2: 0 +group_color: petulant allocation score on rhel7-4: 0 +group_color: petulant allocation score on rhel7-5: 0 +group_color: r192.168.122.207 allocation score on remote-rhel7-3: 0 +group_color: r192.168.122.207 allocation score on rhel7-1: 0 +group_color: r192.168.122.207 allocation score on rhel7-2: 0 +group_color: r192.168.122.207 allocation score on rhel7-4: 0 +group_color: r192.168.122.207 allocation score on rhel7-5: 0 +group_color: r192.168.122.208 allocation score on remote-rhel7-3: 0 +group_color: r192.168.122.208 allocation score on rhel7-1: 0 +group_color: r192.168.122.208 allocation score on rhel7-2: 0 +group_color: r192.168.122.208 allocation score on rhel7-4: 0 +group_color: r192.168.122.208 allocation score on rhel7-5: 0 +native_color: Fencing allocation score on remote-rhel7-3: -INFINITY +native_color: Fencing allocation score on rhel7-1: 0 +native_color: Fencing allocation score on rhel7-2: 0 +native_color: Fencing allocation score on rhel7-4: 0 +native_color: Fencing allocation score on rhel7-5: 0 +native_color: FencingFail allocation score on remote-rhel7-3: -INFINITY +native_color: FencingFail allocation score on rhel7-1: 0 +native_color: FencingFail allocation score on rhel7-2: 0 +native_color: FencingFail allocation score on rhel7-4: 0 +native_color: FencingFail allocation score on rhel7-5: 0 +native_color: lsb-dummy allocation score on remote-rhel7-3: -INFINITY +native_color: lsb-dummy allocation score on rhel7-1: -INFINITY +native_color: lsb-dummy allocation score on rhel7-2: 0 +native_color: lsb-dummy allocation score on rhel7-4: -INFINITY +native_color: lsb-dummy allocation score on rhel7-5: -INFINITY +native_color: migrator allocation score on remote-rhel7-3: 0 +native_color: migrator allocation score on rhel7-1: 0 +native_color: migrator allocation score on rhel7-2: 0 +native_color: migrator allocation score on rhel7-4: 0 +native_color: migrator allocation score on rhel7-5: 1 +native_color: petulant allocation score on remote-rhel7-3: -INFINITY +native_color: petulant allocation score on rhel7-1: -INFINITY +native_color: petulant allocation score on rhel7-2: 0 +native_color: petulant allocation score on rhel7-4: -INFINITY +native_color: petulant allocation score on rhel7-5: -INFINITY +native_color: ping-1:0 allocation score on remote-rhel7-3: -INFINITY +native_color: ping-1:0 allocation score on rhel7-1: 1 +native_color: ping-1:0 allocation score on rhel7-2: 0 +native_color: ping-1:0 allocation score on rhel7-4: 0 +native_color: ping-1:0 allocation score on rhel7-5: 0 +native_color: ping-1:1 allocation score on remote-rhel7-3: -INFINITY +native_color: ping-1:1 allocation score on rhel7-1: -INFINITY +native_color: ping-1:1 allocation score on rhel7-2: 1 +native_color: ping-1:1 allocation score on rhel7-4: 0 +native_color: ping-1:1 allocation score on rhel7-5: 0 +native_color: ping-1:2 allocation score on remote-rhel7-3: -INFINITY +native_color: ping-1:2 allocation score on rhel7-1: -INFINITY +native_color: ping-1:2 allocation score on rhel7-2: -INFINITY +native_color: ping-1:2 allocation score on rhel7-4: 1 +native_color: ping-1:2 allocation score on rhel7-5: 0 +native_color: ping-1:3 allocation score on remote-rhel7-3: -INFINITY +native_color: ping-1:3 allocation score on rhel7-1: -INFINITY +native_color: ping-1:3 allocation score on rhel7-2: -INFINITY +native_color: ping-1:3 allocation score on rhel7-4: -INFINITY +native_color: ping-1:3 allocation score on rhel7-5: 1 +native_color: ping-1:4 allocation score on remote-rhel7-3: -INFINITY +native_color: ping-1:4 allocation score on rhel7-1: -INFINITY +native_color: ping-1:4 allocation score on rhel7-2: -INFINITY +native_color: ping-1:4 allocation score on rhel7-4: -INFINITY +native_color: ping-1:4 allocation score on rhel7-5: -INFINITY +native_color: r192.168.122.207 allocation score on remote-rhel7-3: -INFINITY +native_color: r192.168.122.207 allocation score on rhel7-1: -INFINITY +native_color: r192.168.122.207 allocation score on rhel7-2: 11 +native_color: r192.168.122.207 allocation score on rhel7-4: -INFINITY +native_color: r192.168.122.207 allocation score on rhel7-5: -INFINITY +native_color: r192.168.122.208 allocation score on remote-rhel7-3: -INFINITY +native_color: r192.168.122.208 allocation score on rhel7-1: -INFINITY +native_color: r192.168.122.208 allocation score on rhel7-2: 0 +native_color: r192.168.122.208 allocation score on rhel7-4: -INFINITY +native_color: r192.168.122.208 allocation score on rhel7-5: -INFINITY +native_color: remote-rhel7-3 allocation score on remote-rhel7-3: -INFINITY +native_color: remote-rhel7-3 allocation score on rhel7-1: -INFINITY +native_color: remote-rhel7-3 allocation score on rhel7-2: -INFINITY +native_color: remote-rhel7-3 allocation score on rhel7-4: -INFINITY +native_color: remote-rhel7-3 allocation score on rhel7-5: -INFINITY +native_color: remote-rsc allocation score on remote-rhel7-3: INFINITY +native_color: remote-rsc allocation score on rhel7-1: 0 +native_color: remote-rsc allocation score on rhel7-2: 0 +native_color: remote-rsc allocation score on rhel7-4: 0 +native_color: remote-rsc allocation score on rhel7-5: 0 +native_color: rsc_rhel7-1 allocation score on remote-rhel7-3: 0 +native_color: rsc_rhel7-1 allocation score on rhel7-1: 100 +native_color: rsc_rhel7-1 allocation score on rhel7-2: 0 +native_color: rsc_rhel7-1 allocation score on rhel7-4: 0 +native_color: rsc_rhel7-1 allocation score on rhel7-5: 0 +native_color: rsc_rhel7-2 allocation score on remote-rhel7-3: 0 +native_color: rsc_rhel7-2 allocation score on rhel7-1: 0 +native_color: rsc_rhel7-2 allocation score on rhel7-2: 100 +native_color: rsc_rhel7-2 allocation score on rhel7-4: 0 +native_color: rsc_rhel7-2 allocation score on rhel7-5: 0 +native_color: rsc_rhel7-3 allocation score on remote-rhel7-3: 0 +native_color: rsc_rhel7-3 allocation score on rhel7-1: 0 +native_color: rsc_rhel7-3 allocation score on rhel7-2: 0 +native_color: rsc_rhel7-3 allocation score on rhel7-4: 0 +native_color: rsc_rhel7-3 allocation score on rhel7-5: 0 +native_color: rsc_rhel7-4 allocation score on remote-rhel7-3: 0 +native_color: rsc_rhel7-4 allocation score on rhel7-1: 0 +native_color: rsc_rhel7-4 allocation score on rhel7-2: 0 +native_color: rsc_rhel7-4 allocation score on rhel7-4: 100 +native_color: rsc_rhel7-4 allocation score on rhel7-5: 0 +native_color: rsc_rhel7-5 allocation score on remote-rhel7-3: 0 +native_color: rsc_rhel7-5 allocation score on rhel7-1: 0 +native_color: rsc_rhel7-5 allocation score on rhel7-2: 0 +native_color: rsc_rhel7-5 allocation score on rhel7-4: 0 +native_color: rsc_rhel7-5 allocation score on rhel7-5: 100 +native_color: stateful-1:0 allocation score on remote-rhel7-3: -INFINITY +native_color: stateful-1:0 allocation score on rhel7-1: 6 +native_color: stateful-1:0 allocation score on rhel7-2: -INFINITY +native_color: stateful-1:0 allocation score on rhel7-4: 0 +native_color: stateful-1:0 allocation score on rhel7-5: 0 +native_color: stateful-1:1 allocation score on remote-rhel7-3: -INFINITY +native_color: stateful-1:1 allocation score on rhel7-1: 0 +native_color: stateful-1:1 allocation score on rhel7-2: 11 +native_color: stateful-1:1 allocation score on rhel7-4: 0 +native_color: stateful-1:1 allocation score on rhel7-5: 0 +native_color: stateful-1:2 allocation score on remote-rhel7-3: -INFINITY +native_color: stateful-1:2 allocation score on rhel7-1: -INFINITY +native_color: stateful-1:2 allocation score on rhel7-2: -INFINITY +native_color: stateful-1:2 allocation score on rhel7-4: 6 +native_color: stateful-1:2 allocation score on rhel7-5: 0 +native_color: stateful-1:3 allocation score on remote-rhel7-3: -INFINITY +native_color: stateful-1:3 allocation score on rhel7-1: -INFINITY +native_color: stateful-1:3 allocation score on rhel7-2: -INFINITY +native_color: stateful-1:3 allocation score on rhel7-4: -INFINITY +native_color: stateful-1:3 allocation score on rhel7-5: 6 +native_color: stateful-1:4 allocation score on remote-rhel7-3: -INFINITY +native_color: stateful-1:4 allocation score on rhel7-1: -INFINITY +native_color: stateful-1:4 allocation score on rhel7-2: -INFINITY +native_color: stateful-1:4 allocation score on rhel7-4: -INFINITY +native_color: stateful-1:4 allocation score on rhel7-5: -INFINITY +stateful-1:0 promotion score on rhel7-1: 5 +stateful-1:1 promotion score on rhel7-2: 10 +stateful-1:2 promotion score on rhel7-4: 5 +stateful-1:3 promotion score on rhel7-5: 5 +stateful-1:4 promotion score on none: 0 diff --git a/pengine/test10/remote-reconnect-delay.summary b/pengine/test10/remote-reconnect-delay.summary new file mode 100644 index 0000000..ea11483 --- /dev/null +++ b/pengine/test10/remote-reconnect-delay.summary @@ -0,0 +1,66 @@ +Using the original execution date of: 2017-08-21 17:12:54Z + +Current cluster status: +Online: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] +RemoteOFFLINE: [ remote-rhel7-3 ] + + Fencing (stonith:fence_xvm): Started rhel7-2 + FencingFail (stonith:fence_dummy): Started rhel7-4 + rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-1 + rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Started rhel7-2 + rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Started rhel7-5 + rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Started rhel7-4 + rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Started rhel7-5 + migrator (ocf::pacemaker:Dummy): Started rhel7-5 + Clone Set: Connectivity [ping-1] + Started: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] + Stopped: [ remote-rhel7-3 ] + Master/Slave Set: master-1 [stateful-1] + Masters: [ rhel7-2 ] + Slaves: [ rhel7-1 rhel7-4 rhel7-5 ] + Stopped: [ remote-rhel7-3 ] + Resource Group: group-1 + r192.168.122.207 (ocf::heartbeat:IPaddr2): Started rhel7-2 + petulant (service:DummySD): Started rhel7-2 + r192.168.122.208 (ocf::heartbeat:IPaddr2): Started rhel7-2 + lsb-dummy (lsb:/usr/share/pacemaker/tests/cts/LSBDummy): Started rhel7-2 + remote-rhel7-3 (ocf::pacemaker:remote): FAILED + remote-rsc (ocf::heartbeat:Dummy): Started rhel7-1 + +Transition Summary: + * Restart Fencing ( rhel7-2 ) + +Executing cluster transition: + * Resource action: Fencing stop on rhel7-2 + * Resource action: Fencing start on rhel7-2 + * Resource action: Fencing monitor=120000 on rhel7-2 + * Pseudo action: all_stopped +Using the original execution date of: 2017-08-21 17:12:54Z + +Revised cluster status: +Online: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] +RemoteOFFLINE: [ remote-rhel7-3 ] + + Fencing (stonith:fence_xvm): Started rhel7-2 + FencingFail (stonith:fence_dummy): Started rhel7-4 + rsc_rhel7-1 (ocf::heartbeat:IPaddr2): Started rhel7-1 + rsc_rhel7-2 (ocf::heartbeat:IPaddr2): Started rhel7-2 + rsc_rhel7-3 (ocf::heartbeat:IPaddr2): Started rhel7-5 + rsc_rhel7-4 (ocf::heartbeat:IPaddr2): Started rhel7-4 + rsc_rhel7-5 (ocf::heartbeat:IPaddr2): Started rhel7-5 + migrator (ocf::pacemaker:Dummy): Started rhel7-5 + Clone Set: Connectivity [ping-1] + Started: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ] + Stopped: [ remote-rhel7-3 ] + Master/Slave Set: master-1 [stateful-1] + Masters: [ rhel7-2 ] + Slaves: [ rhel7-1 rhel7-4 rhel7-5 ] + Stopped: [ remote-rhel7-3 ] + Resource Group: group-1 + r192.168.122.207 (ocf::heartbeat:IPaddr2): Started rhel7-2 + petulant (service:DummySD): Started rhel7-2 + r192.168.122.208 (ocf::heartbeat:IPaddr2): Started rhel7-2 + lsb-dummy (lsb:/usr/share/pacemaker/tests/cts/LSBDummy): Started rhel7-2 + remote-rhel7-3 (ocf::pacemaker:remote): FAILED + remote-rsc (ocf::heartbeat:Dummy): Started rhel7-1 + diff --git a/pengine/test10/remote-reconnect-delay.xml b/pengine/test10/remote-reconnect-delay.xml new file mode 100644 index 0000000..e9ed3e6 --- /dev/null +++ b/pengine/test10/remote-reconnect-delay.xml @@ -0,0 +1,504 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + -- 1.8.3.1