Blame SOURCES/092-remote-fencing.patch

356a11
From a72ef2be2240aac04b784710ae81c4671a8189cd Mon Sep 17 00:00:00 2001
356a11
From: Ken Gaillot <kgaillot@redhat.com>
356a11
Date: Sat, 19 Aug 2017 15:45:27 -0500
356a11
Subject: [PATCH 1/2] Fix: pengine: re-enable unrecoverable remote fencing
356a11
356a11
The fence loop referenced by 00fed62 was due to get_remote_node_state()
356a11
considering a failed remote connection resource unrecoverable, even if it was
356a11
waiting for its reconnect interval to expire before attempting reconnection.
356a11
---
356a11
 pengine/allocate.c                            | 43 +++++++++++++++++----------
356a11
 pengine/test10/remote-fence-unclean-3.dot     |  5 ++++
356a11
 pengine/test10/remote-fence-unclean-3.exp     | 35 ++++++++++++++++++++--
356a11
 pengine/test10/remote-fence-unclean-3.summary |  3 ++
356a11
 4 files changed, 68 insertions(+), 18 deletions(-)
356a11
356a11
diff --git a/pengine/allocate.c b/pengine/allocate.c
356a11
index 01b5b98..7956916 100644
356a11
--- a/pengine/allocate.c
356a11
+++ b/pengine/allocate.c
356a11
@@ -905,13 +905,10 @@ probe_resources(pe_working_set_t * data_set)
356a11
             continue;
356a11
 
356a11
         } else if (node->details->online == FALSE && node->details->remote_rsc) {
356a11
-            // TODO figure out why this results in fence loop
356a11
-            /*
356a11
             enum remote_connection_state state = get_remote_node_state(node);
356a11
             if(state == remote_state_failed) {
356a11
                 pe_fence_node(data_set, node, "the connection is unrecoverable");
356a11
             }
356a11
-            */
356a11
             continue;
356a11
 
356a11
         } else if(node->details->online == FALSE) {
356a11
@@ -1904,22 +1901,38 @@ get_remote_node_state(pe_node_t *node)
356a11
      * on that remote node until after it starts elsewhere.
356a11
      */
356a11
     if(remote_rsc->next_role == RSC_ROLE_STOPPED || remote_rsc->allocated_to == NULL) {
356a11
-        /* There is nowhere left to run the connection resource,
356a11
-         * and the resource is in a failed state (either directly
356a11
-         * or because it is located on a failed node).
356a11
-         *
356a11
-         * If there are any resources known to be active on it (stop),
356a11
-         * or if there are resources in an unknown state (probe), we
356a11
-         * must assume the worst and fence it.
356a11
-         */
356a11
-        if (is_set(remote_rsc->flags, pe_rsc_failed)) {
356a11
-            return remote_state_failed;
356a11
-        } else if(cluster_node && cluster_node->details->unclean) {
356a11
+        /* The connection resource is not going to run anywhere */
356a11
+
356a11
+        if (cluster_node && cluster_node->details->unclean) {
356a11
+            /* The remote connection is failed because its resource is on a
356a11
+             * failed node and can't be recovered elsewhere, so we must fence.
356a11
+             */
356a11
             return remote_state_failed;
356a11
-        } else {
356a11
+        }
356a11
+
356a11
+        if (is_not_set(remote_rsc->flags, pe_rsc_failed)) {
356a11
+            /* Connection resource is cleanly stopped */
356a11
             return remote_state_stopped;
356a11
         }
356a11
 
356a11
+        /* Connection resource is failed */
356a11
+
356a11
+        if ((remote_rsc->next_role == RSC_ROLE_STOPPED)
356a11
+            && remote_rsc->remote_reconnect_interval
356a11
+            && node->details->remote_was_fenced) {
356a11
+
356a11
+            /* We won't know whether the connection is recoverable until the
356a11
+             * reconnect interval expires and we reattempt connection.
356a11
+             */
356a11
+            return remote_state_unknown;
356a11
+        }
356a11
+
356a11
+        /* The remote connection is in a failed state. If there are any
356a11
+         * resources known to be active on it (stop) or in an unknown state
356a11
+         * (probe), we must assume the worst and fence it.
356a11
+         */
356a11
+        return remote_state_failed;
356a11
+
356a11
     } else if (cluster_node == NULL) {
356a11
         /* Connection is recoverable but not currently running anywhere, see if we can recover it first */
356a11
         return remote_state_unknown;
356a11
diff --git a/pengine/test10/remote-fence-unclean-3.dot b/pengine/test10/remote-fence-unclean-3.dot
356a11
index 14adaef..b32b77e 100644
356a11
--- a/pengine/test10/remote-fence-unclean-3.dot
356a11
+++ b/pengine/test10/remote-fence-unclean-3.dot
356a11
@@ -1,4 +1,5 @@
356a11
 digraph "g" {
356a11
+"all_stopped" -> "fence1_start_0 overcloud-controller-0" [ style = bold]
356a11
 "all_stopped" [ style=bold color="green" fontcolor="orange"]
356a11
 "fence1_monitor_0 overcloud-controller-0" -> "fence1_start_0 overcloud-controller-0" [ style = bold]
356a11
 "fence1_monitor_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"]
356a11
@@ -11,4 +12,8 @@ digraph "g" {
356a11
 "fence1_start_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"]
356a11
 "overcloud-novacompute-0_stop_0 overcloud-controller-0" -> "all_stopped" [ style = bold]
356a11
 "overcloud-novacompute-0_stop_0 overcloud-controller-0" [ style=bold color="green" fontcolor="black"]
356a11
+"stonith 'reboot' overcloud-novacompute-0" -> "stonith_complete" [ style = bold]
356a11
+"stonith 'reboot' overcloud-novacompute-0" [ style=bold color="green" fontcolor="black"]
356a11
+"stonith_complete" -> "all_stopped" [ style = bold]
356a11
+"stonith_complete" [ style=bold color="green" fontcolor="orange"]
356a11
 }
356a11
diff --git a/pengine/test10/remote-fence-unclean-3.exp b/pengine/test10/remote-fence-unclean-3.exp
356a11
index 64e5a62..b7bb735 100644
356a11
--- a/pengine/test10/remote-fence-unclean-3.exp
356a11
+++ b/pengine/test10/remote-fence-unclean-3.exp
356a11
@@ -21,6 +21,9 @@
356a11
     </action_set>
356a11
     <inputs>
356a11
       <trigger>
356a11
+        <pseudo_event id="42" operation="all_stopped" operation_key="all_stopped"/>
356a11
+      </trigger>
356a11
+      <trigger>
356a11
         <rsc_op id="43" operation="monitor" operation_key="fence1_monitor_0" on_node="overcloud-controller-0" on_node_uuid="1"/>
356a11
       </trigger>
356a11
       <trigger>
356a11
@@ -72,6 +75,29 @@
356a11
   </synapse>
356a11
   <synapse id="6">
356a11
     <action_set>
356a11
+      <pseudo_event id="209" operation="stonith_complete" operation_key="stonith_complete">
356a11
+        <attributes />
356a11
+      </pseudo_event>
356a11
+    </action_set>
356a11
+    <inputs>
356a11
+      <trigger>
356a11
+        <crm_event id="46" operation="stonith" operation_key="stonith-overcloud-novacompute-0-reboot" on_node="overcloud-novacompute-0" on_node_uuid="overcloud-novacompute-0"/>
356a11
+      </trigger>
356a11
+    </inputs>
356a11
+  </synapse>
356a11
+  <synapse id="7">
356a11
+    <action_set>
356a11
+      <crm_event id="46" operation="stonith" operation_key="stonith-overcloud-novacompute-0-reboot" on_node="overcloud-novacompute-0" on_node_uuid="overcloud-novacompute-0">
356a11
+        <attributes CRM_meta_compute_role="true" CRM_meta_on_node="overcloud-novacompute-0" CRM_meta_on_node_uuid="overcloud-novacompute-0" CRM_meta_stonith_action="reboot" />
356a11
+        <downed>
356a11
+          <node id="overcloud-novacompute-0"/>
356a11
+        </downed>
356a11
+      </crm_event>
356a11
+    </action_set>
356a11
+    <inputs/>
356a11
+  </synapse>
356a11
+  <synapse id="8">
356a11
+    <action_set>
356a11
       <pseudo_event id="42" operation="all_stopped" operation_key="all_stopped">
356a11
         <attributes />
356a11
       </pseudo_event>
356a11
@@ -80,6 +106,9 @@
356a11
       <trigger>
356a11
         <rsc_op id="30" operation="stop" operation_key="overcloud-novacompute-0_stop_0" on_node="overcloud-controller-0" on_node_uuid="1"/>
356a11
       </trigger>
356a11
+      <trigger>
356a11
+        <pseudo_event id="209" operation="stonith_complete" operation_key="stonith_complete"/>
356a11
+      </trigger>
356a11
     </inputs>
356a11
   </synapse>
356a11
 </transition_graph>
356a11
diff --git a/pengine/test10/remote-fence-unclean-3.summary b/pengine/test10/remote-fence-unclean-3.summary
356a11
index ec54d8e..6d15598 100644
356a11
--- a/pengine/test10/remote-fence-unclean-3.summary
356a11
+++ b/pengine/test10/remote-fence-unclean-3.summary
356a11
@@ -34,6 +34,7 @@ Containers: [ galera-bundle-0:galera-bundle-docker-0 galera-bundle-1:galera-bund
356a11
    openstack-cinder-backup-docker-0	(ocf::heartbeat:docker):	Started overcloud-controller-1
356a11
 
356a11
 Transition Summary:
356a11
+ * Fence (reboot) overcloud-novacompute-0 'the connection is unrecoverable'
356a11
  * Start   fence1	(overcloud-controller-0)
356a11
  * Stop    overcloud-novacompute-0	(overcloud-controller-0)
356a11
 
356a11
@@ -42,6 +43,8 @@ Executing cluster transition:
356a11
  * Resource action: fence1          monitor on overcloud-controller-1
356a11
  * Resource action: fence1          monitor on overcloud-controller-0
356a11
  * Resource action: overcloud-novacompute-0 stop on overcloud-controller-0
356a11
+ * Fencing overcloud-novacompute-0 (reboot)
356a11
+ * Pseudo action:   stonith_complete
356a11
  * Pseudo action:   all_stopped
356a11
  * Resource action: fence1          start on overcloud-controller-0
356a11
  * Resource action: fence1          monitor=60000 on overcloud-controller-0
356a11
-- 
356a11
1.8.3.1
356a11
356a11
356a11
From c700136f437ca74d3981a419dc07bbf9a7f692fb Mon Sep 17 00:00:00 2001
356a11
From: Ken Gaillot <kgaillot@redhat.com>
356a11
Date: Mon, 21 Aug 2017 15:23:04 -0500
356a11
Subject: [PATCH 2/2] Test: pengine: add regression test for remote connect
356a11
 interval wait
356a11
356a11
---
356a11
 pengine/regression.sh                         |   1 +
356a11
 pengine/test10/remote-reconnect-delay.dot     |   9 +
356a11
 pengine/test10/remote-reconnect-delay.exp     |  49 +++
356a11
 pengine/test10/remote-reconnect-delay.scores  | 207 +++++++++++
356a11
 pengine/test10/remote-reconnect-delay.summary |  66 ++++
356a11
 pengine/test10/remote-reconnect-delay.xml     | 504 ++++++++++++++++++++++++++
356a11
 6 files changed, 836 insertions(+)
356a11
 create mode 100644 pengine/test10/remote-reconnect-delay.dot
356a11
 create mode 100644 pengine/test10/remote-reconnect-delay.exp
356a11
 create mode 100644 pengine/test10/remote-reconnect-delay.scores
356a11
 create mode 100644 pengine/test10/remote-reconnect-delay.summary
356a11
 create mode 100644 pengine/test10/remote-reconnect-delay.xml
356a11
356a11
diff --git a/pengine/regression.sh b/pengine/regression.sh
356a11
index d1a8a3f..7dc3a04 100755
356a11
--- a/pengine/regression.sh
356a11
+++ b/pengine/regression.sh
356a11
@@ -853,6 +853,7 @@ do_test remote-recover-connection "Optimistically recovery of only the connectio
356a11
 do_test remote-recover-all        "Fencing when the connection has no home"
356a11
 do_test remote-recover-no-resources   "Fencing when the connection has no home and no active resources"
356a11
 do_test remote-recover-unknown        "Fencing when the connection has no home and the remote has no operation history"
356a11
+do_test remote-reconnect-delay        "Waiting for remote reconnect interval to expire"
356a11
 
356a11
 echo ""
356a11
 do_test resource-discovery      "Exercises resource-discovery location constraint option."
356a11
diff --git a/pengine/test10/remote-reconnect-delay.dot b/pengine/test10/remote-reconnect-delay.dot
356a11
new file mode 100644
356a11
index 0000000..42c2cef
356a11
--- /dev/null
356a11
+++ b/pengine/test10/remote-reconnect-delay.dot
356a11
@@ -0,0 +1,9 @@
356a11
+digraph "g" {
356a11
+"Fencing_monitor_120000 rhel7-2" [ style=bold color="green" fontcolor="black"]
356a11
+"Fencing_start_0 rhel7-2" -> "Fencing_monitor_120000 rhel7-2" [ style = bold]
356a11
+"Fencing_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"]
356a11
+"Fencing_stop_0 rhel7-2" -> "Fencing_start_0 rhel7-2" [ style = bold]
356a11
+"Fencing_stop_0 rhel7-2" -> "all_stopped" [ style = bold]
356a11
+"Fencing_stop_0 rhel7-2" [ style=bold color="green" fontcolor="black"]
356a11
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
356a11
+}
356a11
diff --git a/pengine/test10/remote-reconnect-delay.exp b/pengine/test10/remote-reconnect-delay.exp
356a11
new file mode 100644
356a11
index 0000000..fc616c2
356a11
--- /dev/null
356a11
+++ b/pengine/test10/remote-reconnect-delay.exp
356a11
@@ -0,0 +1,49 @@
356a11
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="1"  transition_id="0">
356a11
+  <synapse id="0">
356a11
+    <action_set>
356a11
+      <rsc_op id="23" operation="stop" operation_key="Fencing_stop_0" on_node="rhel7-2" on_node_uuid="2">
356a11
+        <primitive id="Fencing" class="stonith" type="fence_xvm"/>
356a11
+        <attributes CRM_meta_name="stop" CRM_meta_on_node="rhel7-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="60000"  multicast_address="239.255.100.100" pcmk_arg_map="domain:uname" pcmk_host_list="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5" pcmk_host_map="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
356a11
+      </rsc_op>
356a11
+    </action_set>
356a11
+    <inputs/>
356a11
+  </synapse>
356a11
+  <synapse id="1">
356a11
+    <action_set>
356a11
+      <rsc_op id="22" operation="start" operation_key="Fencing_start_0" on_node="rhel7-2" on_node_uuid="2">
356a11
+        <primitive id="Fencing" class="stonith" type="fence_xvm"/>
356a11
+        <attributes CRM_meta_name="start" CRM_meta_on_node="rhel7-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="60000"  multicast_address="239.255.100.100" pcmk_arg_map="domain:uname" pcmk_host_list="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5" pcmk_host_map="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
356a11
+      </rsc_op>
356a11
+    </action_set>
356a11
+    <inputs>
356a11
+      <trigger>
356a11
+        <rsc_op id="23" operation="stop" operation_key="Fencing_stop_0" on_node="rhel7-2" on_node_uuid="2"/>
356a11
+      </trigger>
356a11
+    </inputs>
356a11
+  </synapse>
356a11
+  <synapse id="2">
356a11
+    <action_set>
356a11
+      <rsc_op id="8" operation="monitor" operation_key="Fencing_monitor_120000" on_node="rhel7-2" on_node_uuid="2">
356a11
+        <primitive id="Fencing" class="stonith" type="fence_xvm"/>
356a11
+        <attributes CRM_meta_interval="120000" CRM_meta_name="monitor" CRM_meta_on_node="rhel7-2" CRM_meta_on_node_uuid="2" CRM_meta_timeout="120000"  multicast_address="239.255.100.100" pcmk_arg_map="domain:uname" pcmk_host_list="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5" pcmk_host_map="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
356a11
+      </rsc_op>
356a11
+    </action_set>
356a11
+    <inputs>
356a11
+      <trigger>
356a11
+        <rsc_op id="22" operation="start" operation_key="Fencing_start_0" on_node="rhel7-2" on_node_uuid="2"/>
356a11
+      </trigger>
356a11
+    </inputs>
356a11
+  </synapse>
356a11
+  <synapse id="3">
356a11
+    <action_set>
356a11
+      <pseudo_event id="21" operation="all_stopped" operation_key="all_stopped">
356a11
+        <attributes />
356a11
+      </pseudo_event>
356a11
+    </action_set>
356a11
+    <inputs>
356a11
+      <trigger>
356a11
+        <rsc_op id="23" operation="stop" operation_key="Fencing_stop_0" on_node="rhel7-2" on_node_uuid="2"/>
356a11
+      </trigger>
356a11
+    </inputs>
356a11
+  </synapse>
356a11
+</transition_graph>
356a11
diff --git a/pengine/test10/remote-reconnect-delay.scores b/pengine/test10/remote-reconnect-delay.scores
356a11
new file mode 100644
356a11
index 0000000..411af28
356a11
--- /dev/null
356a11
+++ b/pengine/test10/remote-reconnect-delay.scores
356a11
@@ -0,0 +1,207 @@
356a11
+Allocation scores:
356a11
+Using the original execution date of: 2017-08-21 17:12:54Z
356a11
+clone_color: Connectivity allocation score on remote-rhel7-3: 0
356a11
+clone_color: Connectivity allocation score on rhel7-1: 0
356a11
+clone_color: Connectivity allocation score on rhel7-2: 0
356a11
+clone_color: Connectivity allocation score on rhel7-4: 0
356a11
+clone_color: Connectivity allocation score on rhel7-5: 0
356a11
+clone_color: master-1 allocation score on remote-rhel7-3: -INFINITY
356a11
+clone_color: master-1 allocation score on rhel7-1: 0
356a11
+clone_color: master-1 allocation score on rhel7-2: 0
356a11
+clone_color: master-1 allocation score on rhel7-4: 0
356a11
+clone_color: master-1 allocation score on rhel7-5: 0
356a11
+clone_color: ping-1:0 allocation score on remote-rhel7-3: 0
356a11
+clone_color: ping-1:0 allocation score on rhel7-1: 1
356a11
+clone_color: ping-1:0 allocation score on rhel7-2: 0
356a11
+clone_color: ping-1:0 allocation score on rhel7-4: 0
356a11
+clone_color: ping-1:0 allocation score on rhel7-5: 0
356a11
+clone_color: ping-1:1 allocation score on remote-rhel7-3: 0
356a11
+clone_color: ping-1:1 allocation score on rhel7-1: 0
356a11
+clone_color: ping-1:1 allocation score on rhel7-2: 1
356a11
+clone_color: ping-1:1 allocation score on rhel7-4: 0
356a11
+clone_color: ping-1:1 allocation score on rhel7-5: 0
356a11
+clone_color: ping-1:2 allocation score on remote-rhel7-3: 0
356a11
+clone_color: ping-1:2 allocation score on rhel7-1: 0
356a11
+clone_color: ping-1:2 allocation score on rhel7-2: 0
356a11
+clone_color: ping-1:2 allocation score on rhel7-4: 1
356a11
+clone_color: ping-1:2 allocation score on rhel7-5: 0
356a11
+clone_color: ping-1:3 allocation score on remote-rhel7-3: 0
356a11
+clone_color: ping-1:3 allocation score on rhel7-1: 0
356a11
+clone_color: ping-1:3 allocation score on rhel7-2: 0
356a11
+clone_color: ping-1:3 allocation score on rhel7-4: 0
356a11
+clone_color: ping-1:3 allocation score on rhel7-5: 1
356a11
+clone_color: ping-1:4 allocation score on remote-rhel7-3: 0
356a11
+clone_color: ping-1:4 allocation score on rhel7-1: 0
356a11
+clone_color: ping-1:4 allocation score on rhel7-2: 0
356a11
+clone_color: ping-1:4 allocation score on rhel7-4: 0
356a11
+clone_color: ping-1:4 allocation score on rhel7-5: 0
356a11
+clone_color: stateful-1:0 allocation score on remote-rhel7-3: -INFINITY
356a11
+clone_color: stateful-1:0 allocation score on rhel7-1: 6
356a11
+clone_color: stateful-1:0 allocation score on rhel7-2: 0
356a11
+clone_color: stateful-1:0 allocation score on rhel7-4: 0
356a11
+clone_color: stateful-1:0 allocation score on rhel7-5: 0
356a11
+clone_color: stateful-1:1 allocation score on remote-rhel7-3: -INFINITY
356a11
+clone_color: stateful-1:1 allocation score on rhel7-1: 0
356a11
+clone_color: stateful-1:1 allocation score on rhel7-2: 11
356a11
+clone_color: stateful-1:1 allocation score on rhel7-4: 0
356a11
+clone_color: stateful-1:1 allocation score on rhel7-5: 0
356a11
+clone_color: stateful-1:2 allocation score on remote-rhel7-3: -INFINITY
356a11
+clone_color: stateful-1:2 allocation score on rhel7-1: 0
356a11
+clone_color: stateful-1:2 allocation score on rhel7-2: 0
356a11
+clone_color: stateful-1:2 allocation score on rhel7-4: 6
356a11
+clone_color: stateful-1:2 allocation score on rhel7-5: 0
356a11
+clone_color: stateful-1:3 allocation score on remote-rhel7-3: -INFINITY
356a11
+clone_color: stateful-1:3 allocation score on rhel7-1: 0
356a11
+clone_color: stateful-1:3 allocation score on rhel7-2: 0
356a11
+clone_color: stateful-1:3 allocation score on rhel7-4: 0
356a11
+clone_color: stateful-1:3 allocation score on rhel7-5: 6
356a11
+clone_color: stateful-1:4 allocation score on remote-rhel7-3: -INFINITY
356a11
+clone_color: stateful-1:4 allocation score on rhel7-1: 0
356a11
+clone_color: stateful-1:4 allocation score on rhel7-2: 0
356a11
+clone_color: stateful-1:4 allocation score on rhel7-4: 0
356a11
+clone_color: stateful-1:4 allocation score on rhel7-5: 0
356a11
+group_color: group-1 allocation score on remote-rhel7-3: 0
356a11
+group_color: group-1 allocation score on rhel7-1: 0
356a11
+group_color: group-1 allocation score on rhel7-2: 0
356a11
+group_color: group-1 allocation score on rhel7-4: 0
356a11
+group_color: group-1 allocation score on rhel7-5: 0
356a11
+group_color: petulant allocation score on remote-rhel7-3: 0
356a11
+group_color: petulant allocation score on rhel7-1: 0
356a11
+group_color: petulant allocation score on rhel7-2: 0
356a11
+group_color: petulant allocation score on rhel7-4: 0
356a11
+group_color: petulant allocation score on rhel7-5: 0
356a11
+group_color: r192.168.122.207 allocation score on remote-rhel7-3: 0
356a11
+group_color: r192.168.122.207 allocation score on rhel7-1: 0
356a11
+group_color: r192.168.122.207 allocation score on rhel7-2: 0
356a11
+group_color: r192.168.122.207 allocation score on rhel7-4: 0
356a11
+group_color: r192.168.122.207 allocation score on rhel7-5: 0
356a11
+group_color: r192.168.122.208 allocation score on remote-rhel7-3: 0
356a11
+group_color: r192.168.122.208 allocation score on rhel7-1: 0
356a11
+group_color: r192.168.122.208 allocation score on rhel7-2: 0
356a11
+group_color: r192.168.122.208 allocation score on rhel7-4: 0
356a11
+group_color: r192.168.122.208 allocation score on rhel7-5: 0
356a11
+native_color: Fencing allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: Fencing allocation score on rhel7-1: 0
356a11
+native_color: Fencing allocation score on rhel7-2: 0
356a11
+native_color: Fencing allocation score on rhel7-4: 0
356a11
+native_color: Fencing allocation score on rhel7-5: 0
356a11
+native_color: FencingFail allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: FencingFail allocation score on rhel7-1: 0
356a11
+native_color: FencingFail allocation score on rhel7-2: 0
356a11
+native_color: FencingFail allocation score on rhel7-4: 0
356a11
+native_color: FencingFail allocation score on rhel7-5: 0
356a11
+native_color: lsb-dummy allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: lsb-dummy allocation score on rhel7-1: -INFINITY
356a11
+native_color: lsb-dummy allocation score on rhel7-2: 0
356a11
+native_color: lsb-dummy allocation score on rhel7-4: -INFINITY
356a11
+native_color: lsb-dummy allocation score on rhel7-5: -INFINITY
356a11
+native_color: migrator allocation score on remote-rhel7-3: 0
356a11
+native_color: migrator allocation score on rhel7-1: 0
356a11
+native_color: migrator allocation score on rhel7-2: 0
356a11
+native_color: migrator allocation score on rhel7-4: 0
356a11
+native_color: migrator allocation score on rhel7-5: 1
356a11
+native_color: petulant allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: petulant allocation score on rhel7-1: -INFINITY
356a11
+native_color: petulant allocation score on rhel7-2: 0
356a11
+native_color: petulant allocation score on rhel7-4: -INFINITY
356a11
+native_color: petulant allocation score on rhel7-5: -INFINITY
356a11
+native_color: ping-1:0 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: ping-1:0 allocation score on rhel7-1: 1
356a11
+native_color: ping-1:0 allocation score on rhel7-2: 0
356a11
+native_color: ping-1:0 allocation score on rhel7-4: 0
356a11
+native_color: ping-1:0 allocation score on rhel7-5: 0
356a11
+native_color: ping-1:1 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: ping-1:1 allocation score on rhel7-1: -INFINITY
356a11
+native_color: ping-1:1 allocation score on rhel7-2: 1
356a11
+native_color: ping-1:1 allocation score on rhel7-4: 0
356a11
+native_color: ping-1:1 allocation score on rhel7-5: 0
356a11
+native_color: ping-1:2 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: ping-1:2 allocation score on rhel7-1: -INFINITY
356a11
+native_color: ping-1:2 allocation score on rhel7-2: -INFINITY
356a11
+native_color: ping-1:2 allocation score on rhel7-4: 1
356a11
+native_color: ping-1:2 allocation score on rhel7-5: 0
356a11
+native_color: ping-1:3 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: ping-1:3 allocation score on rhel7-1: -INFINITY
356a11
+native_color: ping-1:3 allocation score on rhel7-2: -INFINITY
356a11
+native_color: ping-1:3 allocation score on rhel7-4: -INFINITY
356a11
+native_color: ping-1:3 allocation score on rhel7-5: 1
356a11
+native_color: ping-1:4 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: ping-1:4 allocation score on rhel7-1: -INFINITY
356a11
+native_color: ping-1:4 allocation score on rhel7-2: -INFINITY
356a11
+native_color: ping-1:4 allocation score on rhel7-4: -INFINITY
356a11
+native_color: ping-1:4 allocation score on rhel7-5: -INFINITY
356a11
+native_color: r192.168.122.207 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: r192.168.122.207 allocation score on rhel7-1: -INFINITY
356a11
+native_color: r192.168.122.207 allocation score on rhel7-2: 11
356a11
+native_color: r192.168.122.207 allocation score on rhel7-4: -INFINITY
356a11
+native_color: r192.168.122.207 allocation score on rhel7-5: -INFINITY
356a11
+native_color: r192.168.122.208 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: r192.168.122.208 allocation score on rhel7-1: -INFINITY
356a11
+native_color: r192.168.122.208 allocation score on rhel7-2: 0
356a11
+native_color: r192.168.122.208 allocation score on rhel7-4: -INFINITY
356a11
+native_color: r192.168.122.208 allocation score on rhel7-5: -INFINITY
356a11
+native_color: remote-rhel7-3 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: remote-rhel7-3 allocation score on rhel7-1: -INFINITY
356a11
+native_color: remote-rhel7-3 allocation score on rhel7-2: -INFINITY
356a11
+native_color: remote-rhel7-3 allocation score on rhel7-4: -INFINITY
356a11
+native_color: remote-rhel7-3 allocation score on rhel7-5: -INFINITY
356a11
+native_color: remote-rsc allocation score on remote-rhel7-3: INFINITY
356a11
+native_color: remote-rsc allocation score on rhel7-1: 0
356a11
+native_color: remote-rsc allocation score on rhel7-2: 0
356a11
+native_color: remote-rsc allocation score on rhel7-4: 0
356a11
+native_color: remote-rsc allocation score on rhel7-5: 0
356a11
+native_color: rsc_rhel7-1 allocation score on remote-rhel7-3: 0
356a11
+native_color: rsc_rhel7-1 allocation score on rhel7-1: 100
356a11
+native_color: rsc_rhel7-1 allocation score on rhel7-2: 0
356a11
+native_color: rsc_rhel7-1 allocation score on rhel7-4: 0
356a11
+native_color: rsc_rhel7-1 allocation score on rhel7-5: 0
356a11
+native_color: rsc_rhel7-2 allocation score on remote-rhel7-3: 0
356a11
+native_color: rsc_rhel7-2 allocation score on rhel7-1: 0
356a11
+native_color: rsc_rhel7-2 allocation score on rhel7-2: 100
356a11
+native_color: rsc_rhel7-2 allocation score on rhel7-4: 0
356a11
+native_color: rsc_rhel7-2 allocation score on rhel7-5: 0
356a11
+native_color: rsc_rhel7-3 allocation score on remote-rhel7-3: 0
356a11
+native_color: rsc_rhel7-3 allocation score on rhel7-1: 0
356a11
+native_color: rsc_rhel7-3 allocation score on rhel7-2: 0
356a11
+native_color: rsc_rhel7-3 allocation score on rhel7-4: 0
356a11
+native_color: rsc_rhel7-3 allocation score on rhel7-5: 0
356a11
+native_color: rsc_rhel7-4 allocation score on remote-rhel7-3: 0
356a11
+native_color: rsc_rhel7-4 allocation score on rhel7-1: 0
356a11
+native_color: rsc_rhel7-4 allocation score on rhel7-2: 0
356a11
+native_color: rsc_rhel7-4 allocation score on rhel7-4: 100
356a11
+native_color: rsc_rhel7-4 allocation score on rhel7-5: 0
356a11
+native_color: rsc_rhel7-5 allocation score on remote-rhel7-3: 0
356a11
+native_color: rsc_rhel7-5 allocation score on rhel7-1: 0
356a11
+native_color: rsc_rhel7-5 allocation score on rhel7-2: 0
356a11
+native_color: rsc_rhel7-5 allocation score on rhel7-4: 0
356a11
+native_color: rsc_rhel7-5 allocation score on rhel7-5: 100
356a11
+native_color: stateful-1:0 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: stateful-1:0 allocation score on rhel7-1: 6
356a11
+native_color: stateful-1:0 allocation score on rhel7-2: -INFINITY
356a11
+native_color: stateful-1:0 allocation score on rhel7-4: 0
356a11
+native_color: stateful-1:0 allocation score on rhel7-5: 0
356a11
+native_color: stateful-1:1 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: stateful-1:1 allocation score on rhel7-1: 0
356a11
+native_color: stateful-1:1 allocation score on rhel7-2: 11
356a11
+native_color: stateful-1:1 allocation score on rhel7-4: 0
356a11
+native_color: stateful-1:1 allocation score on rhel7-5: 0
356a11
+native_color: stateful-1:2 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: stateful-1:2 allocation score on rhel7-1: -INFINITY
356a11
+native_color: stateful-1:2 allocation score on rhel7-2: -INFINITY
356a11
+native_color: stateful-1:2 allocation score on rhel7-4: 6
356a11
+native_color: stateful-1:2 allocation score on rhel7-5: 0
356a11
+native_color: stateful-1:3 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: stateful-1:3 allocation score on rhel7-1: -INFINITY
356a11
+native_color: stateful-1:3 allocation score on rhel7-2: -INFINITY
356a11
+native_color: stateful-1:3 allocation score on rhel7-4: -INFINITY
356a11
+native_color: stateful-1:3 allocation score on rhel7-5: 6
356a11
+native_color: stateful-1:4 allocation score on remote-rhel7-3: -INFINITY
356a11
+native_color: stateful-1:4 allocation score on rhel7-1: -INFINITY
356a11
+native_color: stateful-1:4 allocation score on rhel7-2: -INFINITY
356a11
+native_color: stateful-1:4 allocation score on rhel7-4: -INFINITY
356a11
+native_color: stateful-1:4 allocation score on rhel7-5: -INFINITY
356a11
+stateful-1:0 promotion score on rhel7-1: 5
356a11
+stateful-1:1 promotion score on rhel7-2: 10
356a11
+stateful-1:2 promotion score on rhel7-4: 5
356a11
+stateful-1:3 promotion score on rhel7-5: 5
356a11
+stateful-1:4 promotion score on none: 0
356a11
diff --git a/pengine/test10/remote-reconnect-delay.summary b/pengine/test10/remote-reconnect-delay.summary
356a11
new file mode 100644
356a11
index 0000000..ea11483
356a11
--- /dev/null
356a11
+++ b/pengine/test10/remote-reconnect-delay.summary
356a11
@@ -0,0 +1,66 @@
356a11
+Using the original execution date of: 2017-08-21 17:12:54Z
356a11
+
356a11
+Current cluster status:
356a11
+Online: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ]
356a11
+RemoteOFFLINE: [ remote-rhel7-3 ]
356a11
+
356a11
+ Fencing	(stonith:fence_xvm):	Started rhel7-2
356a11
+ FencingFail	(stonith:fence_dummy):	Started rhel7-4
356a11
+ rsc_rhel7-1	(ocf::heartbeat:IPaddr2):	Started rhel7-1
356a11
+ rsc_rhel7-2	(ocf::heartbeat:IPaddr2):	Started rhel7-2
356a11
+ rsc_rhel7-3	(ocf::heartbeat:IPaddr2):	Started rhel7-5
356a11
+ rsc_rhel7-4	(ocf::heartbeat:IPaddr2):	Started rhel7-4
356a11
+ rsc_rhel7-5	(ocf::heartbeat:IPaddr2):	Started rhel7-5
356a11
+ migrator	(ocf::pacemaker:Dummy):	Started rhel7-5
356a11
+ Clone Set: Connectivity [ping-1]
356a11
+     Started: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ]
356a11
+     Stopped: [ remote-rhel7-3 ]
356a11
+ Master/Slave Set: master-1 [stateful-1]
356a11
+     Masters: [ rhel7-2 ]
356a11
+     Slaves: [ rhel7-1 rhel7-4 rhel7-5 ]
356a11
+     Stopped: [ remote-rhel7-3 ]
356a11
+ Resource Group: group-1
356a11
+     r192.168.122.207	(ocf::heartbeat:IPaddr2):	Started rhel7-2
356a11
+     petulant	(service:DummySD):	Started rhel7-2
356a11
+     r192.168.122.208	(ocf::heartbeat:IPaddr2):	Started rhel7-2
356a11
+ lsb-dummy	(lsb:/usr/share/pacemaker/tests/cts/LSBDummy):	Started rhel7-2
356a11
+ remote-rhel7-3	(ocf::pacemaker:remote):	FAILED
356a11
+ remote-rsc	(ocf::heartbeat:Dummy):	Started rhel7-1
356a11
+
356a11
+Transition Summary:
356a11
+ * Restart    Fencing     ( rhel7-2 )  
356a11
+
356a11
+Executing cluster transition:
356a11
+ * Resource action: Fencing         stop on rhel7-2
356a11
+ * Resource action: Fencing         start on rhel7-2
356a11
+ * Resource action: Fencing         monitor=120000 on rhel7-2
356a11
+ * Pseudo action:   all_stopped
356a11
+Using the original execution date of: 2017-08-21 17:12:54Z
356a11
+
356a11
+Revised cluster status:
356a11
+Online: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ]
356a11
+RemoteOFFLINE: [ remote-rhel7-3 ]
356a11
+
356a11
+ Fencing	(stonith:fence_xvm):	Started rhel7-2
356a11
+ FencingFail	(stonith:fence_dummy):	Started rhel7-4
356a11
+ rsc_rhel7-1	(ocf::heartbeat:IPaddr2):	Started rhel7-1
356a11
+ rsc_rhel7-2	(ocf::heartbeat:IPaddr2):	Started rhel7-2
356a11
+ rsc_rhel7-3	(ocf::heartbeat:IPaddr2):	Started rhel7-5
356a11
+ rsc_rhel7-4	(ocf::heartbeat:IPaddr2):	Started rhel7-4
356a11
+ rsc_rhel7-5	(ocf::heartbeat:IPaddr2):	Started rhel7-5
356a11
+ migrator	(ocf::pacemaker:Dummy):	Started rhel7-5
356a11
+ Clone Set: Connectivity [ping-1]
356a11
+     Started: [ rhel7-1 rhel7-2 rhel7-4 rhel7-5 ]
356a11
+     Stopped: [ remote-rhel7-3 ]
356a11
+ Master/Slave Set: master-1 [stateful-1]
356a11
+     Masters: [ rhel7-2 ]
356a11
+     Slaves: [ rhel7-1 rhel7-4 rhel7-5 ]
356a11
+     Stopped: [ remote-rhel7-3 ]
356a11
+ Resource Group: group-1
356a11
+     r192.168.122.207	(ocf::heartbeat:IPaddr2):	Started rhel7-2
356a11
+     petulant	(service:DummySD):	Started rhel7-2
356a11
+     r192.168.122.208	(ocf::heartbeat:IPaddr2):	Started rhel7-2
356a11
+ lsb-dummy	(lsb:/usr/share/pacemaker/tests/cts/LSBDummy):	Started rhel7-2
356a11
+ remote-rhel7-3	(ocf::pacemaker:remote):	FAILED
356a11
+ remote-rsc	(ocf::heartbeat:Dummy):	Started rhel7-1
356a11
+
356a11
diff --git a/pengine/test10/remote-reconnect-delay.xml b/pengine/test10/remote-reconnect-delay.xml
356a11
new file mode 100644
356a11
index 0000000..e9ed3e6
356a11
--- /dev/null
356a11
+++ b/pengine/test10/remote-reconnect-delay.xml
356a11
@@ -0,0 +1,504 @@
356a11
+<cib crm_feature_set="3.0.14" validate-with="pacemaker-2.5" epoch="49" num_updates="13" admin_epoch="0" cib-last-written="Mon Aug 21 12:12:00 2017" update-origin="rhel7-1" update-client="crm_resource" update-user="remote-rhel7-3" have-quorum="1" dc-uuid="2" execution-date="1503335574">
356a11
+  <configuration>
356a11
+    <crm_config>
356a11
+      <cluster_property_set id="cib-bootstrap-options">
356a11
+        <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
356a11
+        <nvpair id="cts-start-failure-is-fatal" name="start-failure-is-fatal" value="false"/>
356a11
+        <nvpair id="cts-pe-input-series-max" name="pe-input-series-max" value="5000"/>
356a11
+        <nvpair id="cts-default-action-timeout" name="default-action-timeout" value="90s"/>
356a11
+        <nvpair id="cts-shutdown-escalation" name="shutdown-escalation" value="5min"/>
356a11
+        <nvpair id="cts-batch-limit" name="batch-limit" value="10"/>
356a11
+        <nvpair id="cts-dc-deadtime" name="dc-deadtime" value="5s"/>
356a11
+        <nvpair id="cts-no-quorum-policy" name="no-quorum-policy" value="stop"/>
356a11
+        <nvpair id="cts-expected-quorum-votes" name="expected-quorum-votes" value="5"/>
356a11
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
356a11
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.17-553.368fb3e.git.el7.centos-368fb3e"/>
356a11
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
356a11
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/>
356a11
+        <nvpair id="cts-recheck-interval-setting" name="cluster-recheck-interval" value="45s"/>
356a11
+      </cluster_property_set>
356a11
+    </crm_config>
356a11
+    <nodes>
356a11
+      <node id="4" uname="rhel7-4"/>
356a11
+      <node id="5" uname="rhel7-5"/>
356a11
+      <node id="3" uname="rhel7-1"/>
356a11
+      <node id="2" uname="rhel7-2"/>
356a11
+    </nodes>
356a11
+    <resources>
356a11
+      <primitive id="Fencing" class="stonith" type="fence_xvm">
356a11
+        <meta_attributes id="Fencing-meta">
356a11
+          <nvpair id="Fencing-migration-threshold" name="migration-threshold" value="5"/>
356a11
+        </meta_attributes>
356a11
+        <instance_attributes id="Fencing-params">
356a11
+          <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
356a11
+          <nvpair id="Fencing-pcmk_arg_map" name="pcmk_arg_map" value="domain:uname"/>
356a11
+          <nvpair id="Fencing-pcmk_host_map" name="pcmk_host_map" value="remote-rhel7-1:rhel7-1;remote-rhel7-2:rhel7-2;remote-rhel7-3:rhel7-3;remote-rhel7-4:rhel7-4;remote-rhel7-5:rhel7-5;"/>
356a11
+          <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 remote-rhel7-1 rhel7-2 remote-rhel7-2 remote-rhel7-3 rhel7-4 remote-rhel7-4 rhel7-5 remote-rhel7-5"/>
356a11
+        </instance_attributes>
356a11
+        <operations>
356a11
+          <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
356a11
+          <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/>
356a11
+          <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/>
356a11
+        </operations>
356a11
+      </primitive>
356a11
+      <primitive id="FencingFail" class="stonith" type="fence_dummy">
356a11
+        <instance_attributes id="FencingFail-params">
356a11
+          <nvpair id="FencingFail-random_sleep_range" name="random_sleep_range" value="30"/>
356a11
+          <nvpair id="FencingFail-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 remote-rhel7-1 rhel7-4 remote-rhel7-4"/>
356a11
+          <nvpair id="FencingFail-mode" name="mode" value="fail"/>
356a11
+        </instance_attributes>
356a11
+      </primitive>
356a11
+      <primitive id="rsc_rhel7-1" class="ocf" type="IPaddr2" provider="heartbeat">
356a11
+        <instance_attributes id="rsc_rhel7-1-params">
356a11
+          <nvpair id="rsc_rhel7-1-ip" name="ip" value="192.168.122.202"/>
356a11
+          <nvpair id="rsc_rhel7-1-cidr_netmask" name="cidr_netmask" value="32"/>
356a11
+        </instance_attributes>
356a11
+        <operations>
356a11
+          <op id="rsc_rhel7-1-monitor-5s" interval="5s" name="monitor"/>
356a11
+        </operations>
356a11
+      </primitive>
356a11
+      <primitive id="rsc_rhel7-2" class="ocf" type="IPaddr2" provider="heartbeat">
356a11
+        <instance_attributes id="rsc_rhel7-2-params">
356a11
+          <nvpair id="rsc_rhel7-2-ip" name="ip" value="192.168.122.203"/>
356a11
+          <nvpair id="rsc_rhel7-2-cidr_netmask" name="cidr_netmask" value="32"/>
356a11
+        </instance_attributes>
356a11
+        <operations>
356a11
+          <op id="rsc_rhel7-2-monitor-5s" interval="5s" name="monitor"/>
356a11
+        </operations>
356a11
+      </primitive>
356a11
+      <primitive id="rsc_rhel7-3" class="ocf" type="IPaddr2" provider="heartbeat">
356a11
+        <instance_attributes id="rsc_rhel7-3-params">
356a11
+          <nvpair id="rsc_rhel7-3-ip" name="ip" value="192.168.122.204"/>
356a11
+          <nvpair id="rsc_rhel7-3-cidr_netmask" name="cidr_netmask" value="32"/>
356a11
+        </instance_attributes>
356a11
+        <operations>
356a11
+          <op id="rsc_rhel7-3-monitor-5s" interval="5s" name="monitor"/>
356a11
+        </operations>
356a11
+      </primitive>
356a11
+      <primitive id="rsc_rhel7-4" class="ocf" type="IPaddr2" provider="heartbeat">
356a11
+        <instance_attributes id="rsc_rhel7-4-params">
356a11
+          <nvpair id="rsc_rhel7-4-ip" name="ip" value="192.168.122.205"/>
356a11
+          <nvpair id="rsc_rhel7-4-cidr_netmask" name="cidr_netmask" value="32"/>
356a11
+        </instance_attributes>
356a11
+        <operations>
356a11
+          <op id="rsc_rhel7-4-monitor-5s" interval="5s" name="monitor"/>
356a11
+        </operations>
356a11
+      </primitive>
356a11
+      <primitive id="rsc_rhel7-5" class="ocf" type="IPaddr2" provider="heartbeat">
356a11
+        <instance_attributes id="rsc_rhel7-5-params">
356a11
+          <nvpair id="rsc_rhel7-5-ip" name="ip" value="192.168.122.206"/>
356a11
+          <nvpair id="rsc_rhel7-5-cidr_netmask" name="cidr_netmask" value="32"/>
356a11
+        </instance_attributes>
356a11
+        <operations>
356a11
+          <op id="rsc_rhel7-5-monitor-5s" interval="5s" name="monitor"/>
356a11
+        </operations>
356a11
+      </primitive>
356a11
+      <primitive id="migrator" class="ocf" type="Dummy" provider="pacemaker">
356a11
+        <meta_attributes id="migrator-meta">
356a11
+          <nvpair id="migrator-allow-migrate" name="allow-migrate" value="1"/>
356a11
+          <nvpair id="migrator-resource-stickiness" name="resource-stickiness" value="1"/>
356a11
+        </meta_attributes>
356a11
+        <instance_attributes id="migrator-params">
356a11
+          <nvpair id="migrator-passwd" name="passwd" value="whatever"/>
356a11
+        </instance_attributes>
356a11
+        <operations>
356a11
+          <op id="migrator-monitor-P10S" interval="P10S" name="monitor"/>
356a11
+        </operations>
356a11
+      </primitive>
356a11
+      <clone id="Connectivity">
356a11
+        <meta_attributes id="Connectivity-meta">
356a11
+          <nvpair id="Connectivity-globally-unique" name="globally-unique" value="false"/>
356a11
+        </meta_attributes>
356a11
+        <primitive id="ping-1" class="ocf" type="ping" provider="pacemaker">
356a11
+          <instance_attributes id="ping-1-params">
356a11
+            <nvpair id="ping-1-debug" name="debug" value="true"/>
356a11
+            <nvpair id="ping-1-host_list" name="host_list" value="192.168.122.70"/>
356a11
+            <nvpair id="ping-1-name" name="name" value="connected"/>
356a11
+          </instance_attributes>
356a11
+          <operations>
356a11
+            <op id="ping-1-monitor-60s" interval="60s" name="monitor"/>
356a11
+          </operations>
356a11
+        </primitive>
356a11
+      </clone>
356a11
+      <master id="master-1">
356a11
+        <meta_attributes id="master-1-meta">
356a11
+          <nvpair id="master-1-master-node-max" name="master-node-max" value="1"/>
356a11
+          <nvpair id="master-1-clone-max" name="clone-max" value="5"/>
356a11
+          <nvpair id="master-1-master-max" name="master-max" value="1"/>
356a11
+          <nvpair id="master-1-clone-node-max" name="clone-node-max" value="1"/>
356a11
+        </meta_attributes>
356a11
+        <primitive id="stateful-1" class="ocf" type="Stateful" provider="pacemaker">
356a11
+          <operations>
356a11
+            <op id="stateful-1-monitor-15s" interval="15s" name="monitor" timeout="60s"/>
356a11
+            <op id="stateful-1-monitor-16s" interval="16s" role="Master" name="monitor" timeout="60s"/>
356a11
+          </operations>
356a11
+        </primitive>
356a11
+      </master>
356a11
+      <group id="group-1">
356a11
+        <primitive id="r192.168.122.207" class="ocf" type="IPaddr2" provider="heartbeat">
356a11
+          <instance_attributes id="r192.168.122.207-params">
356a11
+            <nvpair id="r192.168.122.207-ip" name="ip" value="192.168.122.207"/>
356a11
+            <nvpair id="r192.168.122.207-cidr_netmask" name="cidr_netmask" value="32"/>
356a11
+          </instance_attributes>
356a11
+          <operations>
356a11
+            <op id="r192.168.122.207-monitor-5s" interval="5s" name="monitor"/>
356a11
+          </operations>
356a11
+        </primitive>
356a11
+        <primitive id="petulant" class="service" type="DummySD">
356a11
+          <operations>
356a11
+            <op id="petulant-monitor-P10S" interval="P10S" name="monitor"/>
356a11
+          </operations>
356a11
+        </primitive>
356a11
+        <primitive id="r192.168.122.208" class="ocf" type="IPaddr2" provider="heartbeat">
356a11
+          <instance_attributes id="r192.168.122.208-params">
356a11
+            <nvpair id="r192.168.122.208-ip" name="ip" value="192.168.122.208"/>
356a11
+            <nvpair id="r192.168.122.208-cidr_netmask" name="cidr_netmask" value="32"/>
356a11
+          </instance_attributes>
356a11
+          <operations>
356a11
+            <op id="r192.168.122.208-monitor-5s" interval="5s" name="monitor"/>
356a11
+          </operations>
356a11
+        </primitive>
356a11
+      </group>
356a11
+      <primitive id="lsb-dummy" class="lsb" type="/usr/share/pacemaker/tests/cts/LSBDummy">
356a11
+        <operations>
356a11
+          <op id="lsb-dummy-monitor-5s" interval="5s" name="monitor"/>
356a11
+        </operations>
356a11
+      </primitive>
356a11
+      <primitive class="ocf" id="remote-rhel7-3" provider="pacemaker" type="remote">
356a11
+        <instance_attributes id="remote-instance_attributes">
356a11
+          <nvpair id="remote-instance_attributes-server" name="server" value="rhel7-3"/>
356a11
+          <nvpair id="remote-instance_attributes-reconnect_interval" name="reconnect_interval" value="60s"/>
356a11
+        </instance_attributes>
356a11
+        <operations>
356a11
+          <op id="remote-monitor-interval-60s" interval="60s" name="monitor"/>
356a11
+          <op id="remote-name-start-interval-0-timeout-120" interval="0" name="start" timeout="60"/>
356a11
+        </operations>
356a11
+      </primitive>
356a11
+      <primitive class="ocf" id="remote-rsc" provider="heartbeat" type="Dummy">
356a11
+        <operations>
356a11
+          <op id="remote-rsc-monitor-interval-10s" interval="10s" name="monitor"/>
356a11
+        </operations>
356a11
+        <meta_attributes id="remote-meta_attributes"/>
356a11
+      </primitive>
356a11
+    </resources>
356a11
+    <constraints>
356a11
+      <rsc_location id="prefer-rhel7-1" rsc="rsc_rhel7-1">
356a11
+        <rule id="prefer-rhel7-1-r" score="100" boolean-op="and">
356a11
+          <expression id="prefer-rhel7-1-e" attribute="#uname" operation="eq" value="rhel7-1"/>
356a11
+        </rule>
356a11
+      </rsc_location>
356a11
+      <rsc_location id="prefer-rhel7-2" rsc="rsc_rhel7-2">
356a11
+        <rule id="prefer-rhel7-2-r" score="100" boolean-op="and">
356a11
+          <expression id="prefer-rhel7-2-e" attribute="#uname" operation="eq" value="rhel7-2"/>
356a11
+        </rule>
356a11
+      </rsc_location>
356a11
+      <rsc_location id="prefer-rhel7-4" rsc="rsc_rhel7-4">
356a11
+        <rule id="prefer-rhel7-4-r" score="100" boolean-op="and">
356a11
+          <expression id="prefer-rhel7-4-e" attribute="#uname" operation="eq" value="rhel7-4"/>
356a11
+        </rule>
356a11
+      </rsc_location>
356a11
+      <rsc_location id="prefer-rhel7-5" rsc="rsc_rhel7-5">
356a11
+        <rule id="prefer-rhel7-5-r" score="100" boolean-op="and">
356a11
+          <expression id="prefer-rhel7-5-e" attribute="#uname" operation="eq" value="rhel7-5"/>
356a11
+        </rule>
356a11
+      </rsc_location>
356a11
+      <rsc_location id="prefer-connected" rsc="master-1">
356a11
+        <rule id="connected" score="-INFINITY" boolean-op="or">
356a11
+          <expression id="m1-connected-1" attribute="connected" operation="lt" value="1"/>
356a11
+          <expression id="m1-connected-2" attribute="connected" operation="not_defined"/>
356a11
+        </rule>
356a11
+      </rsc_location>
356a11
+      <rsc_order id="group-1-after-master-1" first="master-1" then="group-1" kind="Mandatory" first-action="promote" then-action="start"/>
356a11
+      <rsc_colocation id="group-1-with-master-1" rsc="group-1" with-rsc="master-1" score="INFINITY" with-rsc-role="Master"/>
356a11
+      <rsc_order id="lsb-dummy-after-group-1" first="group-1" then="lsb-dummy" kind="Mandatory" first-action="start" then-action="start"/>
356a11
+      <rsc_colocation id="lsb-dummy-with-group-1" rsc="lsb-dummy" with-rsc="group-1" score="INFINITY"/>
356a11
+      <rsc_location id="cli-prefer-remote-rsc" rsc="remote-rsc" role="Started" node="remote-rhel7-3" score="INFINITY"/>
356a11
+    </constraints>
356a11
+    <fencing-topology>
356a11
+      <fencing-level id="cts-rhel7-1.1" index="1" target="rhel7-1" devices="FencingFail"/>
356a11
+      <fencing-level id="cts-rhel7-1.2" index="2" target="rhel7-1" devices="Fencing"/>
356a11
+      <fencing-level id="cts-remote-rhel7-1.1" index="1" target="remote-rhel7-1" devices="FencingFail"/>
356a11
+      <fencing-level id="cts-remote-rhel7-1.2" index="2" target="remote-rhel7-1" devices="Fencing"/>
356a11
+      <fencing-level id="cts-rhel7-4.1" index="1" target="rhel7-4" devices="FencingFail"/>
356a11
+      <fencing-level id="cts-rhel7-4.2" index="2" target="rhel7-4" devices="Fencing"/>
356a11
+      <fencing-level id="cts-remote-rhel7-4.1" index="1" target="remote-rhel7-4" devices="FencingFail"/>
356a11
+      <fencing-level id="cts-remote-rhel7-4.2" index="2" target="remote-rhel7-4" devices="Fencing"/>
356a11
+    </fencing-topology>
356a11
+    <alerts>
356a11
+      <alert id="alert-1" path="/var/lib/pacemaker/notify.sh">
356a11
+        <recipient id="alert-1-recipient-1" value="/run/crm/alert.log"/>
356a11
+      </alert>
356a11
+    </alerts>
356a11
+  </configuration>
356a11
+  <status>
356a11
+    <node_state id="3" uname="rhel7-1" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
356a11
+      <transient_attributes id="3">
356a11
+        <instance_attributes id="status-3">
356a11
+          <nvpair id="status-3-connected" name="connected" value="1"/>
356a11
+          <nvpair id="status-3-master-stateful-1" name="master-stateful-1" value="5"/>
356a11
+          <nvpair id="status-3-fail-count-remote-rhel7-3.monitor_60000" name="fail-count-remote-rhel7-3#monitor_60000" value="1"/>
356a11
+          <nvpair id="status-3-last-failure-remote-rhel7-3.monitor_60000" name="last-failure-remote-rhel7-3#monitor_60000" value="1503335523"/>
356a11
+        </instance_attributes>
356a11
+      </transient_attributes>
356a11
+      <lrm id="3">
356a11
+        <lrm_resources>
356a11
+          <lrm_resource id="r192.168.122.207" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="r192.168.122.207_last_0" operation_key="r192.168.122.207_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="31:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;31:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="54" rc-code="7" op-status="0" interval="0" last-run="1503335089" last-rc-change="1503335089" exec-time="155" queue-time="0" op-digest="455141de0d85faf791392b0857f9dea1" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="lsb-dummy" type="/usr/share/pacemaker/tests/cts/LSBDummy" class="lsb">
356a11
+            <lrm_rsc_op id="lsb-dummy_last_0" operation_key="lsb-dummy_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="34:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;34:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="67" rc-code="7" op-status="0" interval="0" last-run="1503335089" last-rc-change="1503335089" exec-time="21" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="r192.168.122.208" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="r192.168.122.208_last_0" operation_key="r192.168.122.208_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="33:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;33:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="63" rc-code="7" op-status="0" interval="0" last-run="1503335089" last-rc-change="1503335089" exec-time="67" queue-time="0" op-digest="d62599e347d2c3a524c13e135846a774" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="FencingFail" type="fence_dummy" class="stonith">
356a11
+            <lrm_rsc_op id="FencingFail_last_0" operation_key="FencingFail_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="39:1:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;39:1:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-1" call-id="143" rc-code="0" op-status="0" interval="0" last-run="1503335297" last-rc-change="1503335297" exec-time="2" queue-time="0" op-digest="9401ff83703cb25678335741cf0bca04" ra-version="4.0.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
356a11
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="37:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;37:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-1" call-id="192" rc-code="0" op-status="0" interval="0" last-run="1503335492" last-rc-change="1503335492" exec-time="12" queue-time="0" op-digest="2409d7001df14f35973634911d9875b1" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="25:20:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;25:20:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-1" call-id="188" rc-code="0" op-status="0" interval="120000" last-rc-change="1503335472" exec-time="5" queue-time="0" op-digest="87e7768825db29d87bc1f2119ebffe0e" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="petulant" type="DummySD" class="service">
356a11
+            <lrm_rsc_op id="petulant_last_0" operation_key="petulant_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="32:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;32:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="59" rc-code="7" op-status="0" interval="0" last-run="1503335089" last-rc-change="1503335089" exec-time="62" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="ping-1" type="ping" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="ping-1_last_0" operation_key="ping-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="60:16:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;60:16:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="45" rc-code="0" op-status="0" interval="0" last-run="1503335084" last-rc-change="1503335084" exec-time="2064" queue-time="0" op-digest="029004076c53d493cd6e9f6661b1b083" ra-version="1.0"/>
356a11
+            <lrm_rsc_op id="ping-1_monitor_60000" operation_key="ping-1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="61:16:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;61:16:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="47" rc-code="0" op-status="0" interval="60000" last-rc-change="1503335086" exec-time="2103" queue-time="0" op-digest="8654ff74c1be19c9385009b41becc0c8" ra-version="1.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="stateful-1" type="Stateful" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="stateful-1_last_0" operation_key="stateful-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="62:17:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;62:17:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="68" rc-code="0" op-status="0" interval="0" last-run="1503335091" last-rc-change="1503335091" exec-time="61" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" ra-version="1.0"/>
356a11
+            <lrm_rsc_op id="stateful-1_monitor_15000" operation_key="stateful-1_monitor_15000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="54:18:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;54:18:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="70" rc-code="0" op-status="0" interval="15000" last-rc-change="1503335092" exec-time="14" queue-time="0" op-digest="873ed4f07792aa8ff18f3254244675ea" ra-version="1.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-1" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-1_last_0" operation_key="rsc_rhel7-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="40:16:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;40:16:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="34" rc-code="0" op-status="0" interval="0" last-run="1503335084" last-rc-change="1503335084" exec-time="4160" queue-time="0" op-digest="75df1567eb9457f8f3c4486bbf875846" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="rsc_rhel7-1_monitor_5000" operation_key="rsc_rhel7-1_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="41:16:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;41:16:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="49" rc-code="0" op-status="0" interval="5000" last-rc-change="1503335088" exec-time="188" queue-time="0" op-digest="045c3d6f5e29b94dc4e3fbfd6c2c0693" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-2" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-2_last_0" operation_key="rsc_rhel7-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="24:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;24:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="17" rc-code="7" op-status="0" interval="0" last-run="1503335084" last-rc-change="1503335084" exec-time="162" queue-time="0" op-digest="ced6f8a1916ebbe555cedafe69985e63" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-3" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-3_last_0" operation_key="rsc_rhel7-3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="25:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;25:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="21" rc-code="7" op-status="0" interval="0" last-run="1503335084" last-rc-change="1503335084" exec-time="148" queue-time="0" op-digest="3a5f279381f73d4be861526d72bb17a3" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-4" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-4_last_0" operation_key="rsc_rhel7-4_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="26:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;26:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="25" rc-code="7" op-status="0" interval="0" last-run="1503335084" last-rc-change="1503335084" exec-time="127" queue-time="0" op-digest="b4b6b30b67042d5bc4c1735b0df27dc0" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-5" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-5_last_0" operation_key="rsc_rhel7-5_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="47:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;47:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="137" rc-code="0" op-status="0" interval="0" last-run="1503335234" last-rc-change="1503335234" exec-time="72" queue-time="0" op-digest="aca525581410dfda70831f2846b9807d"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="migrator" type="Dummy" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="migrator_last_0" operation_key="migrator_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="28:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;28:16:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-1" call-id="33" rc-code="7" op-status="0" interval="0" last-run="1503335084" last-rc-change="1503335084" exec-time="54" queue-time="0" op-digest="5de129d7fe42dbcfe537f2c63b1921b6" ra-version="1.0" op-force-restart=" fail_start_on  fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="5de129d7fe42dbcfe537f2c63b1921b6"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="remote-rhel7-3" type="remote" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="remote-rhel7-3_last_0" operation_key="remote-rhel7-3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="6:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;6:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-1" call-id="18" rc-code="0" op-status="0" interval="0" last-run="1503335525" last-rc-change="1503335525" exec-time="0" queue-time="0" op-digest="261b4fde9eb7965580f152a20c775486" ra-version="0.1" op-force-restart=" reconnect_interval  port " op-restart-digest="1dbaa1569a09b2a43b8d2b8037a48749"/>
356a11
+            <lrm_rsc_op id="remote-rhel7-3_monitor_60000" operation_key="remote-rhel7-3_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="96:23:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;96:23:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-1" call-id="17" rc-code="0" op-status="0" interval="60000" last-rc-change="1503335516" exec-time="0" queue-time="0" op-digest="ed1bab57863fa1dec83afb184f98eb57" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="remote-rhel7-3_last_failure_0" operation_key="remote-rhel7-3_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="96:23:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="4:1;96:23:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-1" call-id="17" rc-code="1" op-status="4" interval="60000" last-rc-change="1503335523" exec-time="0" queue-time="0" op-digest="ed1bab57863fa1dec83afb184f98eb57" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="remote-rsc" type="Dummy" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="remote-rsc_last_0" operation_key="remote-rsc_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="85:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;85:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-1" call-id="215" rc-code="0" op-status="0" interval="0" last-run="1503335525" last-rc-change="1503335525" exec-time="10" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" ra-version="0.9" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+            <lrm_rsc_op id="remote-rsc_monitor_10000" operation_key="remote-rsc_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="86:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;86:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-1" call-id="217" rc-code="0" op-status="0" interval="10000" last-rc-change="1503335525" exec-time="11" queue-time="0" op-digest="8f6a313464b7f9e3a31cb448458b700e" ra-version="0.9"/>
356a11
+          </lrm_resource>
356a11
+        </lrm_resources>
356a11
+      </lrm>
356a11
+    </node_state>
356a11
+    <node_state id="2" uname="rhel7-2" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
356a11
+      <transient_attributes id="2">
356a11
+        <instance_attributes id="status-2">
356a11
+          <nvpair id="status-2-connected" name="connected" value="1"/>
356a11
+          <nvpair id="status-2-master-stateful-1" name="master-stateful-1" value="10"/>
356a11
+        </instance_attributes>
356a11
+      </transient_attributes>
356a11
+      <lrm id="2">
356a11
+        <lrm_resources>
356a11
+          <lrm_resource id="r192.168.122.207" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="r192.168.122.207_last_0" operation_key="r192.168.122.207_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="63:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;63:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="87" rc-code="0" op-status="0" interval="0" last-run="1503334913" last-rc-change="1503334913" exec-time="4104" queue-time="0" op-digest="455141de0d85faf791392b0857f9dea1" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="r192.168.122.207_monitor_5000" operation_key="r192.168.122.207_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="64:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;64:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="89" rc-code="0" op-status="0" interval="5000" last-rc-change="1503334917" exec-time="33" queue-time="0" op-digest="e265b034f446ca5c724ff25c223f1078" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="r192.168.122.208" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="r192.168.122.208_last_0" operation_key="r192.168.122.208_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="67:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;67:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="95" rc-code="0" op-status="0" interval="0" last-run="1503334928" last-rc-change="1503334928" exec-time="4067" queue-time="0" op-digest="d62599e347d2c3a524c13e135846a774" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="r192.168.122.208_monitor_5000" operation_key="r192.168.122.208_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="68:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;68:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="97" rc-code="0" op-status="0" interval="5000" last-rc-change="1503334932" exec-time="29" queue-time="0" op-digest="10ff4c567237e36994b5c19f129f6a1e" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="FencingFail" type="fence_dummy" class="stonith">
356a11
+            <lrm_rsc_op id="FencingFail_last_0" operation_key="FencingFail_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="40:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;40:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-2" call-id="176" rc-code="0" op-status="0" interval="0" last-run="1503335492" last-rc-change="1503335492" exec-time="0" queue-time="0" op-digest="9401ff83703cb25678335741cf0bca04" ra-version="4.0.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="lsb-dummy" type="/usr/share/pacemaker/tests/cts/LSBDummy" class="lsb">
356a11
+            <lrm_rsc_op id="lsb-dummy_last_0" operation_key="lsb-dummy_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="73:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;73:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="99" rc-code="0" op-status="0" interval="0" last-run="1503334932" last-rc-change="1503334932" exec-time="4" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+            <lrm_rsc_op id="lsb-dummy_monitor_5000" operation_key="lsb-dummy_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="74:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;74:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="101" rc-code="0" op-status="0" interval="5000" last-rc-change="1503334932" exec-time="4" queue-time="0" op-digest="8f6a313464b7f9e3a31cb448458b700e"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
356a11
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="38:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;38:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-2" call-id="179" rc-code="0" op-status="0" interval="0" last-run="1503335492" last-rc-change="1503335492" exec-time="34" queue-time="0" op-digest="2409d7001df14f35973634911d9875b1" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="39:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;39:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-2" call-id="181" rc-code="0" op-status="0" interval="120000" last-rc-change="1503335492" exec-time="15" queue-time="0" op-digest="87e7768825db29d87bc1f2119ebffe0e" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="petulant" type="DummySD" class="service">
356a11
+            <lrm_rsc_op id="petulant_last_0" operation_key="petulant_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="65:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;65:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="91" rc-code="0" op-status="0" interval="0" last-run="1503334917" last-rc-change="1503334917" exec-time="10195" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+            <lrm_rsc_op id="petulant_monitor_10000" operation_key="petulant_monitor_10000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="66:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;66:0:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="93" rc-code="0" op-status="0" interval="10000" last-rc-change="1503334928" exec-time="2" queue-time="1" op-digest="8f6a313464b7f9e3a31cb448458b700e"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="ping-1" type="ping" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="ping-1_last_0" operation_key="ping-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="79:1:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" transition-magic="0:0;79:1:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" on_node="rhel7-2" call-id="50" rc-code="0" op-status="0" interval="0" last-run="1503334858" last-rc-change="1503334858" exec-time="2073" queue-time="0" op-digest="029004076c53d493cd6e9f6661b1b083" ra-version="1.0"/>
356a11
+            <lrm_rsc_op id="ping-1_monitor_60000" operation_key="ping-1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="80:1:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" transition-magic="0:0;80:1:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" on_node="rhel7-2" call-id="52" rc-code="0" op-status="0" interval="60000" last-rc-change="1503334860" exec-time="2043" queue-time="0" op-digest="8654ff74c1be19c9385009b41becc0c8" ra-version="1.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="stateful-1" type="Stateful" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="stateful-1_last_0" operation_key="stateful-1_promote_0" operation="promote" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="46:11:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" transition-magic="0:0;46:11:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" on_node="rhel7-2" call-id="82" rc-code="0" op-status="0" interval="0" last-run="1503334912" last-rc-change="1503334912" exec-time="27" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" ra-version="1.0"/>
356a11
+            <lrm_rsc_op id="stateful-1_monitor_16000" operation_key="stateful-1_monitor_16000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="48:0:8:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:8;48:0:8:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="85" rc-code="8" op-status="0" interval="16000" last-rc-change="1503334913" exec-time="21" queue-time="0" op-digest="873ed4f07792aa8ff18f3254244675ea" ra-version="1.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-1" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-1_last_0" operation_key="rsc_rhel7-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="4:1:7:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" transition-magic="0:7;4:1:7:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" on_node="rhel7-2" call-id="13" rc-code="7" op-status="0" interval="0" last-run="1503334832" last-rc-change="1503334832" exec-time="101" queue-time="0" op-digest="75df1567eb9457f8f3c4486bbf875846" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-2" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-2_last_0" operation_key="rsc_rhel7-2_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="63:1:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" transition-magic="0:0;63:1:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" on_node="rhel7-2" call-id="20" rc-code="0" op-status="0" interval="0" last-run="1503334841" last-rc-change="1503334841" exec-time="4053" queue-time="0" op-digest="ced6f8a1916ebbe555cedafe69985e63" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="rsc_rhel7-2_monitor_5000" operation_key="rsc_rhel7-2_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="64:1:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" transition-magic="0:0;64:1:0:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" on_node="rhel7-2" call-id="22" rc-code="0" op-status="0" interval="5000" last-rc-change="1503334845" exec-time="34" queue-time="0" op-digest="4385e7bd76844b9bc880a410b317b8ab" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-3" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-3_last_0" operation_key="rsc_rhel7-3_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="6:1:7:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" transition-magic="0:7;6:1:7:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" on_node="rhel7-2" call-id="27" rc-code="7" op-status="0" interval="0" last-run="1503334845" last-rc-change="1503334845" exec-time="29" queue-time="0" op-digest="3a5f279381f73d4be861526d72bb17a3" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-4" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-4_last_0" operation_key="rsc_rhel7-4_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="7:1:7:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" transition-magic="0:7;7:1:7:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" on_node="rhel7-2" call-id="31" rc-code="7" op-status="0" interval="0" last-run="1503334845" last-rc-change="1503334845" exec-time="29" queue-time="0" op-digest="b4b6b30b67042d5bc4c1735b0df27dc0" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-5" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-5_last_0" operation_key="rsc_rhel7-5_stop_0" operation="stop" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="47:29:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;47:29:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-2" call-id="154" rc-code="0" op-status="0" interval="0" last-run="1503335169" last-rc-change="1503335169" exec-time="42" queue-time="0" op-digest="aca525581410dfda70831f2846b9807d"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="migrator" type="Dummy" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="migrator_last_0" operation_key="migrator_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="9:1:7:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" transition-magic="0:7;9:1:7:2dc02a24-b145-46ac-ba2d-ee462fdab7ab" on_node="rhel7-2" call-id="39" rc-code="7" op-status="0" interval="0" last-run="1503334850" last-rc-change="1503334850" exec-time="11" queue-time="0" op-digest="5de129d7fe42dbcfe537f2c63b1921b6" ra-version="1.0" op-force-restart=" fail_start_on  fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="5de129d7fe42dbcfe537f2c63b1921b6"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="remote-rhel7-3" type="remote" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="remote-rhel7-3_last_0" operation_key="remote-rhel7-3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="34:22:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;34:22:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-2" call-id="10" rc-code="7" op-status="0" interval="0" last-run="1503335492" last-rc-change="1503335492" exec-time="0" queue-time="0" op-digest="261b4fde9eb7965580f152a20c775486" ra-version="0.1" op-force-restart=" reconnect_interval  port " op-restart-digest="1dbaa1569a09b2a43b8d2b8037a48749"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="remote-rsc" type="Dummy" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="remote-rsc_last_0" operation_key="remote-rsc_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="26:26:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;26:26:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-2" call-id="187" rc-code="7" op-status="0" interval="0" last-run="1503335519" last-rc-change="1503335519" exec-time="48" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" ra-version="0.9" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+          </lrm_resource>
356a11
+        </lrm_resources>
356a11
+      </lrm>
356a11
+    </node_state>
356a11
+    <node_state id="4" uname="rhel7-4" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
356a11
+      <lrm id="4">
356a11
+        <lrm_resources>
356a11
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
356a11
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="21:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;21:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1503335449" last-rc-change="1503335449" exec-time="2" queue-time="0" op-digest="2409d7001df14f35973634911d9875b1" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="FencingFail" type="fence_dummy" class="stonith">
356a11
+            <lrm_rsc_op id="FencingFail_last_0" operation_key="FencingFail_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="41:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;41:22:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="78" rc-code="0" op-status="0" interval="0" last-run="1503335492" last-rc-change="1503335492" exec-time="11095" queue-time="0" op-digest="9401ff83703cb25678335741cf0bca04" ra-version="4.0.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-1" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-1_last_0" operation_key="rsc_rhel7-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="23:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;23:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="13" rc-code="7" op-status="0" interval="0" last-run="1503335451" last-rc-change="1503335451" exec-time="176" queue-time="0" op-digest="75df1567eb9457f8f3c4486bbf875846" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-2" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-2_last_0" operation_key="rsc_rhel7-2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="24:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;24:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="17" rc-code="7" op-status="0" interval="0" last-run="1503335451" last-rc-change="1503335451" exec-time="163" queue-time="0" op-digest="ced6f8a1916ebbe555cedafe69985e63" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-3" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-3_last_0" operation_key="rsc_rhel7-3_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="41:23:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;41:23:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="81" rc-code="0" op-status="0" interval="0" last-run="1503335504" last-rc-change="1503335504" exec-time="41" queue-time="0" op-digest="3a5f279381f73d4be861526d72bb17a3" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="rsc_rhel7-3_monitor_5000" operation_key="rsc_rhel7-3_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="30:21:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;30:21:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="74" rc-code="0" op-status="0" interval="5000" last-rc-change="1503335476" exec-time="43" queue-time="0" op-digest="a1b2eeaa8d23ff33ffebd44f45931017" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-4" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-4_last_0" operation_key="rsc_rhel7-4_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="46:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;46:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="34" rc-code="0" op-status="0" interval="0" last-run="1503335451" last-rc-change="1503335451" exec-time="4131" queue-time="0" op-digest="b4b6b30b67042d5bc4c1735b0df27dc0" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="rsc_rhel7-4_monitor_5000" operation_key="rsc_rhel7-4_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="47:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;47:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="49" rc-code="0" op-status="0" interval="5000" last-rc-change="1503335455" exec-time="103" queue-time="0" op-digest="8ee22149973acaa2c4a338cde274ee1b" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="migrator" type="Dummy" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="migrator_last_0" operation_key="migrator_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="28:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;28:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="33" rc-code="7" op-status="0" interval="0" last-run="1503335451" last-rc-change="1503335451" exec-time="40" queue-time="0" op-digest="5de129d7fe42dbcfe537f2c63b1921b6" ra-version="1.0" op-force-restart=" fail_start_on  fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="5de129d7fe42dbcfe537f2c63b1921b6"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-5" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-5_last_0" operation_key="rsc_rhel7-5_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="27:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;27:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="29" rc-code="7" op-status="0" interval="0" last-run="1503335451" last-rc-change="1503335451" exec-time="112" queue-time="0" op-digest="aca525581410dfda70831f2846b9807d" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="ping-1" type="ping" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="ping-1_last_0" operation_key="ping-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="60:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;60:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="45" rc-code="0" op-status="0" interval="0" last-run="1503335451" last-rc-change="1503335451" exec-time="2045" queue-time="0" op-digest="029004076c53d493cd6e9f6661b1b083" ra-version="1.0"/>
356a11
+            <lrm_rsc_op id="ping-1_monitor_60000" operation_key="ping-1_monitor_60000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="61:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;61:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="47" rc-code="0" op-status="0" interval="60000" last-rc-change="1503335453" exec-time="2050" queue-time="0" op-digest="8654ff74c1be19c9385009b41becc0c8" ra-version="1.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="stateful-1" type="Stateful" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="stateful-1_last_0" operation_key="stateful-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="62:18:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;62:18:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="68" rc-code="0" op-status="0" interval="0" last-run="1503335458" last-rc-change="1503335458" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" ra-version="1.0"/>
356a11
+            <lrm_rsc_op id="stateful-1_monitor_15000" operation_key="stateful-1_monitor_15000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="63:18:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;63:18:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="70" rc-code="0" op-status="0" interval="15000" last-rc-change="1503335458" exec-time="13" queue-time="0" op-digest="873ed4f07792aa8ff18f3254244675ea" ra-version="1.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="petulant" type="DummySD" class="service">
356a11
+            <lrm_rsc_op id="petulant_last_0" operation_key="petulant_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="32:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;32:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="59" rc-code="7" op-status="0" interval="0" last-run="1503335455" last-rc-change="1503335455" exec-time="23" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="r192.168.122.207" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="r192.168.122.207_last_0" operation_key="r192.168.122.207_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="31:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;31:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="54" rc-code="7" op-status="0" interval="0" last-run="1503335455" last-rc-change="1503335455" exec-time="66" queue-time="0" op-digest="455141de0d85faf791392b0857f9dea1" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="lsb-dummy" type="/usr/share/pacemaker/tests/cts/LSBDummy" class="lsb">
356a11
+            <lrm_rsc_op id="lsb-dummy_last_0" operation_key="lsb-dummy_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="34:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;34:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="67" rc-code="7" op-status="0" interval="0" last-run="1503335455" last-rc-change="1503335455" exec-time="16" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="r192.168.122.208" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="r192.168.122.208_last_0" operation_key="r192.168.122.208_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="33:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;33:17:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="63" rc-code="7" op-status="0" interval="0" last-run="1503335455" last-rc-change="1503335455" exec-time="43" queue-time="0" op-digest="d62599e347d2c3a524c13e135846a774" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="remote-rhel7-3" type="remote" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="remote-rhel7-3_last_0" operation_key="remote-rhel7-3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="35:22:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;35:22:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="1" rc-code="7" op-status="0" interval="0" last-run="1503335492" last-rc-change="1503335492" exec-time="0" queue-time="0" op-digest="261b4fde9eb7965580f152a20c775486" ra-version="0.1" op-force-restart=" reconnect_interval  port " op-restart-digest="1dbaa1569a09b2a43b8d2b8037a48749"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="remote-rsc" type="Dummy" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="remote-rsc_last_0" operation_key="remote-rsc_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="27:26:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;27:26:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-4" call-id="87" rc-code="7" op-status="0" interval="0" last-run="1503335519" last-rc-change="1503335519" exec-time="18" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" ra-version="0.9" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+          </lrm_resource>
356a11
+        </lrm_resources>
356a11
+      </lrm>
356a11
+      <transient_attributes id="4">
356a11
+        <instance_attributes id="status-4">
356a11
+          <nvpair id="status-4-connected" name="connected" value="1"/>
356a11
+          <nvpair id="status-4-master-stateful-1" name="master-stateful-1" value="5"/>
356a11
+        </instance_attributes>
356a11
+      </transient_attributes>
356a11
+    </node_state>
356a11
+    <node_state id="5" uname="rhel7-5" crmd="online" crm-debug-origin="do_update_resource" in_ccm="true" join="member" expected="member">
356a11
+      <transient_attributes id="5">
356a11
+        <instance_attributes id="status-5">
356a11
+          <nvpair id="status-5-connected" name="connected" value="1"/>
356a11
+          <nvpair id="status-5-master-stateful-1" name="master-stateful-1" value="5"/>
356a11
+        </instance_attributes>
356a11
+      </transient_attributes>
356a11
+      <lrm id="5">
356a11
+        <lrm_resources>
356a11
+          <lrm_resource id="r192.168.122.207" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="r192.168.122.207_last_0" operation_key="r192.168.122.207_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="31:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;31:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="47" rc-code="7" op-status="0" interval="0" last-run="1503335234" last-rc-change="1503335234" exec-time="76" queue-time="0" op-digest="455141de0d85faf791392b0857f9dea1" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="lsb-dummy" type="/usr/share/pacemaker/tests/cts/LSBDummy" class="lsb">
356a11
+            <lrm_rsc_op id="lsb-dummy_last_0" operation_key="lsb-dummy_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="34:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;34:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="67" rc-code="7" op-status="0" interval="0" last-run="1503335238" last-rc-change="1503335238" exec-time="7" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="r192.168.122.208" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="r192.168.122.208_last_0" operation_key="r192.168.122.208_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="33:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;33:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="62" rc-code="7" op-status="0" interval="0" last-run="1503335238" last-rc-change="1503335238" exec-time="62" queue-time="0" op-digest="d62599e347d2c3a524c13e135846a774" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="FencingFail" type="fence_dummy" class="stonith">
356a11
+            <lrm_rsc_op id="FencingFail_last_0" operation_key="FencingFail_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="22:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;22:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="9" rc-code="7" op-status="0" interval="0" last-run="1503335231" last-rc-change="1503335231" exec-time="0" queue-time="0" op-digest="9401ff83703cb25678335741cf0bca04" ra-version="4.0.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
356a11
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="21:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;21:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1503335231" last-rc-change="1503335231" exec-time="3" queue-time="0" op-digest="2409d7001df14f35973634911d9875b1" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="petulant" type="DummySD" class="service">
356a11
+            <lrm_rsc_op id="petulant_last_0" operation_key="petulant_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="32:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;32:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="58" rc-code="7" op-status="0" interval="0" last-run="1503335238" last-rc-change="1503335238" exec-time="28" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="ping-1" type="ping" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="ping-1_last_0" operation_key="ping-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="60:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;60:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="48" rc-code="0" op-status="0" interval="0" last-run="1503335234" last-rc-change="1503335234" exec-time="2083" queue-time="0" op-digest="029004076c53d493cd6e9f6661b1b083" ra-version="1.0"/>
356a11
+            <lrm_rsc_op id="ping-1_monitor_60000" operation_key="ping-1_monitor_60000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="61:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;61:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="51" rc-code="0" op-status="0" interval="60000" last-rc-change="1503335236" exec-time="2116" queue-time="0" op-digest="8654ff74c1be19c9385009b41becc0c8" ra-version="1.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="stateful-1" type="Stateful" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="stateful-1_last_0" operation_key="stateful-1_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="62:41:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;62:41:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="68" rc-code="0" op-status="0" interval="0" last-run="1503335241" last-rc-change="1503335241" exec-time="51" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" ra-version="1.0"/>
356a11
+            <lrm_rsc_op id="stateful-1_monitor_15000" operation_key="stateful-1_monitor_15000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="64:42:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;64:42:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="70" rc-code="0" op-status="0" interval="15000" last-rc-change="1503335241" exec-time="14" queue-time="0" op-digest="873ed4f07792aa8ff18f3254244675ea" ra-version="1.0"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-1" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-1_last_0" operation_key="rsc_rhel7-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="23:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;23:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="13" rc-code="7" op-status="0" interval="0" last-run="1503335233" last-rc-change="1503335233" exec-time="218" queue-time="0" op-digest="75df1567eb9457f8f3c4486bbf875846" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-2" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-2_last_0" operation_key="rsc_rhel7-2_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="24:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:7;24:40:7:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="17" rc-code="7" op-status="0" interval="0" last-run="1503335233" last-rc-change="1503335233" exec-time="181" queue-time="0" op-digest="ced6f8a1916ebbe555cedafe69985e63" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-3" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-3_last_0" operation_key="rsc_rhel7-3_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="32:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;32:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-5" call-id="100" rc-code="0" op-status="0" interval="0" last-run="1503335525" last-rc-change="1503335525" exec-time="4125" queue-time="1" op-digest="3a5f279381f73d4be861526d72bb17a3" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="rsc_rhel7-3_monitor_5000" operation_key="rsc_rhel7-3_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="33:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;33:29:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-5" call-id="102" rc-code="0" op-status="0" interval="5000" last-rc-change="1503335529" exec-time="36" queue-time="0" op-digest="a1b2eeaa8d23ff33ffebd44f45931017" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-4" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-4_last_0" operation_key="rsc_rhel7-4_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="45:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;45:17:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-5" call-id="86" rc-code="0" op-status="0" interval="0" last-run="1503335451" last-rc-change="1503335451" exec-time="52" queue-time="0" op-digest="b4b6b30b67042d5bc4c1735b0df27dc0" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="rsc_rhel7-4_monitor_5000" operation_key="rsc_rhel7-4_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="31:44:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;31:44:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="74" rc-code="0" op-status="0" interval="5000" last-rc-change="1503335283" exec-time="30" queue-time="0" op-digest="8ee22149973acaa2c4a338cde274ee1b" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="rsc_rhel7-5" type="IPaddr2" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="rsc_rhel7-5_last_0" operation_key="rsc_rhel7-5_start_0" operation="start" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="48:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;48:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="49" rc-code="0" op-status="0" interval="0" last-run="1503335234" last-rc-change="1503335234" exec-time="4056" queue-time="0" op-digest="aca525581410dfda70831f2846b9807d" ra-version="0.1"/>
356a11
+            <lrm_rsc_op id="rsc_rhel7-5_monitor_5000" operation_key="rsc_rhel7-5_monitor_5000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.14" transition-key="49:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" transition-magic="0:0;49:40:0:a4c95233-ef19-40fd-89f0-0a7ce1f787a6" on_node="rhel7-5" call-id="53" rc-code="0" op-status="0" interval="5000" last-rc-change="1503335238" exec-time="140" queue-time="0" op-digest="51292f6c89131cf04bf857325f0e4041" ra-version="0.1"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="migrator" type="Dummy" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="migrator_last_0" operation_key="migrator_migrate_from_0" operation="migrate_from" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="43:20:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;43:20:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-5" call-id="89" rc-code="0" op-status="0" interval="0" last-run="1503335472" last-rc-change="1503335472" exec-time="28" queue-time="0" op-digest="5de129d7fe42dbcfe537f2c63b1921b6" ra-version="1.0" op-force-restart=" fail_start_on  fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-secure-params=" passwd " op-secure-digest="5de129d7fe42dbcfe537f2c63b1921b6" migrate_source="rhel7-3" migrate_target="rhel7-5"/>
356a11
+            <lrm_rsc_op id="migrator_monitor_10000" operation_key="migrator_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="41:20:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:0;41:20:0:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-5" call-id="91" rc-code="0" op-status="0" interval="10000" last-rc-change="1503335472" exec-time="16" queue-time="0" op-digest="9210327bce4f7670c7b350bf32101791" ra-version="1.0" op-secure-params=" passwd " op-secure-digest="5de129d7fe42dbcfe537f2c63b1921b6"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="remote-rhel7-3" type="remote" class="ocf" provider="pacemaker">
356a11
+            <lrm_rsc_op id="remote-rhel7-3_last_0" operation_key="remote-rhel7-3_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="36:22:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;36:22:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-5" call-id="2" rc-code="7" op-status="0" interval="0" last-run="1503335492" last-rc-change="1503335492" exec-time="0" queue-time="0" op-digest="261b4fde9eb7965580f152a20c775486" ra-version="0.1" op-force-restart=" reconnect_interval  port " op-restart-digest="1dbaa1569a09b2a43b8d2b8037a48749"/>
356a11
+          </lrm_resource>
356a11
+          <lrm_resource id="remote-rsc" type="Dummy" class="ocf" provider="heartbeat">
356a11
+            <lrm_rsc_op id="remote-rsc_last_0" operation_key="remote-rsc_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.14" transition-key="28:26:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" transition-magic="0:7;28:26:7:231989e8-bacb-4b30-ad73-d034cf9fd2ed" on_node="rhel7-5" call-id="98" rc-code="7" op-status="0" interval="0" last-run="1503335519" last-rc-change="1503335519" exec-time="22" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" ra-version="0.9" op-force-restart=" fake " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
356a11
+          </lrm_resource>
356a11
+        </lrm_resources>
356a11
+      </lrm>
356a11
+    </node_state>
356a11
+    <node_state remote_node="true" id="remote-rhel7-1" uname="remote-rhel7-1" in_ccm="false" crm-debug-origin="post_cache_update"/>
356a11
+    <node_state remote_node="true" id="remote-rhel7-3" uname="remote-rhel7-3" in_ccm="false" crm-debug-origin="remote_node_down" node_fenced="1503335525"/>
356a11
+  </status>
356a11
+</cib>
356a11
-- 
356a11
1.8.3.1
356a11