Blame SOURCES/016-guest-fencing.patch

eae27e
From 73da74305b69b086f8bc7cae697063e2534a79f4 Mon Sep 17 00:00:00 2001
eae27e
From: Ken Gaillot <kgaillot@redhat.com>
eae27e
Date: Tue, 22 Nov 2016 16:37:07 -0600
eae27e
Subject: [PATCH 1/6] Low: pengine: remove unnecessary assert
eae27e
eae27e
it was made obsolete with 1420ff88
eae27e
---
eae27e
 lib/pengine/unpack.c | 6 ++++--
eae27e
 1 file changed, 4 insertions(+), 2 deletions(-)
eae27e
eae27e
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
eae27e
index a9fbcc0..2ef9343 100644
eae27e
--- a/lib/pengine/unpack.c
eae27e
+++ b/lib/pengine/unpack.c
eae27e
@@ -1402,6 +1402,10 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
eae27e
     resource_t *rsc = this_node->details->remote_rsc;
eae27e
     resource_t *container = NULL;
eae27e
 
eae27e
+    /* If there is a node state entry for a (former) Pacemaker Remote node
eae27e
+     * but no resource creating that node, the node's connection resource will
eae27e
+     * be NULL. Consider it an offline remote node in that case.
eae27e
+     */
eae27e
     if (rsc == NULL) {
eae27e
         this_node->details->online = FALSE;
eae27e
         goto remote_online_done;
eae27e
@@ -1409,8 +1413,6 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
eae27e
 
eae27e
     container = rsc->container;
eae27e
 
eae27e
-    CRM_ASSERT(rsc != NULL);
eae27e
-
eae27e
     /* If the resource is currently started, mark it online. */
eae27e
     if (rsc->role == RSC_ROLE_STARTED) {
eae27e
         crm_trace("Remote node %s is set to ONLINE. role == started", this_node->details->id);
eae27e
-- 
eae27e
1.8.3.1
eae27e
eae27e
eae27e
From 5156074d560d85ee84de31b9d1e0bd893999fa4e Mon Sep 17 00:00:00 2001
eae27e
From: Ken Gaillot <kgaillot@redhat.com>
eae27e
Date: Wed, 23 Nov 2016 13:40:47 -0600
eae27e
Subject: [PATCH 2/6] Log: pengine: improve trace messages for Pacemaker Remote
eae27e
 nodes
eae27e
eae27e
---
eae27e
 lib/pengine/unpack.c | 22 ++++++++++++++++------
eae27e
 1 file changed, 16 insertions(+), 6 deletions(-)
eae27e
eae27e
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
eae27e
index 2ef9343..a49e108 100644
eae27e
--- a/lib/pengine/unpack.c
eae27e
+++ b/lib/pengine/unpack.c
eae27e
@@ -70,10 +70,15 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)
eae27e
     /* A guest node is fenced by marking its container as failed */
eae27e
     if (is_container_remote_node(node)) {
eae27e
         resource_t *rsc = node->details->remote_rsc->container;
eae27e
+
eae27e
         if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {
eae27e
             crm_warn("Guest node %s will be fenced (by recovering %s) %s",
eae27e
                 node->details->uname, rsc->id, reason);
eae27e
-            /* node->details->unclean = TRUE; */
eae27e
+
eae27e
+            /* We don't mark the node as unclean, because that would prevent the
eae27e
+             * node from running resources. We want to allow it to run resources
eae27e
+             * in this transition if the recovery succeeds.
eae27e
+             */
eae27e
             node->details->remote_requires_reset = TRUE;
eae27e
             set_bit(rsc->flags, pe_rsc_failed);
eae27e
         }
eae27e
@@ -1415,30 +1420,35 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
eae27e
 
eae27e
     /* If the resource is currently started, mark it online. */
eae27e
     if (rsc->role == RSC_ROLE_STARTED) {
eae27e
-        crm_trace("Remote node %s is set to ONLINE. role == started", this_node->details->id);
eae27e
+        crm_trace("%s node %s presumed ONLINE because connection resource is started",
eae27e
+                  (container? "Guest" : "Remote"), this_node->details->id);
eae27e
         this_node->details->online = TRUE;
eae27e
     }
eae27e
 
eae27e
     /* consider this node shutting down if transitioning start->stop */
eae27e
     if (rsc->role == RSC_ROLE_STARTED && rsc->next_role == RSC_ROLE_STOPPED) {
eae27e
-        crm_trace("Remote node %s shutdown. transition from start to stop role", this_node->details->id);
eae27e
+        crm_trace("%s node %s shutting down because connection resource is stopping",
eae27e
+                  (container? "Guest" : "Remote"), this_node->details->id);
eae27e
         this_node->details->shutdown = TRUE;
eae27e
     }
eae27e
 
eae27e
     /* Now check all the failure conditions. */
eae27e
     if(container && is_set(container->flags, pe_rsc_failed)) {
eae27e
-        crm_trace("Remote node %s is set to UNCLEAN. rsc failed.", this_node->details->id);
eae27e
+        crm_trace("Guest node %s UNCLEAN because guest resource failed",
eae27e
+                  this_node->details->id);
eae27e
         this_node->details->online = FALSE;
eae27e
         this_node->details->remote_requires_reset = TRUE;
eae27e
 
eae27e
     } else if(is_set(rsc->flags, pe_rsc_failed)) {
eae27e
-        crm_trace("Remote node %s is set to OFFLINE. rsc failed.", this_node->details->id);
eae27e
+        crm_trace("%s node %s OFFLINE because connection resource failed",
eae27e
+                  (container? "Guest" : "Remote"), this_node->details->id);
eae27e
         this_node->details->online = FALSE;
eae27e
 
eae27e
     } else if (rsc->role == RSC_ROLE_STOPPED
eae27e
         || (container && container->role == RSC_ROLE_STOPPED)) {
eae27e
 
eae27e
-        crm_trace("Remote node %s is set to OFFLINE. node is stopped.", this_node->details->id);
eae27e
+        crm_trace("%s node %s OFFLINE because its resource is stopped",
eae27e
+                  (container? "Guest" : "Remote"), this_node->details->id);
eae27e
         this_node->details->online = FALSE;
eae27e
         this_node->details->remote_requires_reset = FALSE;
eae27e
     }
eae27e
-- 
eae27e
1.8.3.1
eae27e
eae27e
eae27e
From 225d20cacc5643e113d42159fc713071172d88da Mon Sep 17 00:00:00 2001
eae27e
From: Ken Gaillot <kgaillot@redhat.com>
eae27e
Date: Tue, 22 Nov 2016 16:40:52 -0600
eae27e
Subject: [PATCH 3/6] Fix: pengine: guest node fencing doesn't require stonith
eae27e
 enabled
eae27e
eae27e
Comments elsewhere say as much, but stage6() didn't get the memo
eae27e
---
eae27e
 lib/pengine/utils.c | 13 +++++++++++++
eae27e
 pengine/allocate.c  |  5 +----
eae27e
 2 files changed, 14 insertions(+), 4 deletions(-)
eae27e
eae27e
diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c
eae27e
index cc97db1..6be9bb1 100644
eae27e
--- a/lib/pengine/utils.c
eae27e
+++ b/lib/pengine/utils.c
eae27e
@@ -36,6 +36,19 @@ void unpack_operation(action_t * action, xmlNode * xml_obj, resource_t * contain
eae27e
 static xmlNode *find_rsc_op_entry_helper(resource_t * rsc, const char *key,
eae27e
                                          gboolean include_disabled);
eae27e
 
eae27e
+/*!
eae27e
+ * \internal
eae27e
+ * \brief Check whether we can fence a particular node
eae27e
+ *
eae27e
+ * \param[in] data_set  Working set for cluster
eae27e
+ * \param[in] node      Name of node to check
eae27e
+ *
eae27e
+ * \return TRUE if node can be fenced, FALSE otherwise
eae27e
+ *
eae27e
+ * \note This function should only be called for cluster nodes and baremetal
eae27e
+ *       remote nodes; guest nodes are fenced by stopping their container
eae27e
+ *       resource, so fence execution requirements do not apply to them.
eae27e
+ */
eae27e
 bool pe_can_fence(pe_working_set_t * data_set, node_t *node)
eae27e
 {
eae27e
     if(is_not_set(data_set->flags, pe_flag_stonith_enabled)) {
eae27e
diff --git a/pengine/allocate.c b/pengine/allocate.c
eae27e
index 82abd36..bdf03e5 100644
eae27e
--- a/pengine/allocate.c
eae27e
+++ b/pengine/allocate.c
eae27e
@@ -1374,10 +1374,7 @@ stage6(pe_working_set_t * data_set)
eae27e
          * guest's host.
eae27e
          */
eae27e
         if (is_container_remote_node(node)) {
eae27e
-            /* Guest */
eae27e
-            if (need_stonith
eae27e
-                && node->details->remote_requires_reset
eae27e
-                && pe_can_fence(data_set, node)) {
eae27e
+            if (node->details->remote_requires_reset && need_stonith) {
eae27e
                 resource_t *container = node->details->remote_rsc->container;
eae27e
                 char *key = stop_key(container);
eae27e
                 GListPtr stop_list = find_actions(container->actions, key, NULL);
eae27e
-- 
eae27e
1.8.3.1
eae27e
eae27e
eae27e
From b11887869723f23a330af8b1b0e9ffd935b68ae0 Mon Sep 17 00:00:00 2001
eae27e
From: Ken Gaillot <kgaillot@redhat.com>
eae27e
Date: Wed, 23 Mar 2016 17:57:50 -0500
eae27e
Subject: [PATCH 4/6] Test: pengine: add regression test for when a guest
eae27e
 node's host goes away
eae27e
eae27e
As of this commit, the PE handles this situation badly. Adding the test before
eae27e
the fix allows the changes in behavior to be highlighted.
eae27e
---
eae27e
 pengine/regression.sh                       |   1 +
eae27e
 pengine/test10/guest-node-host-dies.dot     | 107 ++++++
eae27e
 pengine/test10/guest-node-host-dies.exp     | 539 ++++++++++++++++++++++++++++
eae27e
 pengine/test10/guest-node-host-dies.scores  |  80 +++++
eae27e
 pengine/test10/guest-node-host-dies.summary |  73 ++++
eae27e
 pengine/test10/guest-node-host-dies.xml     | 294 +++++++++++++++
eae27e
 6 files changed, 1094 insertions(+)
eae27e
 create mode 100644 pengine/test10/guest-node-host-dies.dot
eae27e
 create mode 100644 pengine/test10/guest-node-host-dies.exp
eae27e
 create mode 100644 pengine/test10/guest-node-host-dies.scores
eae27e
 create mode 100644 pengine/test10/guest-node-host-dies.summary
eae27e
 create mode 100644 pengine/test10/guest-node-host-dies.xml
eae27e
eae27e
diff --git a/pengine/regression.sh b/pengine/regression.sh
eae27e
index be1734b..1bc8e1e 100755
eae27e
--- a/pengine/regression.sh
eae27e
+++ b/pengine/regression.sh
eae27e
@@ -817,6 +817,7 @@ do_test whitebox-unexpectedly-running "Recover container nodes the cluster did n
eae27e
 do_test whitebox-migrate1 "Migrate both container and connection resource"
eae27e
 do_test whitebox-imply-stop-on-fence "imply stop action on container node rsc when host node is fenced"
eae27e
 do_test whitebox-nested-group "Verify guest remote-node works nested in a group"
eae27e
+do_test guest-node-host-dies "Verify guest node is recovered if host goes away"
eae27e
 
eae27e
 echo ""
eae27e
 do_test remote-startup-probes  "Baremetal remote-node startup probes"
eae27e
diff --git a/pengine/test10/guest-node-host-dies.dot b/pengine/test10/guest-node-host-dies.dot
eae27e
new file mode 100644
eae27e
index 0000000..01858b3
eae27e
--- /dev/null
eae27e
+++ b/pengine/test10/guest-node-host-dies.dot
eae27e
@@ -0,0 +1,107 @@
eae27e
+digraph "g" {
eae27e
+"Fencing_monitor_120000 rhel7-4" [ style=bold color="green" fontcolor="black"]
eae27e
+"Fencing_start_0 rhel7-4" -> "Fencing_monitor_120000 rhel7-4" [ style = bold]
eae27e
+"Fencing_start_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
eae27e
+"Fencing_stop_0 rhel7-4" -> "Fencing_start_0 rhel7-4" [ style = bold]
eae27e
+"Fencing_stop_0 rhel7-4" -> "all_stopped" [ style = bold]
eae27e
+"Fencing_stop_0 rhel7-4" [ style=bold color="green" fontcolor="black"]
eae27e
+"all_stopped" [ style=bold color="green" fontcolor="orange"]
eae27e
+"container1_start_0 rhel7-2" -> "lxc1_start_0 rhel7-2" [ style = bold]
eae27e
+"container1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"]
eae27e
+"container1_stop_0 rhel7-1" -> "all_stopped" [ style = bold]
eae27e
+"container1_stop_0 rhel7-1" -> "container1_start_0 rhel7-2" [ style = bold]
eae27e
+"container1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"]
eae27e
+"container2_start_0 rhel7-3" -> "lxc2_start_0 rhel7-3" [ style = bold]
eae27e
+"container2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
eae27e
+"container2_stop_0 rhel7-1" -> "all_stopped" [ style = bold]
eae27e
+"container2_stop_0 rhel7-1" -> "container2_start_0 rhel7-3" [ style = bold]
eae27e
+"container2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms-master_demote_0" -> "lxc-ms-master_demoted_0" [ style = bold]
eae27e
+"lxc-ms-master_demote_0" -> "lxc-ms_demote_0 lxc1" [ style = bold]
eae27e
+"lxc-ms-master_demote_0" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_promote_0" [ style = bold]
eae27e
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_start_0" [ style = bold]
eae27e
+"lxc-ms-master_demoted_0" -> "lxc-ms-master_stop_0" [ style = bold]
eae27e
+"lxc-ms-master_demoted_0" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms-master_promote_0" -> "lxc-ms_promote_0 lxc1" [ style = bold]
eae27e
+"lxc-ms-master_promote_0" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms-master_promoted_0" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms-master_running_0" -> "lxc-ms-master_promote_0" [ style = bold]
eae27e
+"lxc-ms-master_running_0" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms-master_start_0" -> "lxc-ms-master_running_0" [ style = bold]
eae27e
+"lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc1" [ style = bold]
eae27e
+"lxc-ms-master_start_0" -> "lxc-ms_start_0 lxc2" [ style = bold]
eae27e
+"lxc-ms-master_start_0" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms-master_stop_0" -> "lxc-ms-master_stopped_0" [ style = bold]
eae27e
+"lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc1" [ style = bold]
eae27e
+"lxc-ms-master_stop_0" -> "lxc-ms_stop_0 lxc2" [ style = bold]
eae27e
+"lxc-ms-master_stop_0" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms-master_stopped_0" -> "lxc-ms-master_promote_0" [ style = bold]
eae27e
+"lxc-ms-master_stopped_0" -> "lxc-ms-master_start_0" [ style = bold]
eae27e
+"lxc-ms-master_stopped_0" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms_demote_0 lxc1" -> "lxc-ms-master_demoted_0" [ style = bold]
eae27e
+"lxc-ms_demote_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold]
eae27e
+"lxc-ms_demote_0 lxc1" -> "lxc-ms_stop_0 lxc1" [ style = bold]
eae27e
+"lxc-ms_demote_0 lxc1" [ style=bold color="green" fontcolor="black"]
eae27e
+"lxc-ms_monitor_10000 lxc2" [ style=bold color="green" fontcolor="black"]
eae27e
+"lxc-ms_promote_0 lxc1" -> "lxc-ms-master_promoted_0" [ style = bold]
eae27e
+"lxc-ms_promote_0 lxc1" [ style=bold color="green" fontcolor="black"]
eae27e
+"lxc-ms_start_0 lxc1" -> "lxc-ms-master_running_0" [ style = bold]
eae27e
+"lxc-ms_start_0 lxc1" -> "lxc-ms_promote_0 lxc1" [ style = bold]
eae27e
+"lxc-ms_start_0 lxc1" [ style=bold color="green" fontcolor="black"]
eae27e
+"lxc-ms_start_0 lxc2" -> "lxc-ms-master_running_0" [ style = bold]
eae27e
+"lxc-ms_start_0 lxc2" -> "lxc-ms_monitor_10000 lxc2" [ style = bold]
eae27e
+"lxc-ms_start_0 lxc2" [ style=bold color="green" fontcolor="black"]
eae27e
+"lxc-ms_stop_0 lxc1" -> "all_stopped" [ style = bold]
eae27e
+"lxc-ms_stop_0 lxc1" -> "lxc-ms-master_stopped_0" [ style = bold]
eae27e
+"lxc-ms_stop_0 lxc1" -> "lxc-ms_start_0 lxc1" [ style = bold]
eae27e
+"lxc-ms_stop_0 lxc1" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc-ms_stop_0 lxc2" -> "all_stopped" [ style = bold]
eae27e
+"lxc-ms_stop_0 lxc2" -> "lxc-ms-master_stopped_0" [ style = bold]
eae27e
+"lxc-ms_stop_0 lxc2" -> "lxc-ms_start_0 lxc2" [ style = bold]
eae27e
+"lxc-ms_stop_0 lxc2" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc1_monitor_30000 rhel7-2" [ style=bold color="green" fontcolor="black"]
eae27e
+"lxc1_start_0 rhel7-2" -> "lxc-ms_promote_0 lxc1" [ style = bold]
eae27e
+"lxc1_start_0 rhel7-2" -> "lxc-ms_start_0 lxc1" [ style = bold]
eae27e
+"lxc1_start_0 rhel7-2" -> "lxc1_monitor_30000 rhel7-2" [ style = bold]
eae27e
+"lxc1_start_0 rhel7-2" [ style=bold color="green" fontcolor="black"]
eae27e
+"lxc1_stop_0 rhel7-1" -> "all_stopped" [ style = bold]
eae27e
+"lxc1_stop_0 rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold]
eae27e
+"lxc1_stop_0 rhel7-1" -> "lxc1_start_0 rhel7-2" [ style = bold]
eae27e
+"lxc1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"]
eae27e
+"lxc2_monitor_30000 rhel7-3" [ style=bold color="green" fontcolor="black"]
eae27e
+"lxc2_start_0 rhel7-3" -> "lxc-ms_monitor_10000 lxc2" [ style = bold]
eae27e
+"lxc2_start_0 rhel7-3" -> "lxc-ms_start_0 lxc2" [ style = bold]
eae27e
+"lxc2_start_0 rhel7-3" -> "lxc2_monitor_30000 rhel7-3" [ style = bold]
eae27e
+"lxc2_start_0 rhel7-3" [ style=bold color="green" fontcolor="black"]
eae27e
+"lxc2_stop_0 rhel7-1" -> "all_stopped" [ style = bold]
eae27e
+"lxc2_stop_0 rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold]
eae27e
+"lxc2_stop_0 rhel7-1" -> "lxc2_start_0 rhel7-3" [ style = bold]
eae27e
+"lxc2_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"]
eae27e
+"rsc_rhel7-1_monitor_5000 rhel7-5" [ style=bold color="green" fontcolor="black"]
eae27e
+"rsc_rhel7-1_start_0 rhel7-5" -> "rsc_rhel7-1_monitor_5000 rhel7-5" [ style = bold]
eae27e
+"rsc_rhel7-1_start_0 rhel7-5" [ style=bold color="green" fontcolor="black"]
eae27e
+"rsc_rhel7-1_stop_0 rhel7-1" -> "all_stopped" [ style = bold]
eae27e
+"rsc_rhel7-1_stop_0 rhel7-1" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold]
eae27e
+"rsc_rhel7-1_stop_0 rhel7-1" [ style=bold color="green" fontcolor="orange"]
eae27e
+"stonith 'reboot' rhel7-1" -> "container1_stop_0 rhel7-1" [ style = bold]
eae27e
+"stonith 'reboot' rhel7-1" -> "container2_stop_0 rhel7-1" [ style = bold]
eae27e
+"stonith 'reboot' rhel7-1" -> "lxc-ms-master_stop_0" [ style = bold]
eae27e
+"stonith 'reboot' rhel7-1" -> "lxc-ms_stop_0 lxc1" [ style = bold]
eae27e
+"stonith 'reboot' rhel7-1" -> "lxc-ms_stop_0 lxc2" [ style = bold]
eae27e
+"stonith 'reboot' rhel7-1" -> "lxc1_stop_0 rhel7-1" [ style = bold]
eae27e
+"stonith 'reboot' rhel7-1" -> "lxc2_stop_0 rhel7-1" [ style = bold]
eae27e
+"stonith 'reboot' rhel7-1" -> "rsc_rhel7-1_stop_0 rhel7-1" [ style = bold]
eae27e
+"stonith 'reboot' rhel7-1" -> "stonith_complete" [ style = bold]
eae27e
+"stonith 'reboot' rhel7-1" [ style=bold color="green" fontcolor="black"]
eae27e
+"stonith_complete" -> "all_stopped" [ style = bold]
eae27e
+"stonith_complete" -> "container1_start_0 rhel7-2" [ style = bold]
eae27e
+"stonith_complete" -> "container2_start_0 rhel7-3" [ style = bold]
eae27e
+"stonith_complete" -> "lxc-ms_promote_0 lxc1" [ style = bold]
eae27e
+"stonith_complete" -> "lxc-ms_start_0 lxc1" [ style = bold]
eae27e
+"stonith_complete" -> "lxc-ms_start_0 lxc2" [ style = bold]
eae27e
+"stonith_complete" -> "lxc1_start_0 rhel7-2" [ style = bold]
eae27e
+"stonith_complete" -> "lxc2_start_0 rhel7-3" [ style = bold]
eae27e
+"stonith_complete" -> "rsc_rhel7-1_start_0 rhel7-5" [ style = bold]
eae27e
+"stonith_complete" [ style=bold color="green" fontcolor="orange"]
eae27e
+}
eae27e
diff --git a/pengine/test10/guest-node-host-dies.exp b/pengine/test10/guest-node-host-dies.exp
eae27e
new file mode 100644
eae27e
index 0000000..b3c24be
eae27e
--- /dev/null
eae27e
+++ b/pengine/test10/guest-node-host-dies.exp
eae27e
@@ -0,0 +1,539 @@
eae27e
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
eae27e
+  <synapse id="0">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="5" operation="stop" operation_key="Fencing_stop_0" on_node="rhel7-4" on_node_uuid="5">
eae27e
+        <primitive id="Fencing" class="stonith" type="fence_xvm"/>
eae27e
+        <attributes CRM_meta_name="stop" CRM_meta_timeout="60000"  delay="0" multicast_address="239.255.100.100" pcmk_arg_map="domain:uname" pcmk_host_list="rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5"/>
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs/>
eae27e
+  </synapse>
eae27e
+  <synapse id="1">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="4" operation="start" operation_key="Fencing_start_0" on_node="rhel7-4" on_node_uuid="5">
eae27e
+        <primitive id="Fencing" class="stonith" type="fence_xvm"/>
eae27e
+        <attributes CRM_meta_name="start" CRM_meta_timeout="60000"  delay="0" multicast_address="239.255.100.100" pcmk_arg_map="domain:uname" pcmk_host_list="rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5"/>
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="5" operation="stop" operation_key="Fencing_stop_0" on_node="rhel7-4" on_node_uuid="5"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="2">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="1" operation="monitor" operation_key="Fencing_monitor_120000" on_node="rhel7-4" on_node_uuid="5">
eae27e
+        <primitive id="Fencing" class="stonith" type="fence_xvm"/>
eae27e
+        <attributes CRM_meta_interval="120000" CRM_meta_name="monitor" CRM_meta_timeout="120000"  delay="0" multicast_address="239.255.100.100" pcmk_arg_map="domain:uname" pcmk_host_list="rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5"/>
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="4" operation="start" operation_key="Fencing_start_0" on_node="rhel7-4" on_node_uuid="5"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="3">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="8" operation="monitor" operation_key="rsc_rhel7-1_monitor_5000" on_node="rhel7-5" on_node_uuid="3">
eae27e
+        <primitive id="rsc_rhel7-1" class="ocf" provider="heartbeat" type="IPaddr2"/>
eae27e
+        <attributes CRM_meta_interval="5000" CRM_meta_name="monitor" CRM_meta_timeout="20000" cidr_netmask="32"  ip="192.168.122.202"/>
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="7" operation="start" operation_key="rsc_rhel7-1_start_0" on_node="rhel7-5" on_node_uuid="3"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="4">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="7" operation="start" operation_key="rsc_rhel7-1_start_0" on_node="rhel7-5" on_node_uuid="3">
eae27e
+        <primitive id="rsc_rhel7-1" class="ocf" provider="heartbeat" type="IPaddr2"/>
eae27e
+        <attributes CRM_meta_timeout="20000" cidr_netmask="32"  ip="192.168.122.202"/>
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="6" operation="stop" operation_key="rsc_rhel7-1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="5">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="6" operation="stop" operation_key="rsc_rhel7-1_stop_0">
eae27e
+        <attributes CRM_meta_timeout="20000" cidr_netmask="32"  ip="192.168.122.202"/>
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="6">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="10" operation="start" operation_key="container1_start_0" on_node="rhel7-2" on_node_uuid="2">
eae27e
+        <primitive id="container1" class="ocf" provider="heartbeat" type="VirtualDomain"/>
eae27e
+        <attributes CRM_meta_remote_node="lxc1" CRM_meta_timeout="20000" config="/var/lib/pacemaker/cts/lxc/lxc1.xml"  force_stop="true" hypervisor="lxc:///"/>
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="9" operation="stop" operation_key="container1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="7">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="9" operation="stop" operation_key="container1_stop_0">
eae27e
+        <attributes CRM_meta_remote_node="lxc1" CRM_meta_timeout="20000" config="/var/lib/pacemaker/cts/lxc/lxc1.xml"  force_stop="true" hypervisor="lxc:///"/>
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="27" operation="stop" operation_key="lxc1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="8">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="12" operation="start" operation_key="container2_start_0" on_node="rhel7-3" on_node_uuid="4">
eae27e
+        <primitive id="container2" class="ocf" provider="heartbeat" type="VirtualDomain"/>
eae27e
+        <attributes CRM_meta_remote_node="lxc2" CRM_meta_timeout="20000" config="/var/lib/pacemaker/cts/lxc/lxc2.xml"  force_stop="true" hypervisor="lxc:///"/>
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="11" operation="stop" operation_key="container2_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="9">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="11" operation="stop" operation_key="container2_stop_0">
eae27e
+        <attributes CRM_meta_remote_node="lxc2" CRM_meta_timeout="20000" config="/var/lib/pacemaker/cts/lxc/lxc2.xml"  force_stop="true" hypervisor="lxc:///"/>
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="30" operation="stop" operation_key="lxc2_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="10">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="16" operation="promote" operation_key="lxc-ms_promote_0" internal_operation_key="lxc-ms:0_promote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-2">
eae27e
+        <primitive id="lxc-ms" long-id="lxc-ms:0" class="ocf" provider="pacemaker" type="Stateful"/>
eae27e
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="13" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:0_start_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-2"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="14" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-1"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="23" operation="promote" operation_key="lxc-ms-master_promote_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="28" operation="start" operation_key="lxc1_start_0" on_node="rhel7-2" on_node_uuid="2"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="11">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="15" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:0_stop_0">
eae27e
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="14" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-1"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="21" operation="stop" operation_key="lxc-ms-master_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="12">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="14" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-1">
eae27e
+        <primitive id="lxc-ms" long-id="lxc-ms:0" class="ocf" provider="pacemaker" type="Stateful"/>
eae27e
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="25" operation="demote" operation_key="lxc-ms-master_demote_0"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="13">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="13" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:0_start_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-2">
eae27e
+        <primitive id="lxc-ms" long-id="lxc-ms:0" class="ocf" provider="pacemaker" type="Stateful"/>
eae27e
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="15" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:0_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="19" operation="start" operation_key="lxc-ms-master_start_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="28" operation="start" operation_key="lxc1_start_0" on_node="rhel7-2" on_node_uuid="2"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="14">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="18" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:1_stop_0">
eae27e
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="21" operation="stop" operation_key="lxc-ms-master_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="15">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="17" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:1_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
eae27e
+        <primitive id="lxc-ms" long-id="lxc-ms:1" class="ocf" provider="pacemaker" type="Stateful"/>
eae27e
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="18" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="19" operation="start" operation_key="lxc-ms-master_start_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="31" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="4"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="16">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="2" operation="monitor" operation_key="lxc-ms_monitor_10000" internal_operation_key="lxc-ms:1_monitor_10000" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3">
eae27e
+        <primitive id="lxc-ms" long-id="lxc-ms:1" class="ocf" provider="pacemaker" type="Stateful"/>
eae27e
+        <attributes CRM_meta_clone="1" CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="17" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:1_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="31" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="4"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="17" priority="1000000">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="26" operation="demoted" operation_key="lxc-ms-master_demoted_0">
eae27e
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="14" operation="demote" operation_key="lxc-ms_demote_0" internal_operation_key="lxc-ms:0_demote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-1"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="25" operation="demote" operation_key="lxc-ms-master_demote_0"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="18">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="25" operation="demote" operation_key="lxc-ms-master_demote_0">
eae27e
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs/>
eae27e
+  </synapse>
eae27e
+  <synapse id="19" priority="1000000">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="24" operation="promoted" operation_key="lxc-ms-master_promoted_0">
eae27e
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="16" operation="promote" operation_key="lxc-ms_promote_0" internal_operation_key="lxc-ms:0_promote_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-2"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="20">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="23" operation="promote" operation_key="lxc-ms-master_promote_0">
eae27e
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="20" operation="running" operation_key="lxc-ms-master_running_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="22" operation="stopped" operation_key="lxc-ms-master_stopped_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="26" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="21" priority="1000000">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="22" operation="stopped" operation_key="lxc-ms-master_stopped_0">
eae27e
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="15" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:0_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="18" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="21" operation="stop" operation_key="lxc-ms-master_stop_0"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="22">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="21" operation="stop" operation_key="lxc-ms-master_stop_0">
eae27e
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="26" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="23" priority="1000000">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="20" operation="running" operation_key="lxc-ms-master_running_0">
eae27e
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="13" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:0_start_0" on_node="lxc1" on_node_uuid="lxc1" router_node="rhel7-2"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="17" operation="start" operation_key="lxc-ms_start_0" internal_operation_key="lxc-ms:1_start_0" on_node="lxc2" on_node_uuid="lxc2" router_node="rhel7-3"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="19" operation="start" operation_key="lxc-ms-master_start_0"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="24">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="19" operation="start" operation_key="lxc-ms-master_start_0">
eae27e
+        <attributes CRM_meta_clone_max="2" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_master_max="1" CRM_meta_master_node_max="1" CRM_meta_notify="false" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="22" operation="stopped" operation_key="lxc-ms-master_stopped_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="26" operation="demoted" operation_key="lxc-ms-master_demoted_0"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="25">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="29" operation="monitor" operation_key="lxc1_monitor_30000" on_node="rhel7-2" on_node_uuid="2">
eae27e
+        <primitive id="lxc1" class="ocf" provider="pacemaker" type="remote"/>
eae27e
+        <attributes CRM_meta_container="container1" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_timeout="30000" />
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="28" operation="start" operation_key="lxc1_start_0" on_node="rhel7-2" on_node_uuid="2"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="26">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="28" operation="start" operation_key="lxc1_start_0" on_node="rhel7-2" on_node_uuid="2">
eae27e
+        <primitive id="lxc1" class="ocf" provider="pacemaker" type="remote"/>
eae27e
+        <attributes CRM_meta_container="container1" CRM_meta_name="start" CRM_meta_timeout="60000" />
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="10" operation="start" operation_key="container1_start_0" on_node="rhel7-2" on_node_uuid="2"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="27" operation="stop" operation_key="lxc1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="27">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="27" operation="stop" operation_key="lxc1_stop_0">
eae27e
+        <attributes CRM_meta_container="container1" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="28">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="32" operation="monitor" operation_key="lxc2_monitor_30000" on_node="rhel7-3" on_node_uuid="4">
eae27e
+        <primitive id="lxc2" class="ocf" provider="pacemaker" type="remote"/>
eae27e
+        <attributes CRM_meta_container="container2" CRM_meta_interval="30000" CRM_meta_name="monitor" CRM_meta_timeout="30000" />
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="31" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="4"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="29">
eae27e
+    <action_set>
eae27e
+      <rsc_op id="31" operation="start" operation_key="lxc2_start_0" on_node="rhel7-3" on_node_uuid="4">
eae27e
+        <primitive id="lxc2" class="ocf" provider="pacemaker" type="remote"/>
eae27e
+        <attributes CRM_meta_container="container2" CRM_meta_name="start" CRM_meta_timeout="60000" />
eae27e
+      </rsc_op>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="12" operation="start" operation_key="container2_start_0" on_node="rhel7-3" on_node_uuid="4"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="30" operation="stop" operation_key="lxc2_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="30">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="30" operation="stop" operation_key="lxc2_stop_0">
eae27e
+        <attributes CRM_meta_container="container2" CRM_meta_timeout="20000" />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="31">
eae27e
+    <action_set>
eae27e
+      <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1">
eae27e
+        <attributes CRM_meta_connected="1" CRM_meta_on_node="rhel7-1" CRM_meta_on_node_uuid="1" CRM_meta_shutdown="0" CRM_meta_stonith_action="reboot" />
eae27e
+        <downed>
eae27e
+          <node id="1"/>
eae27e
+          <node id="lxc2"/>
eae27e
+          <node id="lxc1"/>
eae27e
+        </downed>
eae27e
+      </crm_event>
eae27e
+    </action_set>
eae27e
+    <inputs/>
eae27e
+  </synapse>
eae27e
+  <synapse id="32">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete">
eae27e
+        <attributes />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <crm_event id="34" operation="stonith" operation_key="stonith-rhel7-1-reboot" on_node="rhel7-1" on_node_uuid="1"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+  <synapse id="33">
eae27e
+    <action_set>
eae27e
+      <pseudo_event id="3" operation="all_stopped" operation_key="all_stopped">
eae27e
+        <attributes />
eae27e
+      </pseudo_event>
eae27e
+    </action_set>
eae27e
+    <inputs>
eae27e
+      <trigger>
eae27e
+        <rsc_op id="5" operation="stop" operation_key="Fencing_stop_0" on_node="rhel7-4" on_node_uuid="5"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="6" operation="stop" operation_key="rsc_rhel7-1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="9" operation="stop" operation_key="container1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="11" operation="stop" operation_key="container2_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="15" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:0_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="18" operation="stop" operation_key="lxc-ms_stop_0" internal_operation_key="lxc-ms:1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="27" operation="stop" operation_key="lxc1_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="30" operation="stop" operation_key="lxc2_stop_0"/>
eae27e
+      </trigger>
eae27e
+      <trigger>
eae27e
+        <pseudo_event id="33" operation="stonith_complete" operation_key="stonith_complete"/>
eae27e
+      </trigger>
eae27e
+    </inputs>
eae27e
+  </synapse>
eae27e
+</transition_graph>
eae27e
diff --git a/pengine/test10/guest-node-host-dies.scores b/pengine/test10/guest-node-host-dies.scores
eae27e
new file mode 100644
eae27e
index 0000000..0d7ad3f
eae27e
--- /dev/null
eae27e
+++ b/pengine/test10/guest-node-host-dies.scores
eae27e
@@ -0,0 +1,80 @@
eae27e
+Allocation scores:
eae27e
+clone_color: lxc-ms-master allocation score on lxc1: INFINITY
eae27e
+clone_color: lxc-ms-master allocation score on lxc2: INFINITY
eae27e
+clone_color: lxc-ms-master allocation score on rhel7-1: 0
eae27e
+clone_color: lxc-ms-master allocation score on rhel7-2: 0
eae27e
+clone_color: lxc-ms-master allocation score on rhel7-3: 0
eae27e
+clone_color: lxc-ms-master allocation score on rhel7-4: 0
eae27e
+clone_color: lxc-ms-master allocation score on rhel7-5: 0
eae27e
+clone_color: lxc-ms:0 allocation score on lxc1: INFINITY
eae27e
+clone_color: lxc-ms:0 allocation score on lxc2: INFINITY
eae27e
+clone_color: lxc-ms:0 allocation score on rhel7-1: 0
eae27e
+clone_color: lxc-ms:0 allocation score on rhel7-2: 0
eae27e
+clone_color: lxc-ms:0 allocation score on rhel7-3: 0
eae27e
+clone_color: lxc-ms:0 allocation score on rhel7-4: 0
eae27e
+clone_color: lxc-ms:0 allocation score on rhel7-5: 0
eae27e
+clone_color: lxc-ms:1 allocation score on lxc1: INFINITY
eae27e
+clone_color: lxc-ms:1 allocation score on lxc2: INFINITY
eae27e
+clone_color: lxc-ms:1 allocation score on rhel7-1: 0
eae27e
+clone_color: lxc-ms:1 allocation score on rhel7-2: 0
eae27e
+clone_color: lxc-ms:1 allocation score on rhel7-3: 0
eae27e
+clone_color: lxc-ms:1 allocation score on rhel7-4: 0
eae27e
+clone_color: lxc-ms:1 allocation score on rhel7-5: 0
eae27e
+lxc-ms:0 promotion score on lxc1: INFINITY
eae27e
+lxc-ms:1 promotion score on lxc2: INFINITY
eae27e
+native_color: Fencing allocation score on lxc1: -INFINITY
eae27e
+native_color: Fencing allocation score on lxc2: -INFINITY
eae27e
+native_color: Fencing allocation score on rhel7-1: 0
eae27e
+native_color: Fencing allocation score on rhel7-2: 0
eae27e
+native_color: Fencing allocation score on rhel7-3: 0
eae27e
+native_color: Fencing allocation score on rhel7-4: 0
eae27e
+native_color: Fencing allocation score on rhel7-5: 0
eae27e
+native_color: container1 allocation score on lxc1: -INFINITY
eae27e
+native_color: container1 allocation score on lxc2: -INFINITY
eae27e
+native_color: container1 allocation score on rhel7-1: -INFINITY
eae27e
+native_color: container1 allocation score on rhel7-2: 0
eae27e
+native_color: container1 allocation score on rhel7-3: 0
eae27e
+native_color: container1 allocation score on rhel7-4: 0
eae27e
+native_color: container1 allocation score on rhel7-5: 0
eae27e
+native_color: container2 allocation score on lxc1: -INFINITY
eae27e
+native_color: container2 allocation score on lxc2: -INFINITY
eae27e
+native_color: container2 allocation score on rhel7-1: -INFINITY
eae27e
+native_color: container2 allocation score on rhel7-2: 0
eae27e
+native_color: container2 allocation score on rhel7-3: 0
eae27e
+native_color: container2 allocation score on rhel7-4: 0
eae27e
+native_color: container2 allocation score on rhel7-5: 0
eae27e
+native_color: lxc-ms:0 allocation score on lxc1: INFINITY
eae27e
+native_color: lxc-ms:0 allocation score on lxc2: INFINITY
eae27e
+native_color: lxc-ms:0 allocation score on rhel7-1: -INFINITY
eae27e
+native_color: lxc-ms:0 allocation score on rhel7-2: 0
eae27e
+native_color: lxc-ms:0 allocation score on rhel7-3: 0
eae27e
+native_color: lxc-ms:0 allocation score on rhel7-4: 0
eae27e
+native_color: lxc-ms:0 allocation score on rhel7-5: 0
eae27e
+native_color: lxc-ms:1 allocation score on lxc1: -INFINITY
eae27e
+native_color: lxc-ms:1 allocation score on lxc2: INFINITY
eae27e
+native_color: lxc-ms:1 allocation score on rhel7-1: -INFINITY
eae27e
+native_color: lxc-ms:1 allocation score on rhel7-2: 0
eae27e
+native_color: lxc-ms:1 allocation score on rhel7-3: 0
eae27e
+native_color: lxc-ms:1 allocation score on rhel7-4: 0
eae27e
+native_color: lxc-ms:1 allocation score on rhel7-5: 0
eae27e
+native_color: lxc1 allocation score on lxc1: -INFINITY
eae27e
+native_color: lxc1 allocation score on lxc2: -INFINITY
eae27e
+native_color: lxc1 allocation score on rhel7-1: -INFINITY
eae27e
+native_color: lxc1 allocation score on rhel7-2: 0
eae27e
+native_color: lxc1 allocation score on rhel7-3: -INFINITY
eae27e
+native_color: lxc1 allocation score on rhel7-4: -INFINITY
eae27e
+native_color: lxc1 allocation score on rhel7-5: -INFINITY
eae27e
+native_color: lxc2 allocation score on lxc1: -INFINITY
eae27e
+native_color: lxc2 allocation score on lxc2: -INFINITY
eae27e
+native_color: lxc2 allocation score on rhel7-1: -INFINITY
eae27e
+native_color: lxc2 allocation score on rhel7-2: -INFINITY
eae27e
+native_color: lxc2 allocation score on rhel7-3: 0
eae27e
+native_color: lxc2 allocation score on rhel7-4: -INFINITY
eae27e
+native_color: lxc2 allocation score on rhel7-5: -INFINITY
eae27e
+native_color: rsc_rhel7-1 allocation score on lxc1: -INFINITY
eae27e
+native_color: rsc_rhel7-1 allocation score on lxc2: -INFINITY
eae27e
+native_color: rsc_rhel7-1 allocation score on rhel7-1: 100
eae27e
+native_color: rsc_rhel7-1 allocation score on rhel7-2: 0
eae27e
+native_color: rsc_rhel7-1 allocation score on rhel7-3: 0
eae27e
+native_color: rsc_rhel7-1 allocation score on rhel7-4: 0
eae27e
+native_color: rsc_rhel7-1 allocation score on rhel7-5: 0
eae27e
diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary
eae27e
new file mode 100644
eae27e
index 0000000..8a1bfd4
eae27e
--- /dev/null
eae27e
+++ b/pengine/test10/guest-node-host-dies.summary
eae27e
@@ -0,0 +1,73 @@
eae27e
+
eae27e
+Current cluster status:
eae27e
+Node rhel7-1 (1): UNCLEAN (offline)
eae27e
+Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
eae27e
+Containers: [ lxc1:container1 lxc2:container2 ]
eae27e
+
eae27e
+ Fencing	(stonith:fence_xvm):	Started rhel7-4 
eae27e
+ rsc_rhel7-1	(ocf::heartbeat:IPaddr2):	Started rhel7-1 ( UNCLEAN ) 
eae27e
+ container1	(ocf::heartbeat:VirtualDomain):	Started rhel7-1 ( UNCLEAN ) 
eae27e
+ container2	(ocf::heartbeat:VirtualDomain):	Started rhel7-1 ( UNCLEAN ) 
eae27e
+ Master/Slave Set: lxc-ms-master [lxc-ms]
eae27e
+     Masters: [ lxc1 ]
eae27e
+     Slaves: [ lxc2 ]
eae27e
+
eae27e
+Transition Summary:
eae27e
+ * Restart Fencing	(Started rhel7-4)
eae27e
+ * Move    rsc_rhel7-1	(Started rhel7-1 -> rhel7-5)
eae27e
+ * Move    container1	(Started rhel7-1 -> rhel7-2)
eae27e
+ * Move    container2	(Started rhel7-1 -> rhel7-3)
eae27e
+ * Restart lxc-ms:0	(Master lxc1)
eae27e
+ * Restart lxc-ms:1	(Slave lxc2)
eae27e
+ * Move    lxc1	(Started rhel7-1 -> rhel7-2)
eae27e
+ * Move    lxc2	(Started rhel7-1 -> rhel7-3)
eae27e
+
eae27e
+Executing cluster transition:
eae27e
+ * Resource action: Fencing         stop on rhel7-4
eae27e
+ * Resource action: Fencing         start on rhel7-4
eae27e
+ * Resource action: Fencing         monitor=120000 on rhel7-4
eae27e
+ * Pseudo action:   lxc-ms-master_demote_0
eae27e
+ * Fencing rhel7-1 (reboot)
eae27e
+ * Pseudo action:   stonith_complete
eae27e
+ * Pseudo action:   rsc_rhel7-1_stop_0
eae27e
+ * Resource action: lxc-ms          demote on lxc1
eae27e
+ * Pseudo action:   lxc-ms-master_demoted_0
eae27e
+ * Pseudo action:   lxc-ms-master_stop_0
eae27e
+ * Pseudo action:   lxc1_stop_0
eae27e
+ * Pseudo action:   lxc2_stop_0
eae27e
+ * Resource action: rsc_rhel7-1     start on rhel7-5
eae27e
+ * Pseudo action:   container1_stop_0
eae27e
+ * Pseudo action:   container2_stop_0
eae27e
+ * Pseudo action:   lxc-ms_stop_0
eae27e
+ * Pseudo action:   lxc-ms_stop_0
eae27e
+ * Pseudo action:   lxc-ms-master_stopped_0
eae27e
+ * Pseudo action:   lxc-ms-master_start_0
eae27e
+ * Pseudo action:   all_stopped
eae27e
+ * Resource action: rsc_rhel7-1     monitor=5000 on rhel7-5
eae27e
+ * Resource action: container1      start on rhel7-2
eae27e
+ * Resource action: container2      start on rhel7-3
eae27e
+ * Resource action: lxc1            start on rhel7-2
eae27e
+ * Resource action: lxc2            start on rhel7-3
eae27e
+ * Resource action: lxc-ms          start on lxc1
eae27e
+ * Resource action: lxc-ms          start on lxc2
eae27e
+ * Resource action: lxc-ms          monitor=10000 on lxc2
eae27e
+ * Pseudo action:   lxc-ms-master_running_0
eae27e
+ * Resource action: lxc1            monitor=30000 on rhel7-2
eae27e
+ * Resource action: lxc2            monitor=30000 on rhel7-3
eae27e
+ * Pseudo action:   lxc-ms-master_promote_0
eae27e
+ * Resource action: lxc-ms          promote on lxc1
eae27e
+ * Pseudo action:   lxc-ms-master_promoted_0
eae27e
+
eae27e
+Revised cluster status:
eae27e
+Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]
eae27e
+OFFLINE: [ rhel7-1 ]
eae27e
+Containers: [ lxc1:container1 lxc2:container2 ]
eae27e
+
eae27e
+ Fencing	(stonith:fence_xvm):	Started rhel7-4 
eae27e
+ rsc_rhel7-1	(ocf::heartbeat:IPaddr2):	Started rhel7-5 
eae27e
+ container1	(ocf::heartbeat:VirtualDomain):	Started rhel7-2 
eae27e
+ container2	(ocf::heartbeat:VirtualDomain):	Started rhel7-3 
eae27e
+ Master/Slave Set: lxc-ms-master [lxc-ms]
eae27e
+     Masters: [ lxc1 ]
eae27e
+     Slaves: [ lxc2 ]
eae27e
+
eae27e
diff --git a/pengine/test10/guest-node-host-dies.xml b/pengine/test10/guest-node-host-dies.xml
eae27e
new file mode 100644
eae27e
index 0000000..a840da1
eae27e
--- /dev/null
eae27e
+++ b/pengine/test10/guest-node-host-dies.xml
eae27e
@@ -0,0 +1,294 @@
eae27e
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.4" epoch="548" num_updates="7" admin_epoch="0" cib-last-written="Wed Mar 23 15:52:27 2016" update-origin="rhel7-2" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="4">
eae27e
+  <configuration>
eae27e
+    <crm_config>
eae27e
+      <cluster_property_set id="cib-bootstrap-options">
eae27e
+        <nvpair id="cts-stonith-enabled" name="stonith-enabled" value="1"/>
eae27e
+        <nvpair id="cts-no-quorum-policy" name="no-quorum-policy" value="stop"/>
eae27e
+        <nvpair id="cts-expected-quorum-votes" name="expected-quorum-votes" value="5"/>
eae27e
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.14-311.el7.centos-d04818f"/>
eae27e
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
eae27e
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="mycluster"/>
eae27e
+        <nvpair id="cib-bootstrap-options-last-lrm-refresh" name="last-lrm-refresh" value="1458766341"/>
eae27e
+      </cluster_property_set>
eae27e
+    </crm_config>
eae27e
+    <nodes>
eae27e
+      <node id="1" uname="rhel7-1"/>
eae27e
+      <node id="2" uname="rhel7-2"/>
eae27e
+      <node id="3" uname="rhel7-5"/>
eae27e
+      <node id="4" uname="rhel7-3"/>
eae27e
+      <node id="5" uname="rhel7-4"/>
eae27e
+    </nodes>
eae27e
+    <resources>
eae27e
+      <primitive class="stonith" id="Fencing" type="fence_xvm">
eae27e
+        <meta_attributes id="Fencing-meta">
eae27e
+          <nvpair id="Fencing-migration-threshold" name="migration-threshold" value="5"/>
eae27e
+        </meta_attributes>
eae27e
+        <instance_attributes id="Fencing-params">
eae27e
+          <nvpair id="Fencing-delay" name="delay" value="0"/>
eae27e
+          <nvpair id="Fencing-multicast_address" name="multicast_address" value="239.255.100.100"/>
eae27e
+          <nvpair id="Fencing-pcmk_arg_map" name="pcmk_arg_map" value="domain:uname"/>
eae27e
+          <nvpair id="Fencing-pcmk_host_list" name="pcmk_host_list" value="rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5"/>
eae27e
+        </instance_attributes>
eae27e
+        <operations>
eae27e
+          <op id="Fencing-monitor-120s" interval="120s" name="monitor" timeout="120s"/>
eae27e
+          <op id="Fencing-stop-0" interval="0" name="stop" timeout="60s"/>
eae27e
+          <op id="Fencing-start-0" interval="0" name="start" timeout="60s"/>
eae27e
+        </operations>
eae27e
+      </primitive>
eae27e
+      <primitive class="ocf" id="rsc_rhel7-1" provider="heartbeat" type="IPaddr2">
eae27e
+        <instance_attributes id="rsc_rhel7-1-params">
eae27e
+          <nvpair id="rsc_rhel7-1-ip" name="ip" value="192.168.122.202"/>
eae27e
+          <nvpair id="rsc_rhel7-1-cidr_netmask" name="cidr_netmask" value="32"/>
eae27e
+        </instance_attributes>
eae27e
+        <operations>
eae27e
+          <op id="rsc_rhel7-1-monitor-5s" interval="5s" name="monitor"/>
eae27e
+        </operations>
eae27e
+      </primitive>
eae27e
+      <primitive class="ocf" id="container1" provider="heartbeat" type="VirtualDomain">
eae27e
+        <instance_attributes id="container1-instance_attributes">
eae27e
+          <nvpair id="container1-instance_attributes-force_stop" name="force_stop" value="true"/>
eae27e
+          <nvpair id="container1-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
eae27e
+          <nvpair id="container1-instance_attributes-config" name="config" value="/var/lib/pacemaker/cts/lxc/lxc1.xml"/>
eae27e
+        </instance_attributes>
eae27e
+        <utilization id="container1-utilization">
eae27e
+          <nvpair id="container1-utilization-cpu" name="cpu" value="1"/>
eae27e
+          <nvpair id="container1-utilization-hv_memory" name="hv_memory" value="196"/>
eae27e
+        </utilization>
eae27e
+        <meta_attributes id="container1-meta_attributes">
eae27e
+          <nvpair id="container1-meta_attributes-remote-node" name="remote-node" value="lxc1"/>
eae27e
+        </meta_attributes>
eae27e
+      </primitive>
eae27e
+      <primitive class="ocf" id="container2" provider="heartbeat" type="VirtualDomain">
eae27e
+        <instance_attributes id="container2-instance_attributes">
eae27e
+          <nvpair id="container2-instance_attributes-force_stop" name="force_stop" value="true"/>
eae27e
+          <nvpair id="container2-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>
eae27e
+          <nvpair id="container2-instance_attributes-config" name="config" value="/var/lib/pacemaker/cts/lxc/lxc2.xml"/>
eae27e
+        </instance_attributes>
eae27e
+        <utilization id="container2-utilization">
eae27e
+          <nvpair id="container2-utilization-cpu" name="cpu" value="1"/>
eae27e
+          <nvpair id="container2-utilization-hv_memory" name="hv_memory" value="196"/>
eae27e
+        </utilization>
eae27e
+        <meta_attributes id="container2-meta_attributes">
eae27e
+          <nvpair id="container2-meta_attributes-remote-node" name="remote-node" value="lxc2"/>
eae27e
+        </meta_attributes>
eae27e
+      </primitive>
eae27e
+      <master id="lxc-ms-master">
eae27e
+        <primitive class="ocf" id="lxc-ms" provider="pacemaker" type="Stateful">
eae27e
+          <instance_attributes id="lxc-ms-instance_attributes"/>
eae27e
+          <operations>
eae27e
+            <op id="lxc-ms-monitor-interval-10s" interval="10s" name="monitor"/>
eae27e
+          </operations>
eae27e
+        </primitive>
eae27e
+        <meta_attributes id="lxc-ms-meta_attributes">
eae27e
+          <nvpair id="lxc-ms-meta_attributes-master-max" name="master-max" value="1"/>
eae27e
+          <nvpair id="lxc-ms-meta_attributes-clone-max" name="clone-max" value="2"/>
eae27e
+        </meta_attributes>
eae27e
+        <meta_attributes id="lxc-ms-master-meta_attributes"/>
eae27e
+      </master>
eae27e
+    </resources>
eae27e
+    <constraints>
eae27e
+      <rsc_location id="prefer-rhel7-1" rsc="rsc_rhel7-1">
eae27e
+        <rule boolean-op="and" id="prefer-rhel7-1-r" score="100">
eae27e
+          <expression attribute="#uname" id="prefer-rhel7-1-e" operation="eq" value="rhel7-1"/>
eae27e
+        </rule>
eae27e
+      </rsc_location>
eae27e
+      <rsc_location id="cli-prefer-container1" rsc="container1" role="Started" node="rhel7-1" score="INFINITY"/>
eae27e
+      <rsc_location id="lxc-rsc_rhel7-1-location-lxc1" node="lxc1" rsc="rsc_rhel7-1" score="-INFINITY"/>
eae27e
+      <rsc_location id="cli-prefer-container2" rsc="container2" role="Started" node="rhel7-1" score="INFINITY"/>
eae27e
+      <rsc_location id="lxc-rsc_rhel7-1-location-lxc2" node="lxc2" rsc="rsc_rhel7-1" score="-INFINITY"/>
eae27e
+      <rsc_location id="lxc-ms-location-lxc1" node="lxc1" rsc="lxc-ms-master" score="INFINITY"/>
eae27e
+      <rsc_location id="lxc-ms-location-lxc2" node="lxc2" rsc="lxc-ms-master" score="INFINITY"/>
eae27e
+    </constraints>
eae27e
+    <fencing-topology />
eae27e
+    <rsc_defaults>
eae27e
+      <meta_attributes id="rsc_defaults-options"/>
eae27e
+    </rsc_defaults>
eae27e
+  </configuration>
eae27e
+  <status>
eae27e
+    <node_state id="1" uname="rhel7-1" in_ccm="false" crmd="offline" crm-debug-origin="do_update_resource" join="down" expected="down">
eae27e
+      <transient_attributes id="1">
eae27e
+        <instance_attributes id="status-1">
eae27e
+          <nvpair id="status-1-shutdown" name="shutdown" value="0"/>
eae27e
+          <nvpair id="status-1-connected" name="connected" value="1"/>
eae27e
+        </instance_attributes>
eae27e
+      </transient_attributes>
eae27e
+      <lrm id="1">
eae27e
+        <lrm_resources>
eae27e
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
eae27e
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="24:28:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;24:28:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1458766249" last-rc-change="1458766249" exec-time="1" queue-time="0" op-digest="7d040e40a8b432739117ea9cba31edde"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="rsc_rhel7-1" type="IPaddr2" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="rsc_rhel7-1_last_0" operation_key="rsc_rhel7-1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="47:29:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;47:29:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="26" rc-code="0" op-status="0" interval="0" last-run="1458766251" last-rc-change="1458766251" exec-time="66" queue-time="0" op-digest="75df1567eb9457f8f3c4486bbf875846"/>
eae27e
+            <lrm_rsc_op id="rsc_rhel7-1_monitor_5000" operation_key="rsc_rhel7-1_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="48:29:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;48:29:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="31" rc-code="0" op-status="0" interval="5000" last-rc-change="1458766251" exec-time="51" queue-time="0" op-digest="045c3d6f5e29b94dc4e3fbfd6c2c0693"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container1_last_0" operation_key="container1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="86:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;86:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="83" rc-code="0" op-status="0" interval="0" last-run="1458766273" last-rc-change="1458766273" exec-time="215" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container2_last_0" operation_key="container2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="96:30:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;96:30:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="80" rc-code="0" op-status="0" interval="0" last-run="1458766270" last-rc-change="1458766270" exec-time="302" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
eae27e
+            <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="104:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;104:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="1" rc-code="0" op-status="0" interval="0" last-run="1458766271" last-rc-change="1458766271" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+            <lrm_rsc_op id="lxc2_monitor_30000" operation_key="lxc2_monitor_30000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="107:32:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;107:32:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="4" rc-code="0" op-status="0" interval="30000" last-rc-change="1458766278" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
eae27e
+            <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="102:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;102:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="2" rc-code="0" op-status="0" interval="0" last-run="1458766273" last-rc-change="1458766273" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+            <lrm_rsc_op id="lxc1_monitor_30000" operation_key="lxc1_monitor_30000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="104:32:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;104:32:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="3" rc-code="0" op-status="0" interval="30000" last-rc-change="1458766278" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
eae27e
+            <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="25:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;25:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="104" rc-code="7" op-status="0" interval="0" last-run="1458766341" last-rc-change="1458766341" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+          </lrm_resource>
eae27e
+        </lrm_resources>
eae27e
+      </lrm>
eae27e
+    </node_state>
eae27e
+    <node_state id="2" uname="rhel7-2" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
eae27e
+      <transient_attributes id="2">
eae27e
+        <instance_attributes id="status-2">
eae27e
+          <nvpair id="status-2-shutdown" name="shutdown" value="0"/>
eae27e
+          <nvpair id="status-2-connected" name="connected" value="1"/>
eae27e
+        </instance_attributes>
eae27e
+      </transient_attributes>
eae27e
+      <lrm id="2">
eae27e
+        <lrm_resources>
eae27e
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
eae27e
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="16:10:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;16:10:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-2" call-id="88" rc-code="0" op-status="0" interval="0" last-run="1458754535" last-rc-change="1458754535" exec-time="1" queue-time="0" op-digest="7d040e40a8b432739117ea9cba31edde"/>
eae27e
+            <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:5:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;28:5:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-2" call-id="86" rc-code="0" op-status="0" interval="120000" last-rc-change="1458754392" exec-time="28" queue-time="0" op-digest="20020ad5e6388d0c273cdb9c0ff35f5b"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="rsc_rhel7-1" type="IPaddr2" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="rsc_rhel7-1_last_0" operation_key="rsc_rhel7-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="20:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;20:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-2" call-id="17" rc-code="7" op-status="0" interval="0" last-run="1458754317" last-rc-change="1458754317" exec-time="33" queue-time="0" op-digest="75df1567eb9457f8f3c4486bbf875846"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container1_last_0" operation_key="container1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="85:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;85:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-2" call-id="105" rc-code="0" op-status="0" interval="0" last-run="1458766271" last-rc-change="1458766271" exec-time="290" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="26:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;26:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-2" call-id="79" rc-code="7" op-status="0" interval="0" last-run="1458754388" last-rc-change="1458754388" exec-time="283" queue-time="0" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="lxc1" type="remote" class="ocf" provider="pacemaker" container="container1">
eae27e
+            <lrm_rsc_op id="lxc1_last_0" operation_key="lxc1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="101:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;101:31:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-2" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1458766271" last-rc-change="1458766271" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+            <lrm_rsc_op id="lxc1_monitor_30000" operation_key="lxc1_monitor_30000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="97:13:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;97:13:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-2" call-id="2" rc-code="0" op-status="0" interval="30000" last-rc-change="1458754547" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
eae27e
+            <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="26:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;26:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-2" call-id="123" rc-code="7" op-status="0" interval="0" last-run="1458766341" last-rc-change="1458766341" exec-time="9" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+          </lrm_resource>
eae27e
+        </lrm_resources>
eae27e
+      </lrm>
eae27e
+    </node_state>
eae27e
+    <node_state id="4" uname="rhel7-3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
eae27e
+      <transient_attributes id="4">
eae27e
+        <instance_attributes id="status-4">
eae27e
+          <nvpair id="status-4-shutdown" name="shutdown" value="0"/>
eae27e
+          <nvpair id="status-4-connected" name="connected" value="1"/>
eae27e
+        </instance_attributes>
eae27e
+      </transient_attributes>
eae27e
+      <lrm id="4">
eae27e
+        <lrm_resources>
eae27e
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
eae27e
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="32:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;32:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1458754314" last-rc-change="1458754314" exec-time="2" queue-time="0" op-digest="7d040e40a8b432739117ea9cba31edde"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container2_last_0" operation_key="container2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="95:30:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;95:30:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-3" call-id="91" rc-code="0" op-status="0" interval="0" last-run="1458766269" last-rc-change="1458766269" exec-time="281" queue-time="1" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="lxc2" type="remote" class="ocf" provider="pacemaker" container="container2">
eae27e
+            <lrm_rsc_op id="lxc2_last_0" operation_key="lxc2_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="113:30:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;113:30:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-3" call-id="3" rc-code="0" op-status="0" interval="0" last-run="1458766269" last-rc-change="1458766269" exec-time="0" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8" op-force-restart=" server " op-restart-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+            <lrm_rsc_op id="lxc2_monitor_30000" operation_key="lxc2_monitor_30000" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="100:13:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;100:13:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-3" call-id="2" rc-code="0" op-status="0" interval="30000" last-rc-change="1458754547" exec-time="0" queue-time="0" op-digest="02a5bcf940fc8d3239701acb11438d6a"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="rsc_rhel7-1" type="IPaddr2" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="rsc_rhel7-1_last_0" operation_key="rsc_rhel7-1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="35:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;35:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-3" call-id="17" rc-code="7" op-status="0" interval="0" last-run="1458754316" last-rc-change="1458754316" exec-time="43" queue-time="0" op-digest="75df1567eb9457f8f3c4486bbf875846"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="build_active_RAs" crm_feature_set="3.0.10" transition-key="27:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;27:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-3" call-id="75" rc-code="7" op-status="0" interval="0" last-run="1458754388" last-rc-change="1458754388" exec-time="141" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
eae27e
+            <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="27:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;27:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-3" call-id="109" rc-code="7" op-status="0" interval="0" last-run="1458766341" last-rc-change="1458766341" exec-time="33" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+          </lrm_resource>
eae27e
+        </lrm_resources>
eae27e
+      </lrm>
eae27e
+    </node_state>
eae27e
+    <node_state id="5" uname="rhel7-4" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
eae27e
+      <transient_attributes id="5">
eae27e
+        <instance_attributes id="status-5">
eae27e
+          <nvpair id="status-5-shutdown" name="shutdown" value="0"/>
eae27e
+          <nvpair id="status-5-connected" name="connected" value="1"/>
eae27e
+        </instance_attributes>
eae27e
+      </transient_attributes>
eae27e
+      <lrm id="5">
eae27e
+        <lrm_resources>
eae27e
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
eae27e
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="14:11:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;14:11:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-4" call-id="86" rc-code="0" op-status="0" interval="0" last-run="1458754535" last-rc-change="1458754535" exec-time="25" queue-time="0" op-digest="7d040e40a8b432739117ea9cba31edde"/>
eae27e
+            <lrm_rsc_op id="Fencing_monitor_120000" operation_key="Fencing_monitor_120000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="15:11:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;15:11:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-4" call-id="87" rc-code="0" op-status="0" interval="120000" last-rc-change="1458754535" exec-time="28" queue-time="0" op-digest="20020ad5e6388d0c273cdb9c0ff35f5b"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="rsc_rhel7-1" type="IPaddr2" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="rsc_rhel7-1_last_0" operation_key="rsc_rhel7-1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="50:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;50:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-4" call-id="17" rc-code="7" op-status="0" interval="0" last-run="1458754316" last-rc-change="1458754316" exec-time="33" queue-time="0" op-digest="75df1567eb9457f8f3c4486bbf875846"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="29:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;29:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-4" call-id="75" rc-code="7" op-status="0" interval="0" last-run="1458754388" last-rc-change="1458754388" exec-time="184" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="30:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;30:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-4" call-id="79" rc-code="7" op-status="0" interval="0" last-run="1458754388" last-rc-change="1458754388" exec-time="180" queue-time="1" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
eae27e
+            <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="28:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;28:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-4" call-id="106" rc-code="7" op-status="0" interval="0" last-run="1458766341" last-rc-change="1458766341" exec-time="15" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+          </lrm_resource>
eae27e
+        </lrm_resources>
eae27e
+      </lrm>
eae27e
+    </node_state>
eae27e
+    <node_state id="3" uname="rhel7-5" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
eae27e
+      <transient_attributes id="3">
eae27e
+        <instance_attributes id="status-3">
eae27e
+          <nvpair id="status-3-shutdown" name="shutdown" value="0"/>
eae27e
+          <nvpair id="status-3-connected" name="connected" value="1"/>
eae27e
+        </instance_attributes>
eae27e
+      </transient_attributes>
eae27e
+      <lrm id="3">
eae27e
+        <lrm_resources>
eae27e
+          <lrm_resource id="Fencing" type="fence_xvm" class="stonith">
eae27e
+            <lrm_rsc_op id="Fencing_last_0" operation_key="Fencing_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="62:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;62:0:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-5" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1458754314" last-rc-change="1458754314" exec-time="2" queue-time="0" op-digest="7d040e40a8b432739117ea9cba31edde"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="rsc_rhel7-1" type="IPaddr2" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="rsc_rhel7-1_last_0" operation_key="rsc_rhel7-1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="46:29:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;46:29:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-5" call-id="87" rc-code="0" op-status="0" interval="0" last-run="1458766251" last-rc-change="1458766251" exec-time="42" queue-time="0" op-digest="75df1567eb9457f8f3c4486bbf875846"/>
eae27e
+            <lrm_rsc_op id="rsc_rhel7-1_monitor_5000" operation_key="rsc_rhel7-1_monitor_5000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="21:11:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;21:11:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-5" call-id="85" rc-code="0" op-status="0" interval="5000" last-rc-change="1458754535" exec-time="81" queue-time="0" op-digest="045c3d6f5e29b94dc4e3fbfd6c2c0693"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container1" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container1_last_0" operation_key="container1_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="31:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;31:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-5" call-id="73" rc-code="7" op-status="0" interval="0" last-run="1458754388" last-rc-change="1458754388" exec-time="319" queue-time="0" op-digest="edbb69efbcbe9c588c5d34e36db6e16d"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="container2" type="VirtualDomain" class="ocf" provider="heartbeat">
eae27e
+            <lrm_rsc_op id="container2_last_0" operation_key="container2_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="32:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;32:4:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-5" call-id="77" rc-code="7" op-status="0" interval="0" last-run="1458754388" last-rc-change="1458754388" exec-time="311" queue-time="1" op-digest="011f8a90c12be82054eaf7a034fc4062"/>
eae27e
+          </lrm_resource>
eae27e
+          <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
eae27e
+            <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="29:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:7;29:59:7:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-5" call-id="105" rc-code="7" op-status="0" interval="0" last-run="1458766341" last-rc-change="1458766341" exec-time="14" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+          </lrm_resource>
eae27e
+        </lrm_resources>
eae27e
+      </lrm>
eae27e
+    </node_state>
eae27e
+    <node_state remote_node="true" id="lxc1" uname="lxc1" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
eae27e
+      <lrm id="lxc1">
eae27e
+        <lrm_resources>
eae27e
+          <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
eae27e
+            <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_promote_0" operation="promote" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="95:61:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;95:61:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="162" rc-code="0" op-status="0" interval="0" last-run="1458766349" last-rc-change="1458766349" exec-time="310" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+          </lrm_resource>
eae27e
+        </lrm_resources>
eae27e
+      </lrm>
eae27e
+      <transient_attributes id="lxc1">
eae27e
+        <instance_attributes id="status-lxc1">
eae27e
+          <nvpair id="status-lxc1-last-failure-lxc-ms" name="last-failure-lxc-ms" value="1458766329"/>
eae27e
+          <nvpair id="status-lxc1-master-lxc-ms" name="master-lxc-ms" value="10"/>
eae27e
+        </instance_attributes>
eae27e
+      </transient_attributes>
eae27e
+    </node_state>
eae27e
+    <node_state remote_node="true" id="lxc2" uname="lxc2" in_ccm="true" crm-debug-origin="do_update_resource" node_fenced="0">
eae27e
+      <transient_attributes id="lxc2">
eae27e
+        <instance_attributes id="status-lxc2">
eae27e
+          <nvpair id="status-lxc2-master-lxc-ms" name="master-lxc-ms" value="5"/>
eae27e
+        </instance_attributes>
eae27e
+      </transient_attributes>
eae27e
+      <lrm id="lxc2">
eae27e
+        <lrm_resources>
eae27e
+          <lrm_resource id="lxc-ms" type="Stateful" class="ocf" provider="pacemaker">
eae27e
+            <lrm_rsc_op id="lxc-ms_last_0" operation_key="lxc-ms_start_0" operation="start" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="95:60:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;95:60:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="70" rc-code="0" op-status="0" interval="0" last-run="1458766348" last-rc-change="1458766348" exec-time="381" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
eae27e
+            <lrm_rsc_op id="lxc-ms_monitor_10000" operation_key="lxc-ms_monitor_10000" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="98:61:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" transition-magic="0:0;98:61:0:f82b0474-2677-4f4b-8ec6-afdc11f4c949" on_node="rhel7-1" call-id="79" rc-code="0" op-status="0" interval="10000" last-rc-change="1458766349" exec-time="9" queue-time="0" op-digest="8f6a313464b7f9e3a31cb448458b700e"/>
eae27e
+          </lrm_resource>
eae27e
+        </lrm_resources>
eae27e
+      </lrm>
eae27e
+    </node_state>
eae27e
+  </status>
eae27e
+</cib>
eae27e
-- 
eae27e
1.8.3.1
eae27e
eae27e
eae27e
From beab7718e14a54f1b50d7c5ff4b0086e09332da3 Mon Sep 17 00:00:00 2001
eae27e
From: Ken Gaillot <kgaillot@redhat.com>
eae27e
Date: Fri, 15 Apr 2016 13:10:17 -0500
eae27e
Subject: [PATCH 5/6] Fix: pengine: create a pseudo-fence for guest node
eae27e
 recovery
eae27e
eae27e
If a guest node needs to be recovered, the PE would previously order actions
eae27e
in relation to the stop action for the guest's container resource, if one
eae27e
was scheduled.
eae27e
eae27e
This had problems: for implied stops due to fencing the guest's host, there
eae27e
would be no stop action, so no ordering could be done; ordering in relation to
eae27e
the stop action made stonith_constraints() mistakenly assume that the host node
eae27e
(the node for the stop action) was the fence target, and thus mistakenly mark
eae27e
the wrong stops/demotes as implied; and, clone notifications for fence events
eae27e
would not get called for guest node recoveries, whether explicit or implied.
eae27e
eae27e
Now, a fence pseudo-event is created for guest node recovery, regardless of
eae27e
whether there is an explicit stop action scheduled for the container. This
eae27e
addresses all those issues, and will allow the crmd to be able to detect
eae27e
implied stops.
eae27e
eae27e
This also allows us to simplify the implied stop/demote detection, since we
eae27e
will check the pseudo-op for implied actions -- we don't need to check the
eae27e
real fence op for implied actions on guest nodes.
eae27e
---
eae27e
 crmd/te_utils.c    |  8 ++++++
eae27e
 pengine/allocate.c | 82 ++++++++++++++++++++++++++++++++++++++++++++----------
eae27e
 pengine/graph.c    | 14 ++++------
eae27e
 pengine/native.c   | 48 ++------------------------------
eae27e
 4 files changed, 83 insertions(+), 69 deletions(-)
eae27e
eae27e
diff --git a/crmd/te_utils.c b/crmd/te_utils.c
eae27e
index 4c708a1..e7bf7ff 100644
eae27e
--- a/crmd/te_utils.c
eae27e
+++ b/crmd/te_utils.c
eae27e
@@ -331,6 +331,14 @@ tengine_stonith_notify(stonith_t * st, stonith_event_t * st_event)
eae27e
             /* The DC always sends updates */
eae27e
             send_stonith_update(NULL, st_event->target, uuid);
eae27e
 
eae27e
+            /* @TODO Ideally, at this point, we'd check whether the fenced node
eae27e
+             * hosted any guest nodes, and call remote_node_down() for them.
eae27e
+             * Unfortunately, the crmd doesn't have a simple, reliable way to
eae27e
+             * map hosts to guests. It might be possible to track this in the
eae27e
+             * peer cache via crm_remote_peer_cache_refresh(). For now, we rely
eae27e
+             * on the PE creating fence pseudo-events for the guests.
eae27e
+             */
eae27e
+
eae27e
             if (st_event->client_origin && safe_str_neq(st_event->client_origin, te_client_id)) {
eae27e
 
eae27e
                 /* Abort the current transition graph if it wasn't us
eae27e
diff --git a/pengine/allocate.c b/pengine/allocate.c
eae27e
index bdf03e5..74b57fb 100644
eae27e
--- a/pengine/allocate.c
eae27e
+++ b/pengine/allocate.c
eae27e
@@ -1341,6 +1341,70 @@ any_managed_resources(pe_working_set_t * data_set)
eae27e
     return FALSE;
eae27e
 }
eae27e
 
eae27e
+/*!
eae27e
+ * \internal
eae27e
+ * \brief Create pseudo-op for guest node fence, and order relative to it
eae27e
+ *
eae27e
+ * \param[in] node      Guest node to fence
eae27e
+ * \param[in] done      STONITH_DONE operation
eae27e
+ * \param[in] data_set  Working set of CIB state
eae27e
+ */
eae27e
+static void
eae27e
+fence_guest(pe_node_t *node, pe_action_t *done, pe_working_set_t *data_set)
eae27e
+{
eae27e
+    resource_t *container = node->details->remote_rsc->container;
eae27e
+    pe_action_t *stop = NULL;
eae27e
+    pe_action_t *stonith_op = NULL;
eae27e
+
eae27e
+    /* The fence action is just a label; we don't do anything differently for
eae27e
+     * off vs. reboot. We specify it explicitly, rather than let it default to
eae27e
+     * cluster's default action, because we are not _initiating_ fencing -- we
eae27e
+     * are creating a pseudo-event to describe fencing that is already occurring
eae27e
+     * by other means (container recovery).
eae27e
+     */
eae27e
+    const char *fence_action = "off";
eae27e
+
eae27e
+    /* Check whether guest's container resource is has any explicit stop or
eae27e
+     * start (the stop may be implied by fencing of the guest's host).
eae27e
+     */
eae27e
+    if (container) {
eae27e
+        stop = find_first_action(container->actions, NULL, CRMD_ACTION_STOP, NULL);
eae27e
+
eae27e
+        if (find_first_action(container->actions, NULL, CRMD_ACTION_START, NULL)) {
eae27e
+            fence_action = "reboot";
eae27e
+        }
eae27e
+    }
eae27e
+
eae27e
+    /* Create a fence pseudo-event, so we have an event to order actions
eae27e
+     * against, and crmd can always detect it.
eae27e
+     */
eae27e
+    stonith_op = pe_fence_op(node, fence_action, FALSE, data_set);
eae27e
+    update_action_flags(stonith_op, pe_action_pseudo | pe_action_runnable);
eae27e
+
eae27e
+
eae27e
+    /* We want to imply stops/demotes after the guest is stopped, not wait until
eae27e
+     * it is restarted, so we always order pseudo-fencing after stop, not start
eae27e
+     * (even though start might be closer to what is done for a real reboot).
eae27e
+     */
eae27e
+    if (stop) {
eae27e
+        order_actions(stop, stonith_op,
eae27e
+                      pe_order_runnable_left|pe_order_implies_then);
eae27e
+        crm_info("Implying guest node %s is down (action %d) "
eae27e
+                 "after container %s is stopped (action %d)",
eae27e
+                 node->details->uname, stonith_op->id,
eae27e
+                 container->id, stop->id);
eae27e
+    } else {
eae27e
+        crm_info("Implying guest node %s is down (action %d) ",
eae27e
+                 node->details->uname, stonith_op->id);
eae27e
+    }
eae27e
+
eae27e
+    /* @TODO: Order pseudo-fence after any (optional) fence of guest's host */
eae27e
+
eae27e
+    /* Order/imply other actions relative to pseudo-fence as with real fence */
eae27e
+    stonith_constraints(node, stonith_op, data_set);
eae27e
+    order_actions(stonith_op, done, pe_order_implies_then);
eae27e
+}
eae27e
+
eae27e
 /*
eae27e
  * Create dependencies for stonith and shutdown operations
eae27e
  */
eae27e
@@ -1369,24 +1433,12 @@ stage6(pe_working_set_t * data_set)
eae27e
     for (gIter = data_set->nodes; gIter != NULL; gIter = gIter->next) {
eae27e
         node_t *node = (node_t *) gIter->data;
eae27e
 
eae27e
-        /* Guest nodes are "fenced" by recovering their container resource.
eae27e
-         * The container stop may be explicit, or implied by the fencing of the
eae27e
-         * guest's host.
eae27e
+        /* Guest nodes are "fenced" by recovering their container resource,
eae27e
+         * so handle them separately.
eae27e
          */
eae27e
         if (is_container_remote_node(node)) {
eae27e
             if (node->details->remote_requires_reset && need_stonith) {
eae27e
-                resource_t *container = node->details->remote_rsc->container;
eae27e
-                char *key = stop_key(container);
eae27e
-                GListPtr stop_list = find_actions(container->actions, key, NULL);
eae27e
-
eae27e
-                crm_info("Implying node %s is down when container %s is stopped (%p)",
eae27e
-                         node->details->uname, container->id, stop_list);
eae27e
-                if(stop_list) {
eae27e
-                    stonith_constraints(node, stop_list->data, data_set);
eae27e
-                }
eae27e
-
eae27e
-                g_list_free(stop_list);
eae27e
-                free(key);
eae27e
+                fence_guest(node, done, data_set);
eae27e
             }
eae27e
             continue;
eae27e
         }
eae27e
diff --git a/pengine/graph.c b/pengine/graph.c
eae27e
index ee7c7c8..569cf6e 100644
eae27e
--- a/pengine/graph.c
eae27e
+++ b/pengine/graph.c
eae27e
@@ -715,13 +715,7 @@ stonith_constraints(node_t * node, action_t * stonith_op, pe_working_set_t * dat
eae27e
 
eae27e
     CRM_CHECK(stonith_op != NULL, return FALSE);
eae27e
     for (r = data_set->resources; r != NULL; r = r->next) {
eae27e
-        resource_t *rsc = (resource_t *) r->data;
eae27e
-
eae27e
-        if ((stonith_op->rsc == NULL)
eae27e
-            || ((stonith_op->rsc != rsc) && (stonith_op->rsc != rsc->container))) {
eae27e
-
eae27e
-            rsc_stonith_ordering(rsc, stonith_op, data_set);
eae27e
-        }
eae27e
+        rsc_stonith_ordering((resource_t *) r->data, stonith_op, data_set);
eae27e
     }
eae27e
     return TRUE;
eae27e
 }
eae27e
@@ -888,7 +882,11 @@ action2xml(action_t * action, gboolean as_input, pe_working_set_t *data_set)
eae27e
     }
eae27e
 
eae27e
     if (safe_str_eq(action->task, CRM_OP_FENCE)) {
eae27e
-        action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
eae27e
+        /* All fences need node info; guest node fences are pseudo-events */
eae27e
+        action_xml = create_xml_node(NULL,
eae27e
+                                     is_set(action->flags, pe_action_pseudo)?
eae27e
+                                     XML_GRAPH_TAG_PSEUDO_EVENT :
eae27e
+                                     XML_GRAPH_TAG_CRM_EVENT);
eae27e
 
eae27e
     } else if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {
eae27e
         action_xml = create_xml_node(NULL, XML_GRAPH_TAG_CRM_EVENT);
eae27e
diff --git a/pengine/native.c b/pengine/native.c
eae27e
index 56a1434..ff4467b 100644
eae27e
--- a/pengine/native.c
eae27e
+++ b/pengine/native.c
eae27e
@@ -2902,48 +2902,6 @@ native_start_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set
eae27e
     }
eae27e
 }
eae27e
 
eae27e
-/* User data to pass to guest node iterator */
eae27e
-struct action_list_s {
eae27e
-    GListPtr search_list; /* list of actions to search */
eae27e
-    GListPtr result_list; /* list of matching actions for this node */
eae27e
-    const char *key;      /* action key to match */
eae27e
-};
eae27e
-
eae27e
-/*!
eae27e
- * \internal
eae27e
- * \brief Prepend a node's actions matching a key to a list
eae27e
- *
eae27e
- * \param[in]     node  Guest node
eae27e
- * \param[in/out] data   User data
eae27e
- */
eae27e
-static void prepend_node_actions(const node_t *node, void *data)
eae27e
-{
eae27e
-    GListPtr actions;
eae27e
-    struct action_list_s *info = (struct action_list_s *) data;
eae27e
-
eae27e
-    actions = find_actions(info->search_list, info->key, node);
eae27e
-    info->result_list = g_list_concat(actions, info->result_list);
eae27e
-}
eae27e
-
eae27e
-static GListPtr
eae27e
-find_fence_target_node_actions(GListPtr search_list, const char *key, node_t *fence_target, pe_working_set_t *data_set)
eae27e
-{
eae27e
-    struct action_list_s action_list;
eae27e
-
eae27e
-    /* Actions on the target that match the key are implied by the fencing */
eae27e
-    action_list.search_list = search_list;
eae27e
-    action_list.result_list = find_actions(search_list, key, fence_target);
eae27e
-    action_list.key = key;
eae27e
-
eae27e
-    /*
eae27e
-     * If the target is a host for any guest nodes, actions on those nodes
eae27e
-     * that match the key are also implied by the fencing.
eae27e
-     */
eae27e
-    pe_foreach_guest_node(data_set, fence_target, prepend_node_actions, &action_list);
eae27e
-
eae27e
-    return action_list.result_list;
eae27e
-}
eae27e
-
eae27e
 static void
eae27e
 native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
eae27e
 {
eae27e
@@ -2963,8 +2921,7 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_
eae27e
 
eae27e
     /* Get a list of stop actions potentially implied by the fencing */
eae27e
     key = stop_key(rsc);
eae27e
-    action_list = find_fence_target_node_actions(rsc->actions, key, target,
eae27e
-                                                 data_set);
eae27e
+    action_list = find_actions(rsc->actions, key, target);
eae27e
     free(key);
eae27e
 
eae27e
     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
eae27e
@@ -3061,8 +3018,7 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_
eae27e
 
eae27e
     /* Get a list of demote actions potentially implied by the fencing */
eae27e
     key = demote_key(rsc);
eae27e
-    action_list = find_fence_target_node_actions(rsc->actions, key, target,
eae27e
-                                                 data_set);
eae27e
+    action_list = find_actions(rsc->actions, key, target);
eae27e
     free(key);
eae27e
 
eae27e
     for (gIter = action_list; gIter != NULL; gIter = gIter->next) {
eae27e
-- 
eae27e
1.8.3.1
eae27e
eae27e
eae27e
From b7ce740edf3d71fcccead2288bf0ab11037f9672 Mon Sep 17 00:00:00 2001
eae27e
From: Ken Gaillot <kgaillot@redhat.com>
eae27e
Date: Wed, 23 Nov 2016 14:56:29 -0600
eae27e
Subject: [PATCH 6/6] Fix: pengine: consider guest node unclean if its host is
eae27e
 unclean
eae27e
eae27e
---
eae27e
 lib/pengine/unpack.c | 12 ++++++++++++
eae27e
 1 file changed, 12 insertions(+)
eae27e
eae27e
diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c
eae27e
index a49e108..6737273 100644
eae27e
--- a/lib/pengine/unpack.c
eae27e
+++ b/lib/pengine/unpack.c
eae27e
@@ -1406,6 +1406,7 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
eae27e
 {
eae27e
     resource_t *rsc = this_node->details->remote_rsc;
eae27e
     resource_t *container = NULL;
eae27e
+    pe_node_t *host = NULL;
eae27e
 
eae27e
     /* If there is a node state entry for a (former) Pacemaker Remote node
eae27e
      * but no resource creating that node, the node's connection resource will
eae27e
@@ -1418,6 +1419,10 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
eae27e
 
eae27e
     container = rsc->container;
eae27e
 
eae27e
+    if (container && (g_list_length(rsc->running_on) == 1)) {
eae27e
+        host = rsc->running_on->data;
eae27e
+    }
eae27e
+
eae27e
     /* If the resource is currently started, mark it online. */
eae27e
     if (rsc->role == RSC_ROLE_STARTED) {
eae27e
         crm_trace("%s node %s presumed ONLINE because connection resource is started",
eae27e
@@ -1451,6 +1456,13 @@ determine_remote_online_status(pe_working_set_t * data_set, node_t * this_node)
eae27e
                   (container? "Guest" : "Remote"), this_node->details->id);
eae27e
         this_node->details->online = FALSE;
eae27e
         this_node->details->remote_requires_reset = FALSE;
eae27e
+
eae27e
+    } else if (host && (host->details->online == FALSE)
eae27e
+               && host->details->unclean) {
eae27e
+        crm_trace("Guest node %s UNCLEAN because host is unclean",
eae27e
+                  this_node->details->id);
eae27e
+        this_node->details->online = FALSE;
eae27e
+        this_node->details->remote_requires_reset = TRUE;
eae27e
     }
eae27e
 
eae27e
 remote_online_done:
eae27e
-- 
eae27e
1.8.3.1
eae27e