From db1de472c1d1a4bef98b67f181a6e4fc8b5cbcfb Mon Sep 17 00:00:00 2001

From: Andrew Beekhof <andrew@beekhof.net>

Date: Mon, 29 May 2017 19:24:50 +1000

Subject: [PATCH] Log: PE: Improve logging of node fencing and shutdown

---

 lib/pengine/unpack.c                               | 58 ++++++++++++----------

 lib/pengine/utils.c                                |  2 +-

 pengine/allocate.c                                 | 47 ++++++++++++++++--

 pengine/allocate.h                                 |  3 +-

 pengine/native.c                                   |  4 +-

 pengine/test10/594.summary                         |  2 +

 pengine/test10/662.summary                         |  1 +

 pengine/test10/797.summary                         |  1 +

 pengine/test10/829.summary                         |  1 +

 pengine/test10/bug-1572-1.summary                  |  1 +

 pengine/test10/bug-1572-2.summary                  |  1 +

 pengine/test10/bug-1573.summary                    |  1 +

 pengine/test10/bug-1820-1.summary                  |  1 +

 pengine/test10/bug-1820.summary                    |  1 +

 pengine/test10/bug-1822.summary                    |  1 +

 pengine/test10/bug-5028-bottom.summary             |  1 +

 pengine/test10/bug-5028-detach.summary             |  1 +

 pengine/test10/bug-5028.summary                    |  1 +

 pengine/test10/bug-5186-partial-migrate.summary    |  1 +

 pengine/test10/bug-cl-5247.summary                 |  1 +

 pengine/test10/bug-lf-2508.summary                 |  1 +

 pengine/test10/bug-lf-2551.summary                 |  1 +

 pengine/test10/bug-lf-2606.summary                 |  1 +

 pengine/test10/bug-rh-1097457.summary              |  1 +

 .../test10/colocate-primitive-with-clone.summary   |  1 +

 pengine/test10/concurrent-fencing.summary          |  3 ++

 pengine/test10/guest-node-host-dies.summary        |  3 ++

 pengine/test10/inc12.summary                       |  6 +++

 pengine/test10/interleave-pseudo-stop.summary      |  1 +

 pengine/test10/master-7.summary                    |  1 +

 pengine/test10/master-8.summary                    |  1 +

 pengine/test10/master-9.summary                    |  1 +

 pengine/test10/migrate-fencing.summary             |  1 +

 pengine/test10/migrate-shutdown.summary            |  4 ++

 pengine/test10/novell-239082.summary               |  1 +

 pengine/test10/novell-252693.summary               |  1 +

 pengine/test10/params-2.summary                    |  1 +

 pengine/test10/rec-node-11.summary                 |  1 +

 pengine/test10/rec-node-12.summary                 |  1 +

 pengine/test10/rec-node-13.summary                 |  1 +

 pengine/test10/rec-node-14.summary                 |  3 ++

 pengine/test10/rec-node-15.summary                 |  1 +

 pengine/test10/rec-node-2.summary                  |  1 +

 pengine/test10/rec-node-4.summary                  |  1 +

 pengine/test10/rec-node-6.summary                  |  1 +

 pengine/test10/rec-node-7.summary                  |  1 +

 pengine/test10/rec-rsc-5.summary                   |  1 +

 .../test10/remote-fence-before-reconnect.summary   |  1 +

 pengine/test10/remote-fence-unclean.summary        |  1 +

 pengine/test10/remote-fence-unclean2.summary       |  1 +

 pengine/test10/remote-partial-migrate2.summary     |  1 +

 pengine/test10/remote-recover-all.summary          |  3 ++

 pengine/test10/remote-recover-connection.summary   |  1 +

 pengine/test10/remote-recover-fail.summary         |  1 +

 pengine/test10/remote-recover-no-resources.summary |  2 +

 pengine/test10/remote-recover-unknown.summary      |  3 ++

 pengine/test10/remote-recovery.summary             |  1 +

 pengine/test10/remote-unclean2.summary             |  1 +

 pengine/test10/simple7.summary                     |  1 +

 .../test10/start-then-stop-with-unfence.summary    |  2 +

 pengine/test10/stonith-0.summary                   |  2 +

 pengine/test10/stonith-1.summary                   |  1 +

 pengine/test10/stonith-2.summary                   |  1 +

 pengine/test10/stonith-3.summary                   |  1 +

 pengine/test10/stonith-4.summary                   |  4 ++

 pengine/test10/stop-failure-no-quorum.summary      |  1 +

 pengine/test10/stop-failure-with-fencing.summary   |  1 +

 pengine/test10/systemhealth1.summary               |  2 +

 pengine/test10/systemhealth2.summary               |  1 +

 pengine/test10/systemhealth3.summary               |  1 +

 pengine/test10/systemhealthm1.summary              |  2 +

 pengine/test10/systemhealthm2.summary              |  1 +

 pengine/test10/systemhealthm3.summary              |  1 +

 pengine/test10/systemhealthn1.summary              |  2 +

 pengine/test10/systemhealthn2.summary              |  1 +

 pengine/test10/systemhealthn3.summary              |  1 +

 pengine/test10/systemhealtho1.summary              |  2 +

 pengine/test10/systemhealtho2.summary              |  1 +

 pengine/test10/systemhealtho3.summary              |  1 +

 pengine/test10/systemhealthp1.summary              |  2 +

 pengine/test10/systemhealthp2.summary              |  1 +

 pengine/test10/systemhealthp3.summary              |  1 +

 pengine/test10/ticket-clone-21.summary             |  2 +

 pengine/test10/ticket-clone-9.summary              |  2 +

 pengine/test10/ticket-group-21.summary             |  1 +

 pengine/test10/ticket-group-9.summary              |  1 +

 pengine/test10/ticket-master-21.summary            |  1 +

 pengine/test10/ticket-master-9.summary             |  1 +

 pengine/test10/ticket-primitive-21.summary         |  1 +

 pengine/test10/ticket-primitive-9.summary          |  1 +

 pengine/test10/unfence-definition.summary          |  5 ++

 pengine/test10/unfence-parameters.summary          |  5 ++

 pengine/test10/unfence-startup.summary             |  5 ++

 pengine/test10/unmanaged-master.summary            |  2 +

 pengine/test10/whitebox-fail1.summary              |  1 +

 pengine/test10/whitebox-fail2.summary              |  1 +

 .../test10/whitebox-imply-stop-on-fence.summary    |  3 ++

 pengine/test10/whitebox-ms-ordering.summary        |  2 +

 .../test10/whitebox-unexpectedly-running.summary   |  1 +

 tools/crm_simulate.c                               |  1 +

 100 files changed, 223 insertions(+), 34 deletions(-)

diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c

index 8caf522..377100c 100644

--- a/lib/pengine/unpack.c

+++ b/lib/pengine/unpack.c

@@ -62,6 +62,7 @@ is_dangling_container_remote_node(node_t *node)

     return FALSE;

 }

+

 void

 pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)

 {

@@ -73,12 +74,12 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)

         if (is_set(rsc->flags, pe_rsc_failed) == FALSE) {

             if (!is_set(rsc->flags, pe_rsc_managed)) {

-                crm_notice("Not fencing guest node %s because the container is "

-                           "unmanaged, otherwise we would do so recovering %s "

-                           "%s", node->details->uname, rsc->id, reason);

+                crm_notice("Not fencing node %s due to '%s': container %s is"

+                           " unmanaged"

+                           "%s", node->details->uname, reason, rsc->id);

             } else {

-                crm_warn("Guest node %s will be fenced (by recovering %s) %s",

-                    node->details->uname, rsc->id, reason);

+                crm_warn("Remote node %s will be fenced due to '%s' by recovering %s",

+                         node->details->uname, rsc->id, reason);

                 /* We don't mark the node as unclean because that would prevent the

                  * node from running resources. We want to allow it to run resources

@@ -88,8 +89,9 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)

                 set_bit(rsc->flags, pe_rsc_failed);

             }

         }

+

     } else if (is_dangling_container_remote_node(node)) {

-        crm_info("Cleaning up dangling connection resource for guest node %s %s"

+        crm_info("Cleaning up dangling connection resource for guest node %s due to '%s'"

                  " (fencing is already done, guest resource no longer exists)",

                  node->details->uname, reason);

         set_bit(node->details->remote_rsc->flags, pe_rsc_failed);

@@ -98,28 +100,32 @@ pe_fence_node(pe_working_set_t * data_set, node_t * node, const char *reason)

         resource_t *rsc = node->details->remote_rsc;

         if (rsc && (!is_set(rsc->flags, pe_rsc_managed))) {

-            crm_notice("Not fencing node %s because connection is unmanaged, "

-                       "otherwise would %s", node->details->uname, reason);

+            crm_notice("Not fencing node %s due to '%s': connection is unmanaged",

+                       node->details->uname, reason);

         } else if(node->details->remote_requires_reset == FALSE) {

             node->details->remote_requires_reset = TRUE;

             if (pe_can_fence(data_set, node)) {

-                crm_warn("Node %s will be fenced %s", node->details->uname, reason);

+                crm_warn("Remote node %s will be fenced due to %s", node->details->uname, reason);

             } else {

-                crm_warn("Node %s is unclean %s", node->details->uname, reason);

+                crm_warn("Remote node %s is unclean due to %s", node->details->uname, reason);

             }

         }

         node->details->unclean = TRUE;

-    } else if (node->details->unclean == FALSE) {

+    } else if (node->details->unclean) {

         if (pe_can_fence(data_set, node)) {

-            crm_warn("Node %s will be fenced %s", node->details->uname, reason);

+            crm_trace("Node %s would also be fenced due to '%s'", node->details->uname, reason);

         } else {

-            crm_warn("Node %s is unclean %s", node->details->uname, reason);

+            crm_trace("Node %s is also unclean due to '%s'", node->details->uname, reason);

         }

+

+    } else if (pe_can_fence(data_set, node)) {

+        crm_warn("Node %s will be fenced due to %s", node->details->uname, reason);

         node->details->unclean = TRUE;

     } else {

-        crm_trace("Node %s would also be fenced '%s'", node->details->uname, reason);

+        crm_warn("Node %s is unclean due to %s", node->details->uname, reason);

+        node->details->unclean = TRUE;

     }

 }

@@ -1260,7 +1266,7 @@ unpack_status(xmlNode * status, pe_working_set_t * data_set)

                 /* Everything else should flow from this automatically

                  * At least until the PE becomes able to migrate off healthy resources

                  */

-                pe_fence_node(data_set, this_node, "because the cluster does not have quorum");

+                pe_fence_node(data_set, this_node, "cluster does not have quorum");

             }

         }

     }

@@ -1316,7 +1322,7 @@ determine_online_status_no_fencing(pe_working_set_t * data_set, xmlNode * node_s

     } else {

         /* mark it unclean */

-        pe_fence_node(data_set, this_node, "because node is unexpectedly down");

+        pe_fence_node(data_set, this_node, "peer is unexpectedly down");

         crm_info("\tin_cluster=%s, is_peer=%s, join=%s, expected=%s",

                  crm_str(in_cluster), crm_str(is_peer), crm_str(join), crm_str(exp_state));

     }

@@ -1373,10 +1379,10 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat

         online = crm_is_true(is_peer);

     } else if (in_cluster == NULL) {

-        pe_fence_node(data_set, this_node, "because the peer has not been seen by the cluster");

+        pe_fence_node(data_set, this_node, "peer has not been seen by the cluster");

     } else if (safe_str_eq(join, CRMD_JOINSTATE_NACK)) {

-        pe_fence_node(data_set, this_node, "because it failed the pacemaker membership criteria");

+        pe_fence_node(data_set, this_node, "peer failed the pacemaker membership criteria");

     } else if (do_terminate == FALSE && safe_str_eq(exp_state, CRMD_JOINSTATE_DOWN)) {

@@ -1395,14 +1401,14 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat

         online = FALSE;

     } else if (crm_is_true(in_cluster) == FALSE) {

-        pe_fence_node(data_set, this_node, "because the node is no longer part of the cluster");

+        pe_fence_node(data_set, this_node, "peer is no longer part of the cluster");

     } else if (crm_is_true(is_peer) == FALSE) {

-        pe_fence_node(data_set, this_node, "because our peer process is no longer available");

+        pe_fence_node(data_set, this_node, "peer process is no longer available");

         /* Everything is running at this point, now check join state */

     } else if (do_terminate) {

-        pe_fence_node(data_set, this_node, "because termination was requested");

+        pe_fence_node(data_set, this_node, "termination was requested");

     } else if (safe_str_eq(join, CRMD_JOINSTATE_MEMBER)) {

         crm_info("Node %s is active", this_node->details->uname);

@@ -1414,7 +1420,7 @@ determine_online_status_fencing(pe_working_set_t * data_set, xmlNode * node_stat

         this_node->details->pending = TRUE;

     } else {

-        pe_fence_node(data_set, this_node, "because the peer was in an unknown state");

+        pe_fence_node(data_set, this_node, "peer was in an unknown state");

         crm_warn("%s: in-cluster=%s, is-peer=%s, join=%s, expected=%s, term=%d, shutdown=%d",

                  this_node->details->uname, crm_str(in_cluster), crm_str(is_peer),

                  crm_str(join), crm_str(exp_state), do_terminate, this_node->details->shutdown);

@@ -1923,14 +1929,14 @@ process_rsc_state(resource_t * rsc, node_t * node,

                  * another node without requiring the baremetal remote nodes to be fenced

                  * as well. */

                 node->details->unseen = TRUE;

-                reason = crm_strdup_printf("because %s is active there. Fencing will be revoked if remote-node connection can be re-established on another cluster-node.", rsc->id);

+                reason = crm_strdup_printf("%s is active there. Fencing will be revoked if remote-node connection can be re-established on another cluster-node.", rsc->id);

             }

             should_fence = TRUE;

         }

         if (should_fence) {

             if (reason == NULL) {

-               reason = crm_strdup_printf("because %s is thought to be active there", rsc->id);

+               reason = crm_strdup_printf("%s is thought to be active there", rsc->id);

             }

             pe_fence_node(data_set, node, reason);

         }

@@ -1953,7 +1959,7 @@ process_rsc_state(resource_t * rsc, node_t * node,

             /* treat it as if it is still running

              * but also mark the node as unclean

              */

-            pe_fence_node(data_set, node, "because of resource failure(s)");

+            pe_fence_node(data_set, node, "resource failure(s)");

             break;

         case action_fail_standby:

@@ -2009,7 +2015,7 @@ process_rsc_state(resource_t * rsc, node_t * node,

                     /* connection resource to baremetal resource failed in a way that

                      * should result in fencing the remote-node. */

-                    pe_fence_node(data_set, tmpnode, "because of connection failure(s)");

+                    pe_fence_node(data_set, tmpnode, "of connection failure(s)");

                 }

             }

diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c

index 7acd90c..44fbb51 100644

--- a/lib/pengine/utils.c

+++ b/lib/pengine/utils.c

@@ -517,7 +517,7 @@ custom_action(resource_t * rsc, char *key, const char *task,

             if (is_set(action->rsc->flags, pe_rsc_managed)

                 && save_action && a_task == stop_rsc

                 && action->node->details->unclean == FALSE) {

-                pe_fence_node(data_set, action->node, "because of unrunnable resource actions");

+                pe_fence_node(data_set, action->node, "resource actions are unrunnable");

             }

         } else if (action->node->details->pending) {

diff --git a/pengine/allocate.c b/pengine/allocate.c

index 795ed56..0020af6 100644

--- a/pengine/allocate.c

+++ b/pengine/allocate.c

@@ -467,7 +467,7 @@ check_actions_for(xmlNode * rsc_entry, resource_t * rsc, node_t * node, pe_worki

             set_bit(action_clear->flags, pe_action_runnable);

             crm_notice("Clearing failure of %s on %s "

-                       "because action definition changed " CRM_XS " %s",

+                       "action definition changed " CRM_XS " %s",

                        rsc->id, node->details->uname, action_clear->uuid);

         }

     }

@@ -1798,7 +1798,7 @@ apply_container_ordering(action_t *action, pe_working_set_t *data_set)

     CRM_ASSERT(container);

     if(is_set(container->flags, pe_rsc_failed)) {

-        pe_fence_node(data_set, action->node, " because the container failed");

+        pe_fence_node(data_set, action->node, "container failed");

     }

     crm_trace("%s %s %s %s %d", action->uuid, action->task, remote_rsc->id, container->id, is_set(container->flags, pe_rsc_failed));

@@ -1967,7 +1967,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set)

                      * way to stop it, it is necessary to fence the

                      * node.

                      */

-                    pe_fence_node(data_set, action->node, "because resources are active and the connection is unrecoverable");

+                    pe_fence_node(data_set, action->node, "resources are active and the connection is unrecoverable");

                 }

                 custom_action_order(action->rsc, NULL, action,

@@ -2009,7 +2009,7 @@ apply_remote_ordering(action_t *action, pe_working_set_t *data_set)

                      * Since we have no way to find out, it is

                      * necessary to fence the node.

                      */

-                    pe_fence_node(data_set, action->node, "because resources are in an unknown state and the connection is unrecoverable");

+                    pe_fence_node(data_set, action->node, "resources are in an unknown state and the connection is unrecoverable");

                 }

                 if(cluster_node && state == remote_state_stopped) {

@@ -2270,6 +2270,7 @@ stage7(pe_working_set_t * data_set)

     crm_trace("Processing reloads");

+    LogNodeActions(data_set, FALSE);

     for (gIter = data_set->resources; gIter != NULL; gIter = gIter->next) {

         resource_t *rsc = (resource_t *) gIter->data;

@@ -2379,6 +2380,44 @@ stage8(pe_working_set_t * data_set)

 }

 void

+LogNodeActions(pe_working_set_t * data_set, gboolean terminal)

+{

+    GListPtr gIter = NULL;

+

+    for (gIter = data_set->actions; gIter != NULL; gIter = gIter->next) {

+        char *node_name = NULL;

+        const char *task = NULL;

+        action_t *action = (action_t *) gIter->data;

+

+        if (action->rsc != NULL) {

+            continue;

+        }

+

+        if (is_container_remote_node(action->node)) {

+            node_name = crm_strdup_printf("%s (resource: %s)", action->node->details->uname, action->node->details->remote_rsc->container->id);

+        } else if(action->node) {

+            node_name = crm_strdup_printf("%s", action->node->details->uname);

+        }

+

+        if (safe_str_eq(action->task, CRM_OP_SHUTDOWN)) {

+            task = "Shutdown";

+        } else if (safe_str_eq(action->task, CRM_OP_FENCE)) {

+            task = "Fence";

+        }

+

+        if(task == NULL) {

+            /* Nothing to report */

+        } else if(terminal) {

+            printf(" * %s %s\n", task, node_name);

+        } else {

+            crm_notice(" * %s %s\n", task, node_name);

+        }

+

+        free(node_name);

+    }

+}

+

+void

 cleanup_alloc_calculations(pe_working_set_t * data_set)

 {

     if (data_set == NULL) {

diff --git a/pengine/allocate.h b/pengine/allocate.h

index 3d7b7ce..d89943d 100644

--- a/pengine/allocate.h

+++ b/pengine/allocate.h

@@ -152,7 +152,8 @@ extern gboolean unpack_location(xmlNode * xml_obj, pe_working_set_t * data_set);

 extern gboolean unpack_rsc_ticket(xmlNode * xml_obj, pe_working_set_t * data_set);

-extern void LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal);

+void LogNodeActions(pe_working_set_t * data_set, gboolean terminal);

+void LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal);

 void container_LogActions(resource_t * rsc, pe_working_set_t * data_set, gboolean terminal);

 extern void cleanup_alloc_calculations(pe_working_set_t * data_set);

diff --git a/pengine/native.c b/pengine/native.c

index 2c4e2a1..dd5ff18 100644

--- a/pengine/native.c

+++ b/pengine/native.c

@@ -1818,7 +1818,7 @@ rsc_ticket_constraint(resource_t * rsc_lh, rsc_ticket_t * rsc_ticket, pe_working

                 for (gIter = rsc_lh->running_on; gIter != NULL; gIter = gIter->next) {

                     node_t *node = (node_t *) gIter->data;

-                    pe_fence_node(data_set, node, "because deadman ticket was lost");

+                    pe_fence_node(data_set, node, "deadman ticket was lost");

                 }

                 break;

@@ -2782,7 +2782,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete,

     if (force == FALSE && running != NULL) {

         /* we already know the status of the resource on this node */

-        pe_rsc_trace(rsc, "Skipping active: %s on %s", rsc->id, node->details->uname);

+        pe_rsc_trace(rsc, "Skipping known: %s on %s", rsc->id, node->details->uname);

         return FALSE;

     }

diff --git a/pengine/test10/594.summary b/pengine/test10/594.summary

index 6c208ee..d9fe8c1 100644

--- a/pengine/test10/594.summary

+++ b/pengine/test10/594.summary

@@ -13,6 +13,8 @@ Online: [ hadev1 hadev2 ]

      child_DoFencing:2	(stonith:ssh):	Started hadev1

 Transition Summary:

+ * Fence hadev3

+ * Shutdown hadev2

  * Move    DcIPaddr	(Started hadev2 -> hadev1)

  * Move    rsc_hadev2	(Started hadev2 -> hadev1)

  * Stop    child_DoFencing:0	(hadev2)

diff --git a/pengine/test10/662.summary b/pengine/test10/662.summary

index 8d73b1d..1726f35 100644

--- a/pengine/test10/662.summary

+++ b/pengine/test10/662.summary

@@ -14,6 +14,7 @@ Online: [ c001n02 c001n03 c001n04 c001n09 ]

      child_DoFencing:3	(stonith:ssh):	Started c001n09

 Transition Summary:

+ * Shutdown c001n02

  * Move    rsc_c001n02	(Started c001n02 -> c001n03)

  * Stop    child_DoFencing:0	(c001n02)

diff --git a/pengine/test10/797.summary b/pengine/test10/797.summary

index 6e78255..3184eae 100644

--- a/pengine/test10/797.summary

+++ b/pengine/test10/797.summary

@@ -15,6 +15,7 @@ Online: [ c001n01 c001n02 c001n03 ]

      child_DoFencing:3	(stonith:ssh):	Stopped 

 Transition Summary:

+ * Shutdown c001n02

  * Stop    DcIPaddr	(Started c001n03)

  * Stop    rsc_c001n08	(Started c001n02)

  * Stop    rsc_c001n02	(Started c001n02)

diff --git a/pengine/test10/829.summary b/pengine/test10/829.summary

index d95a2ff..a9d25e0 100644

--- a/pengine/test10/829.summary

+++ b/pengine/test10/829.summary

@@ -15,6 +15,7 @@ Online: [ c001n01 c001n03 c001n08 ]

      child_DoFencing:3	(stonith:ssh):	Started c001n08

 Transition Summary:

+ * Fence c001n02

  * Move    rsc_c001n02	(Started c001n02 -> c001n01)

  * Stop    child_DoFencing:0	(c001n02)

diff --git a/pengine/test10/bug-1572-1.summary b/pengine/test10/bug-1572-1.summary

index 4280f7b..6c37bb4 100644

--- a/pengine/test10/bug-1572-1.summary

+++ b/pengine/test10/bug-1572-1.summary

@@ -11,6 +11,7 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ]

      IPaddr_147_81_84_133	(ocf::heartbeat:IPaddr):	Started arc-tkincaidlx.wsicorp.com

 Transition Summary:

+ * Shutdown arc-dknightlx

  * Stop    rsc_drbd_7788:0	(arc-dknightlx)

  * Restart rsc_drbd_7788:1	(Master arc-tkincaidlx.wsicorp.com)

  * Restart fs_mirror	(Started arc-tkincaidlx.wsicorp.com)

diff --git a/pengine/test10/bug-1572-2.summary b/pengine/test10/bug-1572-2.summary

index 6174027..a4235a7 100644

--- a/pengine/test10/bug-1572-2.summary

+++ b/pengine/test10/bug-1572-2.summary

@@ -11,6 +11,7 @@ Online: [ arc-dknightlx arc-tkincaidlx.wsicorp.com ]

      IPaddr_147_81_84_133	(ocf::heartbeat:IPaddr):	Started arc-tkincaidlx.wsicorp.com

 Transition Summary:

+ * Shutdown arc-dknightlx

  * Stop    rsc_drbd_7788:0	(arc-dknightlx)

  * Demote  rsc_drbd_7788:1	(Master -> Slave arc-tkincaidlx.wsicorp.com)

  * Stop    fs_mirror	(arc-tkincaidlx.wsicorp.com)

diff --git a/pengine/test10/bug-1573.summary b/pengine/test10/bug-1573.summary

index 0c1fe3d..8fb2820 100644

--- a/pengine/test10/bug-1573.summary

+++ b/pengine/test10/bug-1573.summary

@@ -11,6 +11,7 @@ OFFLINE: [ xen-c ]

      apache_6	(ocf::heartbeat:apache):	Stopped 

 Transition Summary:

+ * Shutdown xen-b

  * Stop    IPaddr_192_168_1_102	(xen-b)

 Executing cluster transition:

diff --git a/pengine/test10/bug-1820-1.summary b/pengine/test10/bug-1820-1.summary

index 5edcbcb..db41ac4 100644

--- a/pengine/test10/bug-1820-1.summary

+++ b/pengine/test10/bug-1820-1.summary

@@ -8,6 +8,7 @@ Online: [ star world ]

      test2	(ocf::heartbeat:Xen):	Started star

 Transition Summary:

+ * Shutdown star

  * Start   p1	(world)

  * Migrate test1	(Started star -> world)

  * Migrate test2	(Started star -> world)

diff --git a/pengine/test10/bug-1820.summary b/pengine/test10/bug-1820.summary

index 592fa67..6d9c021 100644

--- a/pengine/test10/bug-1820.summary

+++ b/pengine/test10/bug-1820.summary

@@ -7,6 +7,7 @@ Online: [ star world ]

      test2	(ocf::heartbeat:Xen):	Started star

 Transition Summary:

+ * Shutdown star

  * Migrate test1	(Started star -> world)

  * Migrate test2	(Started star -> world)

diff --git a/pengine/test10/bug-1822.summary b/pengine/test10/bug-1822.summary

index afb9fd1..325e408 100644

--- a/pengine/test10/bug-1822.summary

+++ b/pengine/test10/bug-1822.summary

@@ -11,6 +11,7 @@ Online: [ process1a process2b ]

          master_slave_procdctl:1	(ocf::heartbeat:procdctl):	Master process1a

 Transition Summary:

+ * Shutdown process1a

  * Demote  master_slave_Stateful:1	(Master -> Stopped process1a)

  * Demote  master_slave_procdctl:1	(Master -> Stopped process1a)

diff --git a/pengine/test10/bug-5028-bottom.summary b/pengine/test10/bug-5028-bottom.summary

index fc5cf8f..b43ba4e 100644

--- a/pengine/test10/bug-5028-bottom.summary

+++ b/pengine/test10/bug-5028-bottom.summary

@@ -7,6 +7,7 @@ Online: [ bl460g6a bl460g6b ]

      dummy02	(ocf::heartbeat:Dummy-stop-NG):	Started bl460g6a

 Transition Summary:

+ * Shutdown bl460g6a

  * Stop    dummy02	(bl460g6a)

 Executing cluster transition:

diff --git a/pengine/test10/bug-5028-detach.summary b/pengine/test10/bug-5028-detach.summary

index 5e93b2a..712bfa1 100644

--- a/pengine/test10/bug-5028-detach.summary

+++ b/pengine/test10/bug-5028-detach.summary

@@ -10,6 +10,7 @@ Online: [ bl460g6a bl460g6b ]

      dummy02	(ocf::heartbeat:Dummy-stop-NG):	FAILED bl460g6a ( blocked ) 

 Transition Summary:

+ * Shutdown bl460g6a

 Executing cluster transition:

  * Cluster action:  do_shutdown on bl460g6a

diff --git a/pengine/test10/bug-5028.summary b/pengine/test10/bug-5028.summary

index ad7657c..a85f75b 100644

--- a/pengine/test10/bug-5028.summary

+++ b/pengine/test10/bug-5028.summary

@@ -7,6 +7,7 @@ Online: [ bl460g6a bl460g6b ]

      dummy02	(ocf::heartbeat:Dummy-stop-NG):	FAILED bl460g6a ( blocked ) 

 Transition Summary:

+ * Shutdown bl460g6a

  * Stop    dummy01	(Started bl460g6a - blocked)

 Executing cluster transition:

diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary

index 4d0b10e..63ed2d7 100644

--- a/pengine/test10/bug-5186-partial-migrate.summary

+++ b/pengine/test10/bug-5186-partial-migrate.summary

@@ -25,6 +25,7 @@ Online: [ bl460g1n6 bl460g1n8 ]

      Started: [ bl460g1n6 bl460g1n8 ]

 Transition Summary:

+ * Fence bl460g1n7

  * Move    prmDummy	(Started bl460g1n7 -> bl460g1n6)

  * Move    prmVM2	(Started bl460g1n7 -> bl460g1n8)

  * Move    prmStonith8-1	(Started bl460g1n7 -> bl460g1n6)

diff --git a/pengine/test10/bug-cl-5247.summary b/pengine/test10/bug-cl-5247.summary

index a13754f..8ea3ff0 100644

--- a/pengine/test10/bug-cl-5247.summary

+++ b/pengine/test10/bug-cl-5247.summary

@@ -18,6 +18,7 @@ Containers: [ pgsr01:prmDB1 ]

      Stopped: [ bl460g8n3 bl460g8n4 ]

 Transition Summary:

+ * Fence pgsr02 (resource: prmDB2)

  * Stop    prmDB2	(bl460g8n4)

  * Restart prmStonith1-2	(Started bl460g8n4)

  * Restart prmStonith2-2	(Started bl460g8n3)

diff --git a/pengine/test10/bug-lf-2508.summary b/pengine/test10/bug-lf-2508.summary

index af5e4e2..7b436d4 100644

--- a/pengine/test10/bug-lf-2508.summary

+++ b/pengine/test10/bug-lf-2508.summary

@@ -35,6 +35,7 @@ Online: [ srv01 srv03 srv04 ]

      Stopped: [ srv04 ]

 Transition Summary:

+ * Fence srv02

  * Start   Dummy01	(srv01)

  * Move    Dummy02	(Started srv02 -> srv04)

  * Stop    prmStonith1-1:1	(srv02)

diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary

index 158eb73..ffb7c6d 100644

--- a/pengine/test10/bug-lf-2551.summary

+++ b/pengine/test10/bug-lf-2551.summary

@@ -81,6 +81,7 @@ Online: [ hex-0 hex-7 hex-8 ]

  vm-64	(ocf::heartbeat:Xen):	Stopped 

 Transition Summary:

+ * Fence hex-9

  * Move    fencing-sbd	(Started hex-9 -> hex-0)

  * Move    dummy1	(Started hex-9 -> hex-0)

  * Stop    dlm:3	(hex-9)

diff --git a/pengine/test10/bug-lf-2606.summary b/pengine/test10/bug-lf-2606.summary

index 3e74d4b..ab93bb3 100644

--- a/pengine/test10/bug-lf-2606.summary

+++ b/pengine/test10/bug-lf-2606.summary

@@ -12,6 +12,7 @@ Online: [ node1 ]

      Slaves: [ node1 ]

 Transition Summary:

+ * Fence node2

  * Stop    rsc1	(node2)

  * Move    rsc2	(Started node2 -> node1)

  * Demote  rsc3:1	(Master -> Stopped node2)

diff --git a/pengine/test10/bug-rh-1097457.summary b/pengine/test10/bug-rh-1097457.summary

index f8d2c14..8fe474a 100644

--- a/pengine/test10/bug-rh-1097457.summary

+++ b/pengine/test10/bug-rh-1097457.summary

@@ -32,6 +32,7 @@ Containers: [ lamaVM1:VM1 lamaVM2:VM2 lamaVM3:VM3 ]

      Started: [ lamaVM1 lamaVM2 lamaVM3 ]

 Transition Summary:

+ * Fence lamaVM2 (resource: VM2)

  * Recover VM2	(Started lama3)

  * Recover FSlun3	(Started lamaVM2 -> lama2)

  * Restart FAKE4	(Started lamaVM2)

diff --git a/pengine/test10/colocate-primitive-with-clone.summary b/pengine/test10/colocate-primitive-with-clone.summary

index 5e4c511..e0f685f 100644

--- a/pengine/test10/colocate-primitive-with-clone.summary

+++ b/pengine/test10/colocate-primitive-with-clone.summary

@@ -53,6 +53,7 @@ Online: [ srv01 srv02 srv03 srv04 ]

      Stopped: [ srv01 ]

 Transition Summary:

+ * Shutdown srv01

  * Start   UmVIPcheck	(srv04)

  * Start   UmIPaddr	(srv04)

  * Start   UmDummy01	(srv04)

diff --git a/pengine/test10/concurrent-fencing.summary b/pengine/test10/concurrent-fencing.summary

index 10b2fde..a274c3b 100644

--- a/pengine/test10/concurrent-fencing.summary

+++ b/pengine/test10/concurrent-fencing.summary

@@ -8,6 +8,9 @@ Node node3 (uuid3): UNCLEAN (offline)

  lsb_dummy	(lsb:/usr/lib/heartbeat/cts/LSBDummy):	Stopped

 Transition Summary:

+ * Fence node3

+ * Fence node2

+ * Fence node1

 Executing cluster transition:

  * Fencing node3 (reboot)

diff --git a/pengine/test10/guest-node-host-dies.summary b/pengine/test10/guest-node-host-dies.summary

index c6180ca..717c43d 100644

--- a/pengine/test10/guest-node-host-dies.summary

+++ b/pengine/test10/guest-node-host-dies.summary

@@ -11,6 +11,9 @@ Online: [ rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]

      Stopped: [ rhel7-1 rhel7-2 rhel7-3 rhel7-4 rhel7-5 ]

 Transition Summary:

+ * Fence rhel7-1

+ * Fence lxc2 (resource: container2)

+ * Fence lxc1 (resource: container1)

  * Restart Fencing	(Started rhel7-4)

  * Move    rsc_rhel7-1	(Started rhel7-1 -> rhel7-5)

  * Recover container1	(Started rhel7-1 -> rhel7-2)

diff --git a/pengine/test10/inc12.summary b/pengine/test10/inc12.summary

index 3df5a59..5068b7e 100644

--- a/pengine/test10/inc12.summary

+++ b/pengine/test10/inc12.summary

@@ -32,6 +32,12 @@ Online: [ c001n02 c001n03 c001n04 c001n05 c001n06 c001n07 ]

      ocf_msdummy:11	(ocf::heartbeat:Stateful):	Slave c001n02 

 Transition Summary:

+ * Shutdown c001n07

+ * Shutdown c001n06

+ * Shutdown c001n05

+ * Shutdown c001n04

+ * Shutdown c001n03

+ * Shutdown c001n02

  * Stop    ocf_192.168.100.181	(c001n02)

  * Stop    heartbeat_192.168.100.182	(c001n02)

  * Stop    ocf_192.168.100.183	(c001n02)

diff --git a/pengine/test10/interleave-pseudo-stop.summary b/pengine/test10/interleave-pseudo-stop.summary

index a682462..cf30da0 100644

--- a/pengine/test10/interleave-pseudo-stop.summary

+++ b/pengine/test10/interleave-pseudo-stop.summary

@@ -17,6 +17,7 @@ Online: [ node2 ]

      Started: [ node2 ]

 Transition Summary: