commit cf7988cb02e96f7a278569dcee410691cb7b85e0

Author: Andrew Beekhof <andrew@beekhof.net>

Date:   Fri Apr 4 11:59:13 2014 +1100

    Fix: PE: Delay unfencing until after we know the state of all resources that require unfencing

    (cherry picked from commit 22666466bd9e50efdd8efb6c1775c71785c900b1)

    Conflicts:

    	include/crm/pengine/internal.h

    	include/crm/pengine/status.h

    	lib/pengine/unpack.c

    	lib/pengine/utils.c

    	pengine/test10/unfence-definition.dot

    	pengine/test10/unfence-definition.exp

    	pengine/test10/unfence-definition.summary

    	pengine/test10/unfence-parameters.dot

    	pengine/test10/unfence-parameters.exp

    	pengine/test10/unfence-parameters.summary

    	pengine/test10/unfence-startup.dot

    	pengine/test10/unfence-startup.exp

    	pengine/test10/unfence-startup.xml

diff --git a/include/crm/pengine/internal.h b/include/crm/pengine/internal.h

index 657c647..b4907c1 100644

--- a/include/crm/pengine/internal.h

+++ b/include/crm/pengine/internal.h

@@ -260,8 +260,13 @@ typedef struct op_digest_cache_s {

 op_digest_cache_t *rsc_action_digest_cmp(resource_t * rsc, xmlNode * xml_op, node_t * node,

                                          pe_working_set_t * data_set);

-gboolean is_remote_node(xmlNode *xml);

+action_t *pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set);

+void trigger_unfencing(

+    resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set);

+

+void set_bit_recursive(resource_t * rsc, unsigned long long flag);

 void clear_bit_recursive(resource_t * rsc, unsigned long long flag);

+gboolean is_remote_node(xmlNode *xml);

 resource_t * rsc_contains_remote_node(pe_working_set_t * data_set, resource_t *rsc);

 #endif

diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h

index ddf9dd7..6542737 100644

--- a/include/crm/pengine/status.h

+++ b/include/crm/pengine/status.h

@@ -191,6 +191,7 @@ struct node_s {

 #  define pe_rsc_needs_quorum	 0x10000000ULL

 #  define pe_rsc_needs_fencing	 0x20000000ULL

 #  define pe_rsc_needs_unfencing 0x40000000ULL

+#  define pe_rsc_have_unfencing  0x80000000ULL

 enum pe_graph_flags {

     pe_graph_none = 0x00000,

diff --git a/lib/pengine/clone.c b/lib/pengine/clone.c

index b13cceb..90d7eca 100644

--- a/lib/pengine/clone.c

+++ b/lib/pengine/clone.c

@@ -47,19 +47,6 @@ mark_as_orphan(resource_t * rsc)

 }

 void

-clear_bit_recursive(resource_t * rsc, unsigned long long flag)

-{

-    GListPtr gIter = rsc->children;

-

-    clear_bit(rsc->flags, flag);

-    for (; gIter != NULL; gIter = gIter->next) {

-        resource_t *child_rsc = (resource_t *) gIter->data;

-

-        clear_bit_recursive(child_rsc, flag);

-    }

-}

-

-void

 force_non_unique_clone(resource_t * rsc, const char *rid, pe_working_set_t * data_set)

 {

     if (rsc->variant == pe_clone || rsc->variant == pe_master) {

diff --git a/lib/pengine/complex.c b/lib/pengine/complex.c

index 0f20481..a3133e7 100644

--- a/lib/pengine/complex.c

+++ b/lib/pengine/complex.c

@@ -575,7 +575,7 @@ common_unpack(xmlNode * xml_obj, resource_t ** rsc,

     } else if (safe_str_eq(value, "fencing")) {

         set_bit((*rsc)->flags, pe_rsc_needs_fencing);

         if (is_set(data_set->flags, pe_flag_stonith_enabled)) {

-            crm_config_warn("%s requires (un)fencing but fencing is disabled", (*rsc)->id);

+            crm_config_warn("%s requires fencing but fencing is disabled", (*rsc)->id);

         }

     } else {

diff --git a/lib/pengine/unpack.c b/lib/pengine/unpack.c

index 39ded96..eb10bd8 100644

--- a/lib/pengine/unpack.c

+++ b/lib/pengine/unpack.c

@@ -2325,6 +2325,31 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op,

      * didnt include target_rc and liked to remap status

      */

     switch (actual_rc_i) {

+        case PCMK_OCF_OK:

+            if (is_probe && target_rc == 7) {

+                task_status_i = PCMK_LRM_OP_DONE;

+                set_bit(rsc->flags, pe_rsc_unexpectedly_running);

+

+                if(is_set(rsc->flags, pe_rsc_needs_unfencing)) {

+                    /* _Require_ unfencing after probing resources that need unfencing and was found active */

+                    crm_warn("Operation %s found resource %s active on %s: triggering unfencing",

+                             task, rsc->id, node->details->uname);

+                    trigger_unfencing(NULL, node, "Unexpected state", NULL, data_set);

+

+                } else {

+                    pe_rsc_info(rsc, "Operation %s found resource %s active on %s",

+                                task, rsc->id, node->details->uname);

+                }

+

+                /* legacy code for pre-0.6.5 operations */

+            } else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) {

+                /* catch status ops that return 0 instead of 8 while they

+                 *   are supposed to be in master mode

+                 */

+                task_status_i = PCMK_LRM_OP_ERROR;

+            }

+            break;

+

         case PCMK_OCF_NOT_RUNNING:

             if (is_probe || target_rc == actual_rc_i) {

                 task_status_i = PCMK_LRM_OP_DONE;

@@ -2415,23 +2440,6 @@ unpack_rsc_op(resource_t * rsc, node_t * node, xmlNode * xml_op,

             }

             break;

-        case PCMK_OCF_OK:

-            if (is_probe && target_rc == 7) {

-                task_status_i = PCMK_LRM_OP_DONE;

-                set_bit(rsc->flags, pe_rsc_unexpectedly_running);

-                pe_rsc_info(rsc, "Operation %s found resource %s active on %s",

-                            task, rsc->id, node->details->uname);

-

-                /* legacy code for pre-0.6.5 operations */

-            } else if (target_rc < 0 && interval > 0 && rsc->role == RSC_ROLE_MASTER) {

-                /* catch status ops that return 0 instead of 8 while they

-                 *   are supposed to be in master mode

-                 */

-                task_status_i = PCMK_LRM_OP_ERROR;

-            }

-

-            break;

-

         default:

             if (task_status_i == PCMK_LRM_OP_DONE) {

                 crm_info("Remapping %s (rc=%d) on %s to an ERROR",

diff --git a/lib/pengine/utils.c b/lib/pengine/utils.c

index 2520eeb..e84d341 100644

--- a/lib/pengine/utils.c

+++ b/lib/pengine/utils.c

@@ -1807,3 +1807,92 @@ rsc_contains_remote_node(pe_working_set_t * data_set, resource_t *rsc)

     return NULL;

 }

+void

+clear_bit_recursive(resource_t * rsc, unsigned long long flag)

+{

+    GListPtr gIter = rsc->children;

+

+    clear_bit(rsc->flags, flag);

+    for (; gIter != NULL; gIter = gIter->next) {

+        resource_t *child_rsc = (resource_t *) gIter->data;

+

+        clear_bit_recursive(child_rsc, flag);

+    }

+}

+

+void

+set_bit_recursive(resource_t * rsc, unsigned long long flag)

+{

+    GListPtr gIter = rsc->children;

+

+    set_bit(rsc->flags, flag);

+    for (; gIter != NULL; gIter = gIter->next) {

+        resource_t *child_rsc = (resource_t *) gIter->data;

+

+        set_bit_recursive(child_rsc, flag);

+    }

+}

+

+action_t *

+pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set)

+{

+    char *key = NULL;

+    action_t *stonith_op = NULL;

+

+    if(op == NULL) {

+        op = data_set->stonith_action;

+    }

+

+    key = g_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);

+

+    if(data_set->singletons) {

+        stonith_op = g_hash_table_lookup(data_set->singletons, key);

+    }

+

+    if(stonith_op == NULL) {

+        stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, optional, TRUE, data_set);

+

+        add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);

+        add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);

+        add_hash_param(stonith_op->meta, "stonith_action", op);

+    }

+

+    if(optional == FALSE) {

+        crm_trace("%s is no longer optional", stonith_op->uuid);

+        pe_clear_action_bit(stonith_op, pe_action_optional);

+    }

+

+    return stonith_op;

+}

+

+void

+trigger_unfencing(

+    resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set) 

+{

+    if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {

+        /* No resources require it */

+        return;

+

+    } else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) {

+        /* Wasnt a stonith device */

+        return;

+

+    } else if(node) {

+        action_t *unfence = pe_fence_op(node, "on", FALSE, data_set);

+

+        crm_notice("Unfencing %s: %s", node->details->uname, reason);

+        if(FALSE && dependancy) {

+            order_actions(unfence, dependancy, pe_order_optional);

+        }

+

+    } else if(rsc) {

+        GHashTableIter iter;

+

+        g_hash_table_iter_init(&iter, rsc->allowed_nodes);

+        while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {

+            if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {

+                trigger_unfencing(rsc, node, reason, dependancy, data_set);

+            }

+        }

+    }

+}

diff --git a/pengine/allocate.c b/pengine/allocate.c

index 88258f4..3480eb9 100644

--- a/pengine/allocate.c

+++ b/pengine/allocate.c

@@ -1267,70 +1267,6 @@ any_managed_resources(pe_working_set_t * data_set)

     return FALSE;

 }

-void

-trigger_unfencing(

-    resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set) 

-{

-    if(is_not_set(data_set->flags, pe_flag_enable_unfencing)) {

-        /* No resources require it */

-        return;

-

-    } else if (rsc != NULL && is_not_set(rsc->flags, pe_rsc_fence_device)) {

-        /* Wasnt a stonith device */

-        return;

-

-    } else if(node) {

-        action_t *unfence = pe_fence_op(node, "on", FALSE, data_set);

-

-        crm_notice("Unfencing %s: %s", node->details->uname, reason);

-        if(FALSE && dependancy) {

-            order_actions(dependancy, unfence, pe_order_optional);

-        }

-

-    } else if(rsc) {

-        GHashTableIter iter;

-

-        g_hash_table_iter_init(&iter, rsc->allowed_nodes);

-        while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {

-            if(node->details->online && node->details->unclean == FALSE && node->details->shutdown == FALSE) {

-                trigger_unfencing(rsc, node, reason, dependancy, data_set);

-            }

-        }

-    }

-}

-

-action_t *

-pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set)

-{

-    char *key = NULL;

-    action_t *stonith_op = NULL;

-

-    if(op == NULL) {

-        op = data_set->stonith_action;

-    }

-

-    key = g_strdup_printf("%s-%s-%s", CRM_OP_FENCE, node->details->uname, op);

-

-    if(data_set->singletons) {

-        stonith_op = g_hash_table_lookup(data_set->singletons, key);

-    }

-

-    if(stonith_op == NULL) {

-        stonith_op = custom_action(NULL, key, CRM_OP_FENCE, node, optional, TRUE, data_set);

-

-        add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET, node->details->uname);

-        add_hash_param(stonith_op->meta, XML_LRM_ATTR_TARGET_UUID, node->details->id);

-        add_hash_param(stonith_op->meta, "stonith_action", op);

-    }

-

-    if(optional == FALSE) {

-        crm_trace("%s is no longer optional", stonith_op->uuid);

-        pe_clear_action_bit(stonith_op, pe_action_optional);

-    }

-

-    return stonith_op;

-}

-

 /*

  * Create dependancies for stonith and shutdown operations

  */

diff --git a/pengine/allocate.h b/pengine/allocate.h

index 712de60..0a352fd 100644

--- a/pengine/allocate.h

+++ b/pengine/allocate.h

@@ -51,8 +51,6 @@ struct resource_alloc_functions_s {

     void (*append_meta) (resource_t * rsc, xmlNode * xml);

 };

-action_t *pe_fence_op(node_t * node, const char *op, bool optional, pe_working_set_t * data_set);

-

 extern GHashTable *rsc_merge_weights(resource_t * rsc, const char *rhs, GHashTable * nodes,

                                      const char *attr, float factor, enum pe_weights flags);

@@ -161,8 +159,6 @@ extern void rsc_migrate_reload(resource_t * rsc, pe_working_set_t * data_set);

 extern void rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op,

                                  pe_working_set_t * data_set);

-void trigger_unfencing(resource_t * rsc, node_t *node, const char *reason, action_t *dependancy, pe_working_set_t * data_set);

-

 extern enum pe_graph_flags native_update_actions(action_t * first, action_t * then, node_t * node,

                                                  enum pe_action_flags flags,

                                                  enum pe_action_flags filter,

diff --git a/pengine/native.c b/pengine/native.c

index d0eb950..3d75dc9 100644

--- a/pengine/native.c

+++ b/pengine/native.c

@@ -1253,14 +1253,15 @@ native_internal_constraints(resource_t * rsc, pe_working_set_t * data_set)

     if (is_stonith == FALSE

         && is_set(data_set->flags, pe_flag_enable_unfencing)

-        && is_set(rsc->flags, pe_rsc_needs_unfencing)) {

+        && is_set(rsc->flags, pe_rsc_needs_unfencing)

+        && is_not_set(rsc->flags, pe_rsc_have_unfencing)) {

         /* Check if the node needs to be unfenced first */

         node_t *node = NULL;

         GHashTableIter iter;

         if(rsc != top) {

             /* Only create these constraints once, rsc is almost certainly cloned */

-            clear_bit_recursive(top, pe_rsc_needs_unfencing);

+            clear_bit_recursive(top, pe_rsc_have_unfencing);

         }

         g_hash_table_iter_init(&iter, rsc->allowed_nodes);

@@ -2482,9 +2483,26 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete,

     probe = custom_action(rsc, key, RSC_STATUS, node, FALSE, TRUE, data_set);

     update_action_flags(probe, pe_action_optional | pe_action_clear);

-    if(is_set(rsc->flags, pe_rsc_fence_device)) {

-        crm_crit("TODO: %s: Trigger when probing the device or the resource that needs unfencing?", rsc->id);

+    /* If enabled, require unfencing before probing any fence devices

+     * but ensure it happens after any resources that require

+     * unfencing have been probed.

+     *

+     * Doing it the other way (requiring unfencing after probing

+     * resources that need it) would result in the node being

+     * unfenced, and all its resources being stopped, whenever a new

+     * resource is added.  Which would be highly suboptimal.

+     *

+     * So essentially, at the point the fencing device(s) have been

+     * probed, we know the state of all resources that require

+     * unfencing and that unfencing occurred.

+     */

+    if(is_set(rsc->flags, pe_rsc_fence_device) && is_set(data_set->flags, pe_flag_enable_unfencing)) {

         trigger_unfencing(NULL, node, "node discovery", probe, data_set);

+

+    } else if(is_set(rsc->flags, pe_rsc_needs_unfencing)) {

+        action_t *unfence = pe_fence_op(node, "on", TRUE, data_set);

+

+        order_actions(probe, unfence, pe_order_optional);

     }

     /*

diff --git a/pengine/test10/unfence-definition.dot b/pengine/test10/unfence-definition.dot

new file mode 100644

index 0000000..e737687

--- /dev/null

+++ b/pengine/test10/unfence-definition.dot

@@ -0,0 +1,102 @@

+digraph "g" {

+"all_stopped" -> "fencing_start_0 virt-1" [ style = bold]

+"all_stopped" [ style=bold color="green" fontcolor="orange"]

+"clvmd-clone_running_0" [ style=bold color="green" fontcolor="orange"]

+"clvmd-clone_start_0" -> "clvmd-clone_running_0" [ style = bold]

+"clvmd-clone_start_0" -> "clvmd:1_start_0 virt-2" [ style = bold]

+"clvmd-clone_start_0" -> "clvmd:2_start_0 virt-3" [ style = bold]

+"clvmd-clone_start_0" -> "clvmd_start_0 virt-1" [ style = bold]

+"clvmd-clone_start_0" [ style=bold color="green" fontcolor="orange"]

+"clvmd-clone_stop_0" -> "clvmd-clone_stopped_0" [ style = bold]

+"clvmd-clone_stop_0" -> "clvmd_stop_0 virt-1" [ style = bold]

+"clvmd-clone_stop_0" [ style=bold color="green" fontcolor="orange"]

+"clvmd-clone_stopped_0" -> "clvmd-clone_start_0" [ style = bold]

+"clvmd-clone_stopped_0" -> "dlm-clone_stop_0" [ style = bold]

+"clvmd-clone_stopped_0" -> "stonith 'on' virt-1" [ style = bold]

+"clvmd-clone_stopped_0" -> "stonith 'on' virt-3" [ style = bold]

+"clvmd-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]

+"clvmd:1_monitor_0 virt-2" -> "probe_complete virt-2" [ style = bold]

+"clvmd:1_monitor_0 virt-2" [ style=bold color="green" fontcolor="black"]

+"clvmd:1_start_0 virt-2" -> "clvmd-clone_running_0" [ style = bold]

+"clvmd:1_start_0 virt-2" -> "clvmd:2_start_0 virt-3" [ style = bold]

+"clvmd:1_start_0 virt-2" [ style=bold color="green" fontcolor="black"]

+"clvmd:2_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold]

+"clvmd:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold]

+"clvmd:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"]

+"clvmd:2_start_0 virt-3" -> "clvmd-clone_running_0" [ style = bold]

+"clvmd:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"]

+"clvmd_start_0 virt-1" -> "clvmd-clone_running_0" [ style = bold]

+"clvmd_start_0 virt-1" -> "clvmd:1_start_0 virt-2" [ style = bold]

+"clvmd_start_0 virt-1" [ style=bold color="green" fontcolor="black"]

+"clvmd_stop_0 virt-1" -> "all_stopped" [ style = bold]

+"clvmd_stop_0 virt-1" -> "clvmd-clone_stopped_0" [ style = bold]

+"clvmd_stop_0 virt-1" -> "clvmd_start_0 virt-1" [ style = bold]

+"clvmd_stop_0 virt-1" -> "dlm_stop_0 virt-1" [ style = bold]

+"clvmd_stop_0 virt-1" [ style=bold color="green" fontcolor="black"]

+"dlm-clone_running_0" -> "clvmd-clone_start_0" [ style = bold]

+"dlm-clone_running_0" [ style=bold color="green" fontcolor="orange"]

+"dlm-clone_start_0" -> "dlm-clone_running_0" [ style = bold]

+"dlm-clone_start_0" -> "dlm:2_start_0 virt-3" [ style = bold]

+"dlm-clone_start_0" -> "dlm_start_0 virt-1" [ style = bold]

+"dlm-clone_start_0" [ style=bold color="green" fontcolor="orange"]

+"dlm-clone_stop_0" -> "dlm-clone_stopped_0" [ style = bold]

+"dlm-clone_stop_0" -> "dlm_stop_0 virt-1" [ style = bold]

+"dlm-clone_stop_0" [ style=bold color="green" fontcolor="orange"]

+"dlm-clone_stopped_0" -> "dlm-clone_start_0" [ style = bold]

+"dlm-clone_stopped_0" -> "stonith 'on' virt-1" [ style = bold]

+"dlm-clone_stopped_0" -> "stonith 'on' virt-3" [ style = bold]

+"dlm-clone_stopped_0" [ style=bold color="green" fontcolor="orange"]

+"dlm:2_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold]

+"dlm:2_monitor_0 virt-3" -> "stonith 'on' virt-3" [ style = bold]

+"dlm:2_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"]

+"dlm:2_start_0 virt-3" -> "clvmd:2_start_0 virt-3" [ style = bold]

+"dlm:2_start_0 virt-3" -> "dlm-clone_running_0" [ style = bold]

+"dlm:2_start_0 virt-3" [ style=bold color="green" fontcolor="black"]

+"dlm_start_0 virt-1" -> "clvmd_start_0 virt-1" [ style = bold]

+"dlm_start_0 virt-1" -> "dlm-clone_running_0" [ style = bold]

+"dlm_start_0 virt-1" [ style=bold color="green" fontcolor="black"]

+"dlm_stop_0 virt-1" -> "all_stopped" [ style = bold]

+"dlm_stop_0 virt-1" -> "dlm-clone_stopped_0" [ style = bold]

+"dlm_stop_0 virt-1" -> "dlm_start_0 virt-1" [ style = bold]

+"dlm_stop_0 virt-1" [ style=bold color="green" fontcolor="black"]

+"fencing_delete_0 virt-1" -> "fencing_start_0 virt-1" [ style = bold]

+"fencing_delete_0 virt-1" [ style=bold color="green" fontcolor="black"]

+"fencing_monitor_0 virt-3" -> "probe_complete virt-3" [ style = bold]

+"fencing_monitor_0 virt-3" [ style=bold color="green" fontcolor="black"]

+"fencing_start_0 virt-1" [ style=bold color="green" fontcolor="black"]

+"fencing_stop_0 virt-1" -> "all_stopped" [ style = bold]

+"fencing_stop_0 virt-1" -> "fencing_delete_0 virt-1" [ style = bold]

+"fencing_stop_0 virt-1" -> "fencing_start_0 virt-1" [ style = bold]

+"fencing_stop_0 virt-1" [ style=bold color="green" fontcolor="black"]

+"probe_complete virt-1" -> "probe_complete" [ style = bold]

+"probe_complete virt-1" [ style=bold color="green" fontcolor="black"]

+"probe_complete virt-2" -> "probe_complete" [ style = bold]

+"probe_complete virt-2" [ style=bold color="green" fontcolor="black"]

+"probe_complete virt-3" -> "probe_complete" [ style = bold]

+"probe_complete virt-3" [ style=bold color="green" fontcolor="black"]

+"probe_complete" -> "clvmd:1_start_0 virt-2" [ style = bold]

+"probe_complete" -> "clvmd:2_start_0 virt-3" [ style = bold]

+"probe_complete" -> "clvmd_start_0 virt-1" [ style = bold]

+"probe_complete" -> "clvmd_stop_0 virt-1" [ style = bold]

+"probe_complete" -> "dlm:2_start_0 virt-3" [ style = bold]

+"probe_complete" -> "dlm_start_0 virt-1" [ style = bold]

+"probe_complete" -> "dlm_stop_0 virt-1" [ style = bold]

+"probe_complete" -> "fencing_start_0 virt-1" [ style = bold]

+"probe_complete" -> "fencing_stop_0 virt-1" [ style = bold]

+"probe_complete" [ style=bold color="green" fontcolor="orange"]

+"stonith 'on' virt-1" -> "clvmd-clone_start_0" [ style = bold]

+"stonith 'on' virt-1" -> "dlm-clone_start_0" [ style = bold]

+"stonith 'on' virt-1" [ style=bold color="green" fontcolor="black"]

+"stonith 'on' virt-3" -> "clvmd-clone_start_0" [ style = bold]

+"stonith 'on' virt-3" -> "dlm-clone_start_0" [ style = bold]

+"stonith 'on' virt-3" [ style=bold color="green" fontcolor="black"]

+"stonith 'reboot' virt-4" -> "stonith_complete" [ style = bold]

+"stonith 'reboot' virt-4" [ style=bold color="green" fontcolor="black"]

+"stonith_complete" -> "all_stopped" [ style = bold]

+"stonith_complete" -> "clvmd:1_start_0 virt-2" [ style = bold]

+"stonith_complete" -> "clvmd:2_start_0 virt-3" [ style = bold]

+"stonith_complete" -> "clvmd_start_0 virt-1" [ style = bold]

+"stonith_complete" -> "dlm:2_start_0 virt-3" [ style = bold]

+"stonith_complete" -> "dlm_start_0 virt-1" [ style = bold]

+"stonith_complete" [ style=bold color="green" fontcolor="orange"]

+}

diff --git a/pengine/test10/unfence-definition.exp b/pengine/test10/unfence-definition.exp

new file mode 100644

index 0000000..9075347

--- /dev/null

+++ b/pengine/test10/unfence-definition.exp

@@ -0,0 +1,495 @@

+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">

+  <synapse id="0">

+    <action_set>

+      <rsc_op id="16" operation="start" operation_key="fencing_start_0" on_node="virt-1" on_node_uuid="1">

+        <primitive id="fencing" class="stonith" type="fence_scsi"/>

+        <attributes CRM_meta_timeout="20000" />

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="2" operation="all_stopped" operation_key="all_stopped"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="6" operation="stop" operation_key="fencing_stop_0" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="7" operation="delete" operation_key="fencing_delete_0" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="8" operation="probe_complete" operation_key="probe_complete"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="1">

+    <action_set>

+      <rsc_op id="13" operation="monitor" operation_key="fencing_monitor_0" on_node="virt-3" on_node_uuid="3">

+        <primitive id="fencing" class="stonith" type="fence_scsi"/>

+        <attributes CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />

+      </rsc_op>

+    </action_set>

+    <inputs/>

+  </synapse>

+  <synapse id="2">

+    <action_set>

+      <rsc_op id="7" operation="delete" operation_key="fencing_delete_0" on_node="virt-1" on_node_uuid="1">

+        <primitive id="fencing" class="stonith" type="fence_scsi"/>

+        <attributes CRM_meta_timeout="20000" />

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <rsc_op id="6" operation="stop" operation_key="fencing_stop_0" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="3">

+    <action_set>

+      <rsc_op id="6" operation="stop" operation_key="fencing_stop_0" on_node="virt-1" on_node_uuid="1">

+        <primitive id="fencing" class="stonith" type="fence_scsi"/>

+        <attributes CRM_meta_timeout="20000" />

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="8" operation="probe_complete" operation_key="probe_complete"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="4">

+    <action_set>

+      <rsc_op id="18" operation="start" operation_key="dlm_start_0" internal_operation_key="dlm:0_start_0" on_node="virt-1" on_node_uuid="1">

+        <primitive id="dlm" long-id="dlm:0" class="ocf" provider="pacemaker" type="controld"/>

+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="8" operation="probe_complete" operation_key="probe_complete"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="17" operation="stop" operation_key="dlm_stop_0" internal_operation_key="dlm:0_stop_0" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="22" operation="start" operation_key="dlm-clone_start_0"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="34" operation="stonith_complete" operation_key="stonith_complete"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="5">

+    <action_set>

+      <rsc_op id="17" operation="stop" operation_key="dlm_stop_0" internal_operation_key="dlm:0_stop_0" on_node="virt-1" on_node_uuid="1">

+        <primitive id="dlm" long-id="dlm:0" class="ocf" provider="pacemaker" type="controld"/>

+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="8" operation="probe_complete" operation_key="probe_complete"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="24" operation="stop" operation_key="dlm-clone_stop_0"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="26" operation="stop" operation_key="clvmd_stop_0" internal_operation_key="clvmd:0_stop_0" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="6">

+    <action_set>

+      <rsc_op id="21" operation="start" operation_key="dlm:2_start_0" on_node="virt-3" on_node_uuid="3">

+        <primitive id="dlm" long-id="dlm:2" class="ocf" provider="pacemaker" type="controld"/>

+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="8" operation="probe_complete" operation_key="probe_complete"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="22" operation="start" operation_key="dlm-clone_start_0"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="34" operation="stonith_complete" operation_key="stonith_complete"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="7">

+    <action_set>

+      <rsc_op id="14" operation="monitor" operation_key="dlm:2_monitor_0" on_node="virt-3" on_node_uuid="3">

+        <primitive id="dlm" long-id="dlm:2" class="ocf" provider="pacemaker" type="controld"/>

+        <attributes CRM_meta_clone="2" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_op_target_rc="7" CRM_meta_timeout="20000" />

+      </rsc_op>

+    </action_set>

+    <inputs/>

+  </synapse>

+  <synapse id="8" priority="1000000">

+    <action_set>

+      <pseudo_event id="25" operation="stopped" operation_key="dlm-clone_stopped_0">

+        <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />

+      </pseudo_event>

+    </action_set>

+    <inputs>

+      <trigger>

+        <rsc_op id="17" operation="stop" operation_key="dlm_stop_0" internal_operation_key="dlm:0_stop_0" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="24" operation="stop" operation_key="dlm-clone_stop_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="9">

+    <action_set>

+      <pseudo_event id="24" operation="stop" operation_key="dlm-clone_stop_0">

+        <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />

+      </pseudo_event>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="33" operation="stopped" operation_key="clvmd-clone_stopped_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="10" priority="1000000">

+    <action_set>

+      <pseudo_event id="23" operation="running" operation_key="dlm-clone_running_0">

+        <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />

+      </pseudo_event>

+    </action_set>

+    <inputs>

+      <trigger>

+        <rsc_op id="18" operation="start" operation_key="dlm_start_0" internal_operation_key="dlm:0_start_0" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="21" operation="start" operation_key="dlm:2_start_0" on_node="virt-3" on_node_uuid="3"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="22" operation="start" operation_key="dlm-clone_start_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="11">

+    <action_set>

+      <pseudo_event id="22" operation="start" operation_key="dlm-clone_start_0">

+        <attributes CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />

+      </pseudo_event>

+    </action_set>

+    <inputs>

+      <trigger>

+        <crm_event id="1" operation="stonith" operation_key="stonith-virt-1-on" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+      <trigger>

+        <crm_event id="4" operation="stonith" operation_key="stonith-virt-3-on" on_node="virt-3" on_node_uuid="3"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="25" operation="stopped" operation_key="dlm-clone_stopped_0"/>

+      </trigger>

+    </inputs>

+  </synapse>

+  <synapse id="12">

+    <action_set>

+      <rsc_op id="27" operation="start" operation_key="clvmd_start_0" internal_operation_key="clvmd:0_start_0" on_node="virt-1" on_node_uuid="1">

+        <primitive id="clvmd" long-id="clvmd:0" class="ocf" provider="heartbeat" type="clvm"/>

+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="4" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />

+      </rsc_op>

+    </action_set>

+    <inputs>

+      <trigger>

+        <pseudo_event id="8" operation="probe_complete" operation_key="probe_complete"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="18" operation="start" operation_key="dlm_start_0" internal_operation_key="dlm:0_start_0" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+      <trigger>

+        <rsc_op id="26" operation="stop" operation_key="clvmd_stop_0" internal_operation_key="clvmd:0_stop_0" on_node="virt-1" on_node_uuid="1"/>

+      </trigger>

+      <trigger>

+        <pseudo_event id="30" operation="start" operation_key="clvmd-clone_start_0"/>