Blame SOURCES/pacemaker-rollup-7-1-3d781d3.patch

13f052
diff --git a/cib/io.c b/cib/io.c
13f052
index e2873a8..4e2b24a 100644
13f052
--- a/cib/io.c
13f052
+++ b/cib/io.c
13f052
@@ -254,9 +254,7 @@ readCibXmlFile(const char *dir, const char *file, gboolean discard_status)
13f052
     if (cib_writes_enabled && use_valgrind) {
13f052
         if (crm_is_true(use_valgrind) || strstr(use_valgrind, "cib")) {
13f052
             cib_writes_enabled = FALSE;
13f052
-            crm_err("*********************************************************");
13f052
             crm_err("*** Disabling disk writes to avoid confusing Valgrind ***");
13f052
-            crm_err("*********************************************************");
13f052
         }
13f052
     }
13f052
 
13f052
diff --git a/crmd/crmd_lrm.h b/crmd/crmd_lrm.h
13f052
index 81a53c5..78432df 100644
13f052
--- a/crmd/crmd_lrm.h
13f052
+++ b/crmd/crmd_lrm.h
13f052
@@ -37,6 +37,8 @@ typedef struct resource_history_s {
13f052
     GHashTable *stop_params;
13f052
 } rsc_history_t;
13f052
 
13f052
+void history_free(gpointer data);
13f052
+
13f052
 /* TDOD - Replace this with lrmd_event_data_t */
13f052
 struct recurring_op_s {
13f052
     int call_id;
13f052
diff --git a/crmd/lrm.c b/crmd/lrm.c
13f052
index 062f769..418e7cf 100644
13f052
--- a/crmd/lrm.c
13f052
+++ b/crmd/lrm.c
13f052
@@ -103,6 +103,80 @@ copy_meta_keys(gpointer key, gpointer value, gpointer user_data)
13f052
     }
13f052
 }
13f052
 
13f052
+/*
13f052
+ * \internal
13f052
+ * \brief Remove a recurring operation from a resource's history
13f052
+ *
13f052
+ * \param[in,out] history  Resource history to modify
13f052
+ * \param[in]     op       Operation to remove
13f052
+ *
13f052
+ * \return TRUE if the operation was found and removed, FALSE otherwise
13f052
+ */
13f052
+static gboolean
13f052
+history_remove_recurring_op(rsc_history_t *history, const lrmd_event_data_t *op)
13f052
+{
13f052
+    GList *iter;
13f052
+
13f052
+    for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
13f052
+        lrmd_event_data_t *existing = iter->data;
13f052
+
13f052
+        if ((op->interval == existing->interval)
13f052
+            && crm_str_eq(op->rsc_id, existing->rsc_id, TRUE)
13f052
+            && safe_str_eq(op->op_type, existing->op_type)) {
13f052
+
13f052
+            history->recurring_op_list = g_list_delete_link(history->recurring_op_list, iter);
13f052
+            lrmd_free_event(existing);
13f052
+            return TRUE;
13f052
+        }
13f052
+    }
13f052
+    return FALSE;
13f052
+}
13f052
+
13f052
+/*
13f052
+ * \internal
13f052
+ * \brief Free all recurring operations in resource history
13f052
+ *
13f052
+ * \param[in,out] history  Resource history to modify
13f052
+ */
13f052
+static void
13f052
+history_free_recurring_ops(rsc_history_t *history)
13f052
+{
13f052
+    GList *iter;
13f052
+
13f052
+    for (iter = history->recurring_op_list; iter != NULL; iter = iter->next) {
13f052
+        lrmd_free_event(iter->data);
13f052
+    }
13f052
+    g_list_free(history->recurring_op_list);
13f052
+    history->recurring_op_list = NULL;
13f052
+}
13f052
+
13f052
+/*
13f052
+ * \internal
13f052
+ * \brief Free resource history
13f052
+ *
13f052
+ * \param[in,out] history  Resource history to free
13f052
+ */
13f052
+void
13f052
+history_free(gpointer data)
13f052
+{
13f052
+    rsc_history_t *history = (rsc_history_t*)data;
13f052
+
13f052
+    if (history->stop_params) {
13f052
+        g_hash_table_destroy(history->stop_params);
13f052
+    }
13f052
+
13f052
+    /* Don't need to free history->rsc.id because it's set to history->id */
13f052
+    free(history->rsc.type);
13f052
+    free(history->rsc.class);
13f052
+    free(history->rsc.provider);
13f052
+
13f052
+    lrmd_free_event(history->failed);
13f052
+    lrmd_free_event(history->last);
13f052
+    free(history->id);
13f052
+    history_free_recurring_ops(history);
13f052
+    free(history);
13f052
+}
13f052
+
13f052
 static void
13f052
 update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_data_t * op)
13f052
 {
13f052
@@ -145,25 +219,10 @@ update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_
13f052
     target_rc = rsc_op_expected_rc(op);
13f052
     if (op->op_status == PCMK_LRM_OP_CANCELLED) {
13f052
         if (op->interval > 0) {
13f052
-            GList *gIter, *gIterNext;
13f052
-
13f052
             crm_trace("Removing cancelled recurring op: %s_%s_%d", op->rsc_id, op->op_type,
13f052
                       op->interval);
13f052
-
13f052
-            for (gIter = entry->recurring_op_list; gIter != NULL; gIter = gIterNext) {
13f052
-                lrmd_event_data_t *existing = gIter->data;
13f052
-
13f052
-                gIterNext = gIter->next;
13f052
-
13f052
-                if (crm_str_eq(op->rsc_id, existing->rsc_id, TRUE)
13f052
-                    && safe_str_eq(op->op_type, existing->op_type)
13f052
-                    && op->interval == existing->interval) {
13f052
-                    lrmd_free_event(existing);
13f052
-                    entry->recurring_op_list = g_list_delete_link(entry->recurring_op_list, gIter);
13f052
-                }
13f052
-            }
13f052
+            history_remove_recurring_op(entry, op);
13f052
             return;
13f052
-
13f052
         } else {
13f052
             crm_trace("Skipping %s_%s_%d rc=%d, status=%d", op->rsc_id, op->op_type, op->interval,
13f052
                       op->rc, op->op_status);
13f052
@@ -201,32 +260,17 @@ update_history_cache(lrm_state_t * lrm_state, lrmd_rsc_info_t * rsc, lrmd_event_
13f052
     }
13f052
 
13f052
     if (op->interval > 0) {
13f052
-        GListPtr iter = NULL;
13f052
-
13f052
-        for(iter = entry->recurring_op_list; iter; iter = iter->next) {
13f052
-            lrmd_event_data_t *o = iter->data;
13f052
-
13f052
-            /* op->rsc_id is implied */
13f052
-            if(op->interval == o->interval && strcmp(op->op_type, o->op_type) == 0) {
13f052
-                crm_trace("Removing existing recurring op entry: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
13f052
-                entry->recurring_op_list = g_list_remove(entry->recurring_op_list, o);
13f052
-                break;
13f052
-            }
13f052
-        }
13f052
+        /* Ensure there are no duplicates */
13f052
+        history_remove_recurring_op(entry, op);
13f052
 
13f052
         crm_trace("Adding recurring op: %s_%s_%d", op->rsc_id, op->op_type, op->interval);
13f052
         entry->recurring_op_list = g_list_prepend(entry->recurring_op_list, lrmd_copy_event(op));
13f052
 
13f052
     } else if (entry->recurring_op_list && safe_str_eq(op->op_type, RSC_STATUS) == FALSE) {
13f052
-        GList *gIter = entry->recurring_op_list;
13f052
-
13f052
         crm_trace("Dropping %d recurring ops because of: %s_%s_%d",
13f052
-                  g_list_length(gIter), op->rsc_id, op->op_type, op->interval);
13f052
-        for (; gIter != NULL; gIter = gIter->next) {
13f052
-            lrmd_free_event(gIter->data);
13f052
-        }
13f052
-        g_list_free(entry->recurring_op_list);
13f052
-        entry->recurring_op_list = NULL;
13f052
+                  g_list_length(entry->recurring_op_list), op->rsc_id,
13f052
+                  op->op_type, op->interval);
13f052
+        history_free_recurring_ops(entry);
13f052
     }
13f052
 }
13f052
 
13f052
diff --git a/crmd/lrm_state.c b/crmd/lrm_state.c
13f052
index 374c806..162ad03 100644
13f052
--- a/crmd/lrm_state.c
13f052
+++ b/crmd/lrm_state.c
13f052
@@ -32,24 +32,6 @@ int lrmd_internal_proxy_send(lrmd_t * lrmd, xmlNode *msg);
13f052
 void lrmd_internal_set_proxy_callback(lrmd_t * lrmd, void *userdata, void (*callback)(lrmd_t *lrmd, void *userdata, xmlNode *msg));
13f052
 
13f052
 static void
13f052
-history_cache_destroy(gpointer data)
13f052
-{
13f052
-    rsc_history_t *entry = data;
13f052
-
13f052
-    if (entry->stop_params) {
13f052
-        g_hash_table_destroy(entry->stop_params);
13f052
-    }
13f052
-
13f052
-    free(entry->rsc.type);
13f052
-    free(entry->rsc.class);
13f052
-    free(entry->rsc.provider);
13f052
-
13f052
-    lrmd_free_event(entry->failed);
13f052
-    lrmd_free_event(entry->last);
13f052
-    free(entry->id);
13f052
-    free(entry);
13f052
-}
13f052
-static void
13f052
 free_rsc_info(gpointer value)
13f052
 {
13f052
     lrmd_rsc_info_t *rsc_info = value;
13f052
@@ -155,7 +137,7 @@ lrm_state_create(const char *node_name)
13f052
                                                g_str_equal, g_hash_destroy_str, free_recurring_op);
13f052
 
13f052
     state->resource_history = g_hash_table_new_full(crm_str_hash,
13f052
-                                                    g_str_equal, NULL, history_cache_destroy);
13f052
+                                                    g_str_equal, NULL, history_free);
13f052
 
13f052
     g_hash_table_insert(lrm_state_table, (char *)state->node_name, state);
13f052
     return state;
13f052
diff --git a/cts/CM_ais.py b/cts/CM_ais.py
13f052
index 44f91cd..a34f9b1 100644
13f052
--- a/cts/CM_ais.py
13f052
+++ b/cts/CM_ais.py
13f052
@@ -49,42 +49,46 @@ class crm_ais(crm_lha):
13f052
     def NodeUUID(self, node):
13f052
         return node
13f052
 
13f052
-    def ais_components(self):
13f052
+    def ais_components(self, extra={}):
13f052
 
13f052
         complist = []
13f052
         if not len(self.fullcomplist.keys()):
13f052
             for c in ["cib", "lrmd", "crmd", "attrd" ]:
13f052
-               self.fullcomplist[c] = Process(
13f052
-                   self, c, 
13f052
-                   pats = self.templates.get_component(self.name, c),
13f052
-                   badnews_ignore = self.templates.get_component(self.name, "%s-ignore"%c),
13f052
-                   common_ignore = self.templates.get_component(self.name, "common-ignore"))
13f052
-
13f052
-               self.fullcomplist["pengine"] = Process(
13f052
-                   self, "pengine", 
13f052
-                   dc_pats = self.templates.get_component(self.name, "pengine"),
13f052
-                   badnews_ignore = self.templates.get_component(self.name, "pengine-ignore"),
13f052
-                   common_ignore = self.templates.get_component(self.name, "common-ignore"))
13f052
-
13f052
-               self.fullcomplist["stonith-ng"] = Process(
13f052
-                   self, "stonith-ng", process="stonithd", 
13f052
-                   pats = self.templates.get_component(self.name, "stonith"),
13f052
-                   badnews_ignore = self.templates.get_component(self.name, "stonith-ignore"),
13f052
-                   common_ignore = self.templates.get_component(self.name, "common-ignore"))
13f052
-
13f052
+                self.fullcomplist[c] = Process(
13f052
+                    self, c, 
13f052
+                    pats = self.templates.get_component(self.name, c),
13f052
+                    badnews_ignore = self.templates.get_component(self.name, "%s-ignore" % c),
13f052
+                    common_ignore = self.templates.get_component(self.name, "common-ignore"))
13f052
+
13f052
+            # pengine uses dc_pats instead of pats
13f052
+            self.fullcomplist["pengine"] = Process(
13f052
+                self, "pengine", 
13f052
+                dc_pats = self.templates.get_component(self.name, "pengine"),
13f052
+                badnews_ignore = self.templates.get_component(self.name, "pengine-ignore"),
13f052
+                common_ignore = self.templates.get_component(self.name, "common-ignore"))
13f052
+
13f052
+            # stonith-ng's process name is different from its component name
13f052
+            self.fullcomplist["stonith-ng"] = Process(
13f052
+                self, "stonith-ng", process="stonithd", 
13f052
+                pats = self.templates.get_component(self.name, "stonith"),
13f052
+                badnews_ignore = self.templates.get_component(self.name, "stonith-ignore"),
13f052
+                common_ignore = self.templates.get_component(self.name, "common-ignore"))
13f052
+
13f052
+            # add (or replace) any extra components passed in
13f052
+            self.fullcomplist.update(extra)
13f052
+
13f052
+        # Processes running under valgrind can't be shot with "killall -9 processname",
13f052
+        # so don't include them in the returned list
13f052
         vgrind = self.Env["valgrind-procs"].split()
13f052
         for key in self.fullcomplist.keys():
13f052
             if self.Env["valgrind-tests"]:
13f052
-               if key in vgrind:
13f052
-               # Processes running under valgrind can't be shot with "killall -9 processname"
13f052
+                if key in vgrind:
13f052
                     self.log("Filtering %s from the component list as it is being profiled by valgrind" % key)
13f052
                     continue
13f052
             if key == "stonith-ng" and not self.Env["DoFencing"]:
13f052
                 continue
13f052
-
13f052
             complist.append(self.fullcomplist[key])
13f052
 
13f052
-        #self.complist = [ fullcomplist["pengine"] ]
13f052
         return complist
13f052
 
13f052
 
13f052
@@ -100,17 +104,14 @@ class crm_cs_v0(crm_ais):
13f052
         crm_ais.__init__(self, Environment, randseed=randseed, name=name)
13f052
 
13f052
     def Components(self):
13f052
-        self.ais_components()
13f052
-        c = "corosync"
13f052
-
13f052
-        self.fullcomplist[c] = Process(
13f052
-            self, c, 
13f052
-            pats = self.templates.get_component(self.name, c),
13f052
-            badnews_ignore = self.templates.get_component(self.name, "%s-ignore"%c),
13f052
+        extra = {}
13f052
+        extra["corosync"] = Process(
13f052
+            self, "corosync", 
13f052
+            pats = self.templates.get_component(self.name, "corosync"),
13f052
+            badnews_ignore = self.templates.get_component(self.name, "corosync-ignore"),
13f052
             common_ignore = self.templates.get_component(self.name, "common-ignore")
13f052
         )
13f052
-
13f052
-        return self.ais_components()
13f052
+        return self.ais_components(extra=extra)
13f052
 
13f052
 
13f052
 class crm_cs_v1(crm_cs_v0):
13f052
diff --git a/cts/environment.py b/cts/environment.py
13f052
index a3399c3..61d4211 100644
13f052
--- a/cts/environment.py
13f052
+++ b/cts/environment.py
13f052
@@ -59,7 +59,7 @@ class Environment:
13f052
         self["stonith-params"] = "hostlist=all,livedangerously=yes"
13f052
         self["loop-minutes"] = 60
13f052
         self["valgrind-prefix"] = None
13f052
-        self["valgrind-procs"] = "cib crmd attrd pengine stonith-ng"
13f052
+        self["valgrind-procs"] = "attrd cib crmd lrmd pengine stonith-ng"
13f052
         self["valgrind-opts"] = """--leak-check=full --show-reachable=yes --trace-children=no --num-callers=25 --gen-suppressions=all --suppressions="""+CTSvars.CTS_home+"""/cts.supp"""
13f052
 
13f052
         self["experimental-tests"] = 0
13f052
@@ -578,6 +578,10 @@ class Environment:
13f052
             elif args[i] == "--valgrind-tests":
13f052
                 self["valgrind-tests"] = 1
13f052
 
13f052
+            elif args[i] == "--valgrind-procs":
13f052
+                self["valgrind-procs"] = args[i+1]
13f052
+                skipthis = 1
13f052
+
13f052
             elif args[i] == "--no-loop-tests":
13f052
                 self["loop-tests"] = 0
13f052
 
13f052
diff --git a/cts/patterns.py b/cts/patterns.py
13f052
index 1bc05a6..493b690 100644
13f052
--- a/cts/patterns.py
13f052
+++ b/cts/patterns.py
13f052
@@ -7,7 +7,9 @@ class BasePatterns:
13f052
     def __init__(self, name):
13f052
         self.name = name
13f052
         patternvariants[name] = self
13f052
-        self.ignore = []
13f052
+        self.ignore = [
13f052
+            "avoid confusing Valgrind",
13f052
+        ]
13f052
         self.BadNews = []
13f052
         self.components = {}
13f052
         self.commands = {
13f052
@@ -140,7 +142,7 @@ class crm_lha(BasePatterns):
13f052
                 r"Parameters to .* changed",
13f052
             ]
13f052
 
13f052
-        self.ignore = [
13f052
+        self.ignore = self.ignore + [
13f052
                 r"(ERROR|error):.*\s+assert\s+at\s+crm_glib_handler:"
13f052
                 "(ERROR|error): Message hist queue is filling up",
13f052
                 "stonithd.*CRIT: external_hostlist:.*'vmware gethosts' returned an empty hostlist",
13f052
@@ -177,7 +179,7 @@ class crm_cs_v0(BasePatterns):
13f052
             "Pat:PacemakerUp"  : "%s\W.*pacemakerd.*Starting Pacemaker",
13f052
         })
13f052
 
13f052
-        self.ignore = [
13f052
+        self.ignore = self.ignore + [
13f052
             r"crm_mon:",
13f052
             r"crmadmin:",
13f052
             r"update_trace_data",
13f052
diff --git a/extra/ansible/docker/group_vars/all b/extra/ansible/docker/group_vars/all
13f052
new file mode 100644
13f052
index 0000000..935e88a
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/group_vars/all
13f052
@@ -0,0 +1,5 @@
13f052
+max: 4
13f052
+prefix: ansible-pcmk
13f052
+base_image: centos:centos7
13f052
+subnet: 172.17.200
13f052
+pacemaker_authkey: this_is_very_insecure
13f052
\ No newline at end of file
13f052
diff --git a/extra/ansible/docker/hosts b/extra/ansible/docker/hosts
13f052
new file mode 100644
13f052
index 0000000..5b0fb71
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/hosts
13f052
@@ -0,0 +1,7 @@
13f052
+[controllers]
13f052
+oss-uk-1.clusterlabs.org
13f052
+
13f052
+[containers]
13f052
+ansible-1
13f052
+ansible-2
13f052
+ansible-3
13f052
diff --git a/extra/ansible/docker/roles/docker-host/files/docker-enter b/extra/ansible/docker/roles/docker-host/files/docker-enter
13f052
new file mode 100644
13f052
index 0000000..04c4822
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/files/docker-enter
13f052
@@ -0,0 +1,29 @@
13f052
+#! /bin/sh -e
13f052
+
13f052
+case "$1" in
13f052
+  -h|--help)
13f052
+    echo "Usage: docker-enter CONTAINER [COMMAND]"
13f052
+    exit 0
13f052
+    ;;
13f052
+esac
13f052
+
13f052
+if [ $(id -ru) -ne 0 ]; then
13f052
+  echo "You have to be root."
13f052
+  exit 1
13f052
+fi
13f052
+
13f052
+if [ $# -eq 0 ]; then
13f052
+  echo "Usage: docker-enter CONTAINER [COMMAND]"
13f052
+  exit 1
13f052
+fi
13f052
+
13f052
+container=$1; shift
13f052
+PID=$(docker inspect --format {{.State.Pid}} "$container")
13f052
+
13f052
+if [ $# -ne 0 ]; then
13f052
+   nsenter --target $PID --mount --uts --ipc --net --pid -- $*
13f052
+   exit $?
13f052
+fi
13f052
+
13f052
+nsenter --target $PID --mount --uts --ipc --net --pid
13f052
+exit 0
13f052
diff --git a/extra/ansible/docker/roles/docker-host/files/fence_docker_cts b/extra/ansible/docker/roles/docker-host/files/fence_docker_cts
13f052
new file mode 100644
13f052
index 0000000..6d6f025
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/files/fence_docker_cts
13f052
@@ -0,0 +1,202 @@
13f052
+#!/bin/bash
13f052
+#
13f052
+# Copyright (c) 2014 David Vossel <dvossel@redhat.com>
13f052
+#					All Rights Reserved.
13f052
+#
13f052
+# This program is free software; you can redistribute it and/or modify
13f052
+# it under the terms of version 2 of the GNU General Public License as
13f052
+# published by the Free Software Foundation.
13f052
+#
13f052
+# This program is distributed in the hope that it would be useful, but
13f052
+# WITHOUT ANY WARRANTY; without even the implied warranty of
13f052
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
13f052
+#
13f052
+# Further, this software is distributed without any warranty that it is
13f052
+# free of the rightful claim of any third person regarding infringement
13f052
+# or the like.  Any license provided herein, whether implied or
13f052
+# otherwise, applies only to this software file.  Patent licenses, if
13f052
+# any, provided herein do not apply to combinations of this program with
13f052
+# other software, or any other product whatsoever.
13f052
+#
13f052
+# You should have received a copy of the GNU General Public License
13f052
+# along with this program; if not, write the Free Software Foundation,
13f052
+# Inc., 59 Temple Place - Suite 330, Boston MA 02111-1307, USA.
13f052
+#
13f052
+#######################################################################
13f052
+
13f052
+port=""
13f052
+action="list"		 # Default fence action
13f052
+
13f052
+function usage()
13f052
+{
13f052
+cat <
13f052
+`basename $0` - A fencing agent for docker containers for testing purposes
13f052
+ 
13f052
+Usage: `basename $0` -o|--action [-n|--port] [options]
13f052
+Options:
13f052
+ -h, --help 		This text
13f052
+ -V, --version		Version information
13f052
+ 
13f052
+Commands:
13f052
+ -o, --action		Action to perform: on|off|reboot|status|monitor
13f052
+ -n, --port 		The name of a container to control/check
13f052
+
13f052
+EOF
13f052
+	exit 0;
13f052
+}
13f052
+
13f052
+function metadata()
13f052
+{
13f052
+cat <
13f052
+
13f052
+<resource-agent name="fence_docker_cts" shortdesc="docker fencing agent for testing purposes" >
13f052
+	<longdesc>
13f052
+		fence_docker_cts fences docker containers for testing purposes.
13f052
+	</longdesc>
13f052
+	<parameters>
13f052
+	<parameter name="action" unique="1" required="0">
13f052
+		<getopt mixed="-o, --action=[action]" />
13f052
+		<content type="string" default="reboot" />
13f052
+		<shortdesc lang="en">Fencing Action</shortdesc>
13f052
+	</parameter>
13f052
+	<parameter name="port" unique="1" required="0">
13f052
+		<getopt mixed="-n, --port=[port]" />
13f052
+		<content type="string" />
13f052
+		<shortdesc lang="en">The name/id of docker container to control/check</shortdesc>
13f052
+	</parameter>
13f052
+	</parameters>
13f052
+	<actions>
13f052
+	<action name="on" />
13f052
+	<action name="off" />
13f052
+	<action name="reboot" />
13f052
+	<action name="status" />
13f052
+	<action name="list" />
13f052
+	<action name="monitor" />
13f052
+	<action name="metadata" />
13f052
+	</actions>
13f052
+</resource-agent>
13f052
+EOF
13f052
+	exit 0;
13f052
+}
13f052
+
13f052
+function docker_log() {
13f052
+	if ! [ "$action" = "list" ]; then
13f052
+		printf "$*\n" 1>&2
13f052
+	fi
13f052
+}
13f052
+
13f052
+# stdin option processing
13f052
+if [ -z $1 ]; then
13f052
+	# If there are no command line args, look for options from stdin
13f052
+	while read line; do
13f052
+		for word in $(echo "$line"); do
13f052
+			case $word in 
13f052
+			option=*|action=*) action=`echo $word | sed s/.*=//`;;
13f052
+			port=*)			port=`echo $word | sed s/.*=//`;;
13f052
+			node=*)			port=`echo $word | sed s/.*=//`;;
13f052
+			nodename=*)			port=`echo $word | sed s/.*=//`;;
13f052
+			--);;
13f052
+			*) docker_log "Invalid command: $word";;
13f052
+			esac
13f052
+		done
13f052
+	done
13f052
+fi
13f052
+
13f052
+# Command line option processing
13f052
+while true ; do
13f052
+	if [ -z "$1" ]; then
13f052
+		break;
13f052
+	fi
13f052
+	case "$1" in
13f052
+	-o|--action|--option) action=$2;	shift; shift;;
13f052
+	-n|--port)			port=$2;	  shift; shift;;
13f052
+	-V|--version) echo "1.0.0"; exit 0;;
13f052
+	--help|-h) 
13f052
+		usage;
13f052
+		exit 0;;
13f052
+	--) shift ; break ;;
13f052
+	*) docker_log "Unknown option: $1. See --help for details."; exit 1;;
13f052
+	esac
13f052
+done
13f052
+
13f052
+action=`echo $action | tr 'A-Z' 'a-z'`
13f052
+case $action in
13f052
+	hostlist|list) action=list;;
13f052
+	stat|status)   action=status;;
13f052
+	restart|reboot|reset)  action=reboot;;
13f052
+	poweron|on)	action=start;;
13f052
+	poweroff|off)  action=stop;;
13f052
+esac
13f052
+
13f052
+function fence_done() 
13f052
+{
13f052
+	if [ $1 -eq 0 ]; then
13f052
+		docker_log "Operation $action (port=$port) passed"
13f052
+	else
13f052
+		docker_log "Operation $action (port=$port) failed: $1"
13f052
+	fi
13f052
+	if [ -z "$returnfile" ]; then
13f052
+		rm -f $returnfile
13f052
+	fi
13f052
+	if [ -z "$helperscript" ]; then
13f052
+		rm -f $helperscript
13f052
+	fi
13f052
+	exit $1
13f052
+}
13f052
+
13f052
+case $action in 
13f052
+	metadata) metadata;;
13f052
+esac
13f052
+
13f052
+returnfile=$(mktemp /tmp/fence_docker_cts_returnfileXXXX)
13f052
+returnstring=""
13f052
+helper_script=$(mktemp /tmp/fence_docker_cts_helperXXXX)
13f052
+
13f052
+exec_action()
13f052
+{
13f052
+	echo "#!/bin/bash" > $helper_script
13f052
+	echo "sleep 10000" >> $helper_script
13f052
+	chmod 755 $helper_script
13f052
+	src="$(uname -n)"
13f052
+
13f052
+	$helper_script "$src" "$action" "$returnfile" "$port" > /dev/null 2>&1 &
13f052
+	pid=$!
13f052
+	docker_log "waiting on pid $pid"
13f052
+	wait $pid > /dev/null 2>&1
13f052
+	returnstring=$(cat $returnfile)
13f052
+
13f052
+	if [ -z "$returnstring" ]; then
13f052
+		docker_log "fencing daemon did not respond"
13f052
+		fence_done 1
13f052
+	fi
13f052
+
13f052
+	if [ "$returnstring" == "fail" ]; then
13f052
+		docker_log "fencing daemon failed to execute action [$action on port $port]"
13f052
+		fence_done 1
13f052
+	fi
13f052
+
13f052
+	return 0
13f052
+}
13f052
+
13f052
+exec_action
13f052
+case $action in 
13f052
+	list)
13f052
+		cat $returnfile
13f052
+		fence_done 0
13f052
+		;;
13f052
+
13f052
+	status)
13f052
+		# 0 if container is on
13f052
+		# 1 if container can not be contacted or unknown
13f052
+		# 2 if container is off
13f052
+		if [ "$returnstring" = "true" ]; then
13f052
+			fence_done 0
13f052
+		else 
13f052
+			fence_done 2
13f052
+		fi
13f052
+		;;
13f052
+	monitor|stop|start|reboot) : ;;
13f052
+	*) docker_log "Unknown action: $action"; fence_done 1;;
13f052
+esac
13f052
+
13f052
+fence_done $?
13f052
diff --git a/extra/ansible/docker/roles/docker-host/files/launch.sh b/extra/ansible/docker/roles/docker-host/files/launch.sh
13f052
new file mode 100644
13f052
index 0000000..66bebf4
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/files/launch.sh
13f052
@@ -0,0 +1,4 @@
13f052
+#!/bin/bash
13f052
+while true; do
13f052
+	sleep 1
13f052
+done
13f052
diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_remote_start b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_start
13f052
new file mode 100644
13f052
index 0000000..1bf0320
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_start
13f052
@@ -0,0 +1,18 @@
13f052
+#!/bin/bash
13f052
+/usr/sbin/ip_start
13f052
+pid=$(pidof pacemaker_remoted)
13f052
+if [ "$?" -ne 0 ];  then
13f052
+	mkdir -p /var/run
13f052
+
13f052
+	export PCMK_debugfile=$pcmklogs
13f052
+	(pacemaker_remoted &) & > /dev/null 2>&1
13f052
+	sleep 5
13f052
+
13f052
+	pid=$(pidof pacemaker_remoted)
13f052
+	if [ "$?" -ne 0 ]; then
13f052
+		echo "startup of pacemaker failed"
13f052
+		exit 1
13f052
+	fi
13f052
+	echo "$pid" > /var/run/pacemaker_remoted.pid
13f052
+fi
13f052
+exit 0
13f052
diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_remote_stop b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_stop
13f052
new file mode 100644
13f052
index 0000000..074cd59
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/files/pcmk_remote_stop
13f052
@@ -0,0 +1,36 @@
13f052
+#!/bin/bash
13f052
+status()
13f052
+{
13f052
+	pid=$(pidof $1 2>/dev/null)
13f052
+	rtrn=$?
13f052
+	if [ $rtrn -ne 0 ]; then
13f052
+		echo "$1 is stopped"
13f052
+	else
13f052
+		echo "$1 (pid $pid) is running..."
13f052
+	fi
13f052
+	return $rtrn
13f052
+}
13f052
+stop()
13f052
+{
13f052
+	desc="Pacemaker Remote"
13f052
+	prog=$1
13f052
+	shutdown_prog=$prog
13f052
+
13f052
+	if status $shutdown_prog > /dev/null 2>&1; then
13f052
+	    kill -TERM $(pidof $prog) > /dev/null 2>&1
13f052
+
13f052
+	    while status $prog > /dev/null 2>&1; do
13f052
+		sleep 1
13f052
+		echo -n "."
13f052
+	    done
13f052
+	else
13f052
+	    echo -n "$desc is already stopped"
13f052
+	fi
13f052
+
13f052
+	rm -f /var/lock/subsystem/pacemaker
13f052
+	rm -f /var/run/${prog}.pid
13f052
+	killall -q -9 'crmd stonithd attrd cib lrmd pacemakerd pacemaker_remoted'
13f052
+}
13f052
+
13f052
+stop "pacemaker_remoted"
13f052
+exit 0
13f052
diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_start b/extra/ansible/docker/roles/docker-host/files/pcmk_start
13f052
new file mode 100644
13f052
index 0000000..d8b2ba8
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/files/pcmk_start
13f052
@@ -0,0 +1,23 @@
13f052
+#!/bin/bash
13f052
+
13f052
+/usr/sbin/ip_start
13f052
+sed -i 's@to_syslog:.*yes@to_logfile: yes\nlogfile: /var/log/pacemaker.log@g' /etc/corosync/corosync.conf
13f052
+
13f052
+/usr/share/corosync/corosync start > /dev/null 2>&1
13f052
+
13f052
+pid=$(pidof pacemakerd)
13f052
+if [ "$?" -ne 0 ];  then
13f052
+	mkdir -p /var/run
13f052
+
13f052
+	export PCMK_debugfile=$pcmklogs
13f052
+	(pacemakerd &) & > /dev/null 2>&1
13f052
+	sleep 5
13f052
+
13f052
+	pid=$(pidof pacemakerd)
13f052
+	if [ "$?" -ne 0 ]; then
13f052
+		echo "startup of pacemaker failed"
13f052
+		exit 1
13f052
+	fi
13f052
+	echo "$pid" > /var/run/pacemakerd.pid
13f052
+fi
13f052
+exit 0
13f052
diff --git a/extra/ansible/docker/roles/docker-host/files/pcmk_stop b/extra/ansible/docker/roles/docker-host/files/pcmk_stop
13f052
new file mode 100644
13f052
index 0000000..a8f395a
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/files/pcmk_stop
13f052
@@ -0,0 +1,45 @@
13f052
+#!/bin/bash
13f052
+status()
13f052
+{
13f052
+	pid=$(pidof $1 2>/dev/null)
13f052
+	rtrn=$?
13f052
+	if [ $rtrn -ne 0 ]; then
13f052
+		echo "$1 is stopped"
13f052
+	else
13f052
+		echo "$1 (pid $pid) is running..."
13f052
+	fi
13f052
+	return $rtrn
13f052
+}
13f052
+stop()
13f052
+{
13f052
+	desc="Pacemaker Cluster Manager"
13f052
+	prog=$1
13f052
+	shutdown_prog=$prog
13f052
+
13f052
+	if ! status $prog > /dev/null 2>&1; then
13f052
+	    shutdown_prog="crmd"
13f052
+	fi
13f052
+
13f052
+	cname=$(crm_node --name)
13f052
+	crm_attribute -N $cname -n standby -v true -l reboot
13f052
+
13f052
+	if status $shutdown_prog > /dev/null 2>&1; then
13f052
+	    kill -TERM $(pidof $prog) > /dev/null 2>&1
13f052
+
13f052
+	    while status $prog > /dev/null 2>&1; do
13f052
+		sleep 1
13f052
+		echo -n "."
13f052
+	    done
13f052
+	else
13f052
+	    echo -n "$desc is already stopped"
13f052
+	fi
13f052
+
13f052
+	rm -f /var/lock/subsystem/pacemaker
13f052
+	rm -f /var/run/${prog}.pid
13f052
+	killall -q -9 'crmd stonithd attrd cib lrmd pacemakerd pacemaker_remoted'
13f052
+}
13f052
+
13f052
+stop "pacemakerd"
13f052
+/usr/share/corosync/corosync stop > /dev/null 2>&1
13f052
+killall -q -9 'corosync'
13f052
+exit 0
13f052
diff --git a/extra/ansible/docker/roles/docker-host/tasks/main.yml b/extra/ansible/docker/roles/docker-host/tasks/main.yml
13f052
new file mode 100644
13f052
index 0000000..ce69adf
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/tasks/main.yml
13f052
@@ -0,0 +1,77 @@
13f052
+---
13f052
+#local_action: command /usr/bin/take_out_of_pool {{ inventory_hostname }}
13f052
+- name: Update docker  
13f052
+  yum: pkg=docker state=latest
13f052
+- name: Start docker
13f052
+  service: name=docker state=started enabled=yes
13f052
+- name: Install helper
13f052
+  copy: src=docker-enter dest=/usr/sbin/ mode=0755
13f052
+- name: Download image
13f052
+  shell: docker pull {{ base_image }}
13f052
+- name: Cleanup kill
13f052
+  shell: docker kill $(docker ps -a | grep {{ prefix }} | awk '{print $1}') || echo "Nothing to kill"
13f052
+- name: Cleanup remove
13f052
+  shell: docker rm $(docker ps -a | grep {{ prefix }} | awk '{print $1}') || echo "Nothing to remove"
13f052
+- name: Cleanup docker skeleton
13f052
+  file: path={{ prefix }} state=absent
13f052
+- name: Create docker skeleton
13f052
+  file: path={{ prefix }}/{{ item }} state=directory recurse=yes
13f052
+  with_items:
13f052
+  - rpms
13f052
+  - repos
13f052
+  - bin_files
13f052
+  - launch_scripts
13f052
+- name: Create IP helper
13f052
+  template: src=ip_start.j2 dest={{ prefix }}/bin_files/ip_start mode=0755
13f052
+- name: Copy helper scripts
13f052
+  copy: src={{ item }} dest={{ prefix }}/bin_files/{{ item }} mode=0755
13f052
+  with_items:
13f052
+  - pcmk_stop
13f052
+  - pcmk_start
13f052
+  - pcmk_remote_stop
13f052
+  - pcmk_remote_start
13f052
+  - fence_docker_cts
13f052
+- name: Copy launch script
13f052
+  copy: src=launch.sh dest={{ prefix }}/launch_scripts/launch.sh mode=0755
13f052
+- name: Copy authorized keys
13f052
+  shell: cp /root/.ssh/authorized_keys {{ prefix }}
13f052
+- name: Create docker file
13f052
+  template: src=Dockerfile.j2 dest={{ prefix }}/Dockerfile
13f052
+- name: Making image
13f052
+  shell: docker build -t {{ prefix }} {{ prefix }}
13f052
+- name: Launch images
13f052
+  shell: docker run -d -i -t -P -h {{ prefix }}-{{ item }} --name={{ prefix }}-{{ item }} -p 2200{{ item }}:22 $(docker images | grep {{ prefix }}.*latest | awk '{print $3}') /bin/bash
13f052
+  with_sequence: count={{ max }}
13f052
+- name: Calculate IPs
13f052
+  shell: for n in $(seq {{ max }} ); do echo {{ subnet }}.${n}; done | tr '\n' ' '
13f052
+  register: node_ips
13f052
+- name: Start the IP
13f052
+  shell: docker-enter {{ prefix }}-{{ item }} ip_start
13f052
+  with_sequence: count={{ max }}
13f052
+- name: Configure cluster
13f052
+  shell: docker-enter {{ prefix }}-{{ item }} pcs cluster setup --local --name {{ prefix }} {{ node_ips.stdout }}
13f052
+  with_sequence: count={{ max }}
13f052
+- name: Start the cluster
13f052
+  shell: docker-enter {{ prefix }}-{{ item }} pcmk_start
13f052
+  with_sequence: count={{ max }}
13f052
+- name: Set cluster options
13f052
+  shell: docker-enter {{ prefix }}-1 pcs property set stonith-enabled=false
13f052
+- name: Configure VIP
13f052
+  shell: docker-enter {{ prefix }}-1 pcs resource create ClusterIP ocf:heartbeat:IPaddr2 ip={{ subnet }}.100 cidr_netmask=32 op monitor interval=30s
13f052
+- name: Configure 
13f052
+  shell: docker-enter {{ prefix }}-1 pcs resource defaults resource-stickiness=100
13f052
+- name: Configure 
13f052
+  shell: docker-enter {{ prefix }}-1 pcs resource create WebSite apache configfile=/etc/httpd/conf/httpd.conf statusurl="http://localhost/server-status" op monitor interval=1min
13f052
+- name: Configure 
13f052
+  shell: docker-enter {{ prefix }}-1 pcs constraint colocation add WebSite with ClusterIP INFINITY
13f052
+- name: Configure 
13f052
+  shell: docker-enter {{ prefix }}-1 pcs constraint order ClusterIP then WebSite
13f052
+- name: Configure 
13f052
+  shell: docker-enter {{ prefix }}-1 pcs constraint location WebSite prefers {{ prefix }}-1=50
13f052
+# TODO: Enable fencing
13f052
+# TODO: Make this a full LAMP stack similar to https://github.com/ansible/ansible-examples/tree/master/lamp_simple
13f052
+# TODO: Create a Pacemaker module?
13f052
+
13f052
+#  run_once: true
13f052
+#  delegate_to: web01.example.org
13f052
+
13f052
diff --git a/extra/ansible/docker/roles/docker-host/templates/Dockerfile.j2 b/extra/ansible/docker/roles/docker-host/templates/Dockerfile.j2
13f052
new file mode 100644
13f052
index 0000000..1d57175
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/templates/Dockerfile.j2
13f052
@@ -0,0 +1,16 @@
13f052
+FROM {{ base_image }}
13f052
+ADD /repos /etc/yum.repos.d/
13f052
+#ADD /rpms /root/
13f052
+#RUN yum install -y /root/*.rpm
13f052
+ADD /launch_scripts /root/
13f052
+ADD /bin_files /usr/sbin/
13f052
+
13f052
+RUN mkdir -p /root/.ssh; chmod 700 /root/.ssh
13f052
+ADD authorized_keys /root/.ssh/
13f052
+
13f052
+RUN yum install -y openssh-server net-tools pacemaker pacemaker-cts resource-agents pcs corosync which fence-agents-common sysvinit-tools
13f052
+RUN mkdir -p /etc/pacemaker/
13f052
+RUN echo {{ pacemaker_authkey }} > /etc/pacemaker/authkey
13f052
+RUN /usr/sbin/sshd
13f052
+
13f052
+ENTRYPOINT ["/root/launch.sh"]
13f052
diff --git a/extra/ansible/docker/roles/docker-host/templates/ip_start.j2 b/extra/ansible/docker/roles/docker-host/templates/ip_start.j2
13f052
new file mode 100755
13f052
index 0000000..edbd392
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/roles/docker-host/templates/ip_start.j2
13f052
@@ -0,0 +1,3 @@
13f052
+offset=$(hostname | sed s/.*-//)
13f052
+export OCF_ROOT=/usr/lib/ocf/ OCF_RESKEY_ip={{ subnet }}.${offset} OCF_RESKEY_cidr_netmask=32
13f052
+/usr/lib/ocf/resource.d/heartbeat/IPaddr2 start
13f052
diff --git a/extra/ansible/docker/site.yml b/extra/ansible/docker/site.yml
13f052
new file mode 100644
13f052
index 0000000..0cc65e4
13f052
--- /dev/null
13f052
+++ b/extra/ansible/docker/site.yml
13f052
@@ -0,0 +1,12 @@
13f052
+---
13f052
+# See /etc/ansible/hosts or -i hosts
13f052
+- hosts: controllers
13f052
+  remote_user: root
13f052
+  roles:
13f052
+    - docker-host
13f052
+
13f052
+#- hosts: containers
13f052
+#  gather_facts: no
13f052
+#  remote_user: root
13f052
+#  roles:
13f052
+#    - docker-container
13f052
diff --git a/include/crm/msg_xml.h b/include/crm/msg_xml.h
13f052
index 42f9003..15f1b3c 100644
13f052
--- a/include/crm/msg_xml.h
13f052
+++ b/include/crm/msg_xml.h
13f052
@@ -194,6 +194,7 @@
13f052
 #  define XML_RSC_ATTR_INTERLEAVE	"interleave"
13f052
 #  define XML_RSC_ATTR_INCARNATION	"clone"
13f052
 #  define XML_RSC_ATTR_INCARNATION_MAX	"clone-max"
13f052
+#  define XML_RSC_ATTR_INCARNATION_MIN	"clone-min"
13f052
 #  define XML_RSC_ATTR_INCARNATION_NODEMAX	"clone-node-max"
13f052
 #  define XML_RSC_ATTR_MASTER_MAX	"master-max"
13f052
 #  define XML_RSC_ATTR_MASTER_NODEMAX	"master-node-max"
13f052
diff --git a/include/crm/pengine/status.h b/include/crm/pengine/status.h
13f052
index 4214959..b95b1e5 100644
13f052
--- a/include/crm/pengine/status.h
13f052
+++ b/include/crm/pengine/status.h
13f052
@@ -256,7 +256,6 @@ struct resource_s {
13f052
     int stickiness;
13f052
     int sort_index;
13f052
     int failure_timeout;
13f052
-    int remote_reconnect_interval;
13f052
     int effective_priority;
13f052
     int migration_threshold;
13f052
 
13f052
@@ -295,6 +294,7 @@ struct resource_s {
13f052
 
13f052
     const char *isolation_wrapper;
13f052
     gboolean exclusive_discover;
13f052
+    int remote_reconnect_interval;
13f052
 };
13f052
 
13f052
 struct pe_action_s {
13f052
@@ -324,6 +324,26 @@ struct pe_action_s {
13f052
     GHashTable *meta;
13f052
     GHashTable *extra;
13f052
 
13f052
+    /* 
13f052
+     * These two varables are associated with the constraint logic
13f052
+     * that involves first having one or more actions runnable before
13f052
+     * then allowing this action to execute.
13f052
+     *
13f052
+     * These varables are used with features such as 'clone-min' which
13f052
+     * requires at minimum X number of cloned instances to be running
13f052
+     * before an order dependency can run. Another option that uses
13f052
+     * this is 'require-all=false' in ordering constrants. This option
13f052
+     * says "only required one instance of a resource to start before
13f052
+     * allowing dependencies to start" basicall require-all=false is
13f052
+     * the same as clone-min=1.
13f052
+     */
13f052
+
13f052
+    /* current number of known runnable actions in the before list. */
13f052
+    int runnable_before;
13f052
+    /* the number of "before" runnable actions required for this action
13f052
+     * to be considered runnable */ 
13f052
+    int required_runnable_before;
13f052
+
13f052
     GListPtr actions_before;    /* action_warpper_t* */
13f052
     GListPtr actions_after;     /* action_warpper_t* */
13f052
 };
13f052
diff --git a/lib/cib/Makefile.am b/lib/cib/Makefile.am
13f052
index e84f4f7..1e50511 100644
13f052
--- a/lib/cib/Makefile.am
13f052
+++ b/lib/cib/Makefile.am
13f052
@@ -28,7 +28,7 @@ noinst_HEADERS		=
13f052
 libcib_la_SOURCES	= cib_ops.c cib_utils.c cib_client.c cib_native.c cib_attrs.c
13f052
 libcib_la_SOURCES      += cib_file.c cib_remote.c
13f052
 
13f052
-libcib_la_LDFLAGS	= -version-info 4:1:0 -L$(top_builddir)/lib/pengine/.libs
13f052
+libcib_la_LDFLAGS	= -version-info 4:2:0 -L$(top_builddir)/lib/pengine/.libs
13f052
 libcib_la_LIBADD        = $(CRYPTOLIB) $(top_builddir)/lib/pengine/libpe_rules.la $(top_builddir)/lib/common/libcrmcommon.la
13f052
 libcib_la_CFLAGS	= -I$(top_srcdir)
13f052
 
13f052
diff --git a/lib/cluster/Makefile.am b/lib/cluster/Makefile.am
13f052
index 29413ba..29daeb2 100644
13f052
--- a/lib/cluster/Makefile.am
13f052
+++ b/lib/cluster/Makefile.am
13f052
@@ -28,7 +28,7 @@ header_HEADERS =
13f052
 lib_LTLIBRARIES	= libcrmcluster.la 
13f052
 
13f052
 libcrmcluster_la_SOURCES = election.c cluster.c membership.c
13f052
-libcrmcluster_la_LDFLAGS = -version-info 4:2:0 $(CLUSTERLIBS) 
13f052
+libcrmcluster_la_LDFLAGS = -version-info 5:0:1 $(CLUSTERLIBS) 
13f052
 libcrmcluster_la_LIBADD  = $(top_builddir)/lib/common/libcrmcommon.la $(top_builddir)/lib/fencing/libstonithd.la
13f052
 libcrmcluster_la_DEPENDENCIES = $(top_builddir)/lib/common/libcrmcommon.la $(top_builddir)/lib/fencing/libstonithd.la 
13f052
 
13f052
diff --git a/lib/common/Makefile.am b/lib/common/Makefile.am
13f052
index a593f40..f5c0766 100644
13f052
--- a/lib/common/Makefile.am
13f052
+++ b/lib/common/Makefile.am
13f052
@@ -37,7 +37,7 @@ if BUILD_CIBSECRETS
13f052
 libcrmcommon_la_SOURCES	+= cib_secrets.c
13f052
 endif
13f052
 
13f052
-libcrmcommon_la_LDFLAGS	= -version-info 7:0:4
13f052
+libcrmcommon_la_LDFLAGS	= -version-info 8:0:5
13f052
 libcrmcommon_la_LIBADD  = @LIBADD_DL@ $(GNUTLSLIBS)
13f052
 libcrmcommon_la_SOURCES += $(top_builddir)/lib/gnu/md5.c
13f052
 
13f052
diff --git a/lib/fencing/Makefile.am b/lib/fencing/Makefile.am
13f052
index 2bdcfeb..fbe02e4 100644
13f052
--- a/lib/fencing/Makefile.am
13f052
+++ b/lib/fencing/Makefile.am
13f052
@@ -25,7 +25,7 @@ AM_CPPFLAGS         = -I$(top_builddir)/include  -I$(top_srcdir)/include     \
13f052
 lib_LTLIBRARIES = libstonithd.la
13f052
 
13f052
 libstonithd_la_SOURCES = st_client.c
13f052
-libstonithd_la_LDFLAGS = -version-info 3:2:1
13f052
+libstonithd_la_LDFLAGS = -version-info 3:3:1
13f052
 libstonithd_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la
13f052
 
13f052
 AM_CFLAGS = $(AM_CPPFLAGS)
13f052
diff --git a/lib/lrmd/Makefile.am b/lib/lrmd/Makefile.am
13f052
index f961ae1..820654c 100644
13f052
--- a/lib/lrmd/Makefile.am
13f052
+++ b/lib/lrmd/Makefile.am
13f052
@@ -25,7 +25,7 @@ AM_CPPFLAGS         = -I$(top_builddir)/include  -I$(top_srcdir)/include     \
13f052
 lib_LTLIBRARIES = liblrmd.la
13f052
 
13f052
 liblrmd_la_SOURCES = lrmd_client.c proxy_common.c
13f052
-liblrmd_la_LDFLAGS = -version-info 3:0:2
13f052
+liblrmd_la_LDFLAGS = -version-info 3:1:2
13f052
 liblrmd_la_LIBADD = $(top_builddir)/lib/common/libcrmcommon.la	\
13f052
 			$(top_builddir)/lib/services/libcrmservice.la \
13f052
 			$(top_builddir)/lib/fencing/libstonithd.la
13f052
diff --git a/lib/pengine/Makefile.am b/lib/pengine/Makefile.am
13f052
index 78da075..60d1770 100644
13f052
--- a/lib/pengine/Makefile.am
13f052
+++ b/lib/pengine/Makefile.am
13f052
@@ -26,11 +26,11 @@ lib_LTLIBRARIES	= libpe_rules.la libpe_status.la
13f052
 ## SOURCES
13f052
 noinst_HEADERS	= unpack.h variant.h
13f052
 
13f052
-libpe_rules_la_LDFLAGS	= -version-info 2:4:0
13f052
+libpe_rules_la_LDFLAGS	= -version-info 2:5:0
13f052
 libpe_rules_la_SOURCES	= rules.c common.c
13f052
 libpe_rules_la_LIBADD	= $(top_builddir)/lib/common/libcrmcommon.la
13f052
 
13f052
-libpe_status_la_LDFLAGS	= -version-info 8:0:4
13f052
+libpe_status_la_LDFLAGS	= -version-info 9:0:5
13f052
 libpe_status_la_SOURCES	=  status.c unpack.c utils.c complex.c native.c group.c clone.c rules.c common.c
13f052
 libpe_status_la_LIBADD	=  @CURSESLIBS@ $(top_builddir)/lib/common/libcrmcommon.la
13f052
 
13f052
diff --git a/lib/services/dbus.c b/lib/services/dbus.c
13f052
index 6341fc5..e2efecb 100644
13f052
--- a/lib/services/dbus.c
13f052
+++ b/lib/services/dbus.c
13f052
@@ -64,11 +64,14 @@ pcmk_dbus_find_error(const char *method, DBusPendingCall* pending, DBusMessage *
13f052
     } else {
13f052
         DBusMessageIter args;
13f052
         int dtype = dbus_message_get_type(reply);
13f052
+        char *sig;
13f052
 
13f052
         switch(dtype) {
13f052
             case DBUS_MESSAGE_TYPE_METHOD_RETURN:
13f052
                 dbus_message_iter_init(reply, &args);
13f052
-                crm_trace("Call to %s returned '%s'", method, dbus_message_iter_get_signature(&args));
13f052
+                sig = dbus_message_iter_get_signature(&args);
13f052
+                crm_trace("Call to %s returned '%s'", method, sig);
13f052
+                dbus_free(sig);
13f052
                 break;
13f052
             case DBUS_MESSAGE_TYPE_INVALID:
13f052
                 error.message = "Invalid reply";
13f052
@@ -217,11 +220,14 @@ bool pcmk_dbus_type_check(DBusMessage *msg, DBusMessageIter *field, int expected
13f052
 
13f052
     if(dtype != expected) {
13f052
         DBusMessageIter args;
13f052
+        char *sig;
13f052
 
13f052
         dbus_message_iter_init(msg, &args);
13f052
+        sig = dbus_message_iter_get_signature(&args);
13f052
         do_crm_log_alias(LOG_ERR, __FILE__, function, line,
13f052
-                         "Unexepcted DBus type, expected %c in '%s' instead of %c",
13f052
-                         expected, dbus_message_iter_get_signature(&args), dtype);
13f052
+                         "Unexpected DBus type, expected %c in '%s' instead of %c",
13f052
+                         expected, sig, dtype);
13f052
+        dbus_free(sig);
13f052
         return FALSE;
13f052
     }
13f052
 
13f052
diff --git a/lib/services/services.c b/lib/services/services.c
13f052
index 08bff88..7e2b9f7 100644
13f052
--- a/lib/services/services.c
13f052
+++ b/lib/services/services.c
13f052
@@ -348,6 +348,34 @@ services_action_create_generic(const char *exec, const char *args[])
13f052
     return op;
13f052
 }
13f052
 
13f052
+#if SUPPORT_DBUS
13f052
+/*
13f052
+ * \internal
13f052
+ * \brief Update operation's pending DBus call, unreferencing old one if needed
13f052
+ *
13f052
+ * \param[in,out] op       Operation to modify
13f052
+ * \param[in]     pending  Pending call to set
13f052
+ */
13f052
+void
13f052
+services_set_op_pending(svc_action_t *op, DBusPendingCall *pending)
13f052
+{
13f052
+    if (op->opaque->pending && (op->opaque->pending != pending)) {
13f052
+        if (pending) {
13f052
+            crm_info("Lost pending DBus call (%p)", op->opaque->pending);
13f052
+        } else {
13f052
+            crm_trace("Done with pending DBus call (%p)", op->opaque->pending);
13f052
+        }
13f052
+        dbus_pending_call_unref(op->opaque->pending);
13f052
+    }
13f052
+    op->opaque->pending = pending;
13f052
+    if (pending) {
13f052
+        crm_trace("Updated pending DBus call (%p)", pending);
13f052
+    } else {
13f052
+        crm_trace("Cleared pending DBus call");
13f052
+    }
13f052
+}
13f052
+#endif
13f052
+
13f052
 void
13f052
 services_action_cleanup(svc_action_t * op)
13f052
 {
13f052
diff --git a/lib/services/services_private.h b/lib/services/services_private.h
13f052
index 183afb5..a98cd91 100644
13f052
--- a/lib/services/services_private.h
13f052
+++ b/lib/services/services_private.h
13f052
@@ -63,4 +63,8 @@ void handle_blocked_ops(void);
13f052
 
13f052
 gboolean is_op_blocked(const char *rsc);
13f052
 
13f052
+#if SUPPORT_DBUS
13f052
+void services_set_op_pending(svc_action_t *op, DBusPendingCall *pending);
13f052
+#endif
13f052
+
13f052
 #endif                          /* __MH_SERVICES_PRIVATE_H__ */
13f052
diff --git a/lib/services/systemd.c b/lib/services/systemd.c
13f052
index 749d61c..e1e1bc9 100644
13f052
--- a/lib/services/systemd.c
13f052
+++ b/lib/services/systemd.c
13f052
@@ -461,7 +461,12 @@ systemd_async_dispatch(DBusPendingCall *pending, void *user_data)
13f052
 
13f052
     if(op) {
13f052
         crm_trace("Got result: %p for %p for %s, %s", reply, pending, op->rsc, op->action);
13f052
-        op->opaque->pending = NULL;
13f052
+        if (pending == op->opaque->pending) {
13f052
+            op->opaque->pending = NULL;
13f052
+        } else {
13f052
+            crm_info("Received unexpected reply for pending DBus call (%p vs %p)",
13f052
+                     op->opaque->pending, pending);
13f052
+        }
13f052
         systemd_exec_result(reply, op);
13f052
 
13f052
     } else {
13f052
@@ -499,10 +504,7 @@ systemd_unit_check(const char *name, const char *state, void *userdata)
13f052
     }
13f052
 
13f052
     if (op->synchronous == FALSE) {
13f052
-        if (op->opaque->pending) {
13f052
-            dbus_pending_call_unref(op->opaque->pending);
13f052
-        }
13f052
-        op->opaque->pending = NULL;
13f052
+        services_set_op_pending(op, NULL);
13f052
         operation_finalize(op);
13f052
     }
13f052
 }
13f052
@@ -535,7 +537,7 @@ systemd_unit_exec_with_unit(svc_action_t * op, const char *unit)
13f052
             return op->rc == PCMK_OCF_OK;
13f052
         } else if (pending) {
13f052
             dbus_pending_call_ref(pending);
13f052
-            op->opaque->pending = pending;
13f052
+            services_set_op_pending(op, pending);
13f052
             return TRUE;
13f052
         }
13f052
 
13f052
@@ -617,8 +619,7 @@ systemd_unit_exec_with_unit(svc_action_t * op, const char *unit)
13f052
 
13f052
         dbus_message_unref(msg);
13f052
         if(pending) {
13f052
-            dbus_pending_call_ref(pending);
13f052
-            op->opaque->pending = pending;
13f052
+            services_set_op_pending(op, pending);
13f052
             return TRUE;
13f052
         }
13f052
         return FALSE;
13f052
diff --git a/lib/transition/Makefile.am b/lib/transition/Makefile.am
13f052
index 8ce7775..04d18fe 100644
13f052
--- a/lib/transition/Makefile.am
13f052
+++ b/lib/transition/Makefile.am
13f052
@@ -27,7 +27,7 @@ lib_LTLIBRARIES	= libtransitioner.la
13f052
 noinst_HEADERS		= 
13f052
 libtransitioner_la_SOURCES	= unpack.c graph.c utils.c
13f052
 
13f052
-libtransitioner_la_LDFLAGS	= -version-info 2:3:0
13f052
+libtransitioner_la_LDFLAGS	= -version-info 2:4:0
13f052
 libtransitioner_la_CFLAGS	= -I$(top_builddir)
13f052
 libtransitioner_la_LIBADD       = $(top_builddir)/lib/common/libcrmcommon.la
13f052
 
13f052
diff --git a/pengine/Makefile.am b/pengine/Makefile.am
13f052
index 31532cf..0e12a1f 100644
13f052
--- a/pengine/Makefile.am
13f052
+++ b/pengine/Makefile.am
13f052
@@ -61,7 +61,7 @@ endif
13f052
 noinst_HEADERS	= allocate.h utils.h pengine.h
13f052
 #utils.h pengine.h
13f052
 
13f052
-libpengine_la_LDFLAGS	= -version-info 8:0:4
13f052
+libpengine_la_LDFLAGS	= -version-info 9:0:5
13f052
 # -L$(top_builddir)/lib/pils -lpils -export-dynamic -module -avoid-version
13f052
 libpengine_la_SOURCES	= pengine.c allocate.c utils.c constraints.c
13f052
 libpengine_la_SOURCES  += native.c group.c clone.c master.c graph.c utilization.c
13f052
diff --git a/pengine/allocate.c b/pengine/allocate.c
13f052
index 68cafd4..ec5a18d 100644
13f052
--- a/pengine/allocate.c
13f052
+++ b/pengine/allocate.c
13f052
@@ -1962,7 +1962,6 @@ expand_node_list(GListPtr list)
13f052
             if(node_list) {
13f052
                 existing_len = strlen(node_list);
13f052
             }
13f052
-
13f052
             crm_trace("Adding %s (%dc) at offset %d", node->details->uname, len - 2, existing_len);
13f052
             node_list = realloc_safe(node_list, len + existing_len);
13f052
             sprintf(node_list + existing_len, "%s%s", existing_len == 0 ? "":" ", node->details->uname);
13f052
diff --git a/pengine/allocate.h b/pengine/allocate.h
13f052
index f6602c6..73f750e 100644
13f052
--- a/pengine/allocate.h
13f052
+++ b/pengine/allocate.h
13f052
@@ -171,5 +171,6 @@ extern enum pe_graph_flags clone_update_actions(action_t * first, action_t * the
13f052
                                                 enum pe_action_flags filter, enum pe_ordering type);
13f052
 
13f052
 gboolean update_action_flags(action_t * action, enum pe_action_flags flags);
13f052
+gboolean update_action(action_t * action);
13f052
 
13f052
 #endif
13f052
diff --git a/pengine/clone.c b/pengine/clone.c
13f052
index 3840a0a..ebf53ed 100644
13f052
--- a/pengine/clone.c
13f052
+++ b/pengine/clone.c
13f052
@@ -21,6 +21,7 @@
13f052
 #include <crm/msg_xml.h>
13f052
 #include <allocate.h>
13f052
 #include <utils.h>
13f052
+#include <allocate.h>
13f052
 
13f052
 #define VARIANT_CLONE 1
13f052
 #include <lib/pengine/variant.h>
13f052
@@ -1338,6 +1339,8 @@ clone_update_actions(action_t * first, action_t * then, node_t * node, enum pe_a
13f052
         changed |= native_update_actions(first, then, node, flags, filter, type);
13f052
 
13f052
         for (; gIter != NULL; gIter = gIter->next) {
13f052
+            enum pe_graph_flags child_changed = pe_graph_none;
13f052
+            GListPtr lpc = NULL;
13f052
             resource_t *child = (resource_t *) gIter->data;
13f052
             action_t *child_action = find_first_action(child->actions, NULL, then->task, node);
13f052
 
13f052
@@ -1345,9 +1348,17 @@ clone_update_actions(action_t * first, action_t * then, node_t * node, enum pe_a
13f052
                 enum pe_action_flags child_flags = child->cmds->action_flags(child_action, node);
13f052
 
13f052
                 if (is_set(child_flags, pe_action_runnable)) {
13f052
-                    changed |=
13f052
+                                     
13f052
+                    child_changed |=
13f052
                         child->cmds->update_actions(first, child_action, node, flags, filter, type);
13f052
                 }
13f052
+                changed |= child_changed;
13f052
+                if (child_changed & pe_graph_updated_then) {
13f052
+                   for (lpc = child_action->actions_after; lpc != NULL; lpc = lpc->next) {
13f052
+                        action_wrapper_t *other = (action_wrapper_t *) lpc->data;
13f052
+                        update_action(other->action);
13f052
+                    }
13f052
+                }
13f052
             }
13f052
         }
13f052
     }
13f052
diff --git a/pengine/constraints.c b/pengine/constraints.c
13f052
index 1f44811..7527aa6 100644
13f052
--- a/pengine/constraints.c
13f052
+++ b/pengine/constraints.c
13f052
@@ -256,7 +256,7 @@ unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
13f052
     resource_t *rsc_then = NULL;
13f052
     resource_t *rsc_first = NULL;
13f052
     gboolean invert_bool = TRUE;
13f052
-    gboolean require_all = TRUE;
13f052
+    int min_required_before = 0;
13f052
     enum pe_order_kind kind = pe_order_kind_mandatory;
13f052
     enum pe_ordering cons_weight = pe_order_optional;
13f052
 
13f052
@@ -351,7 +351,15 @@ unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
13f052
         && crm_is_true(require_all_s) == FALSE
13f052
         && rsc_first->variant >= pe_clone) {
13f052
 
13f052
-        require_all = FALSE;
13f052
+        /* require-all=false means only one instance of the clone is required */
13f052
+        min_required_before = 1;
13f052
+    } else if (rsc_first->variant >= pe_clone) {
13f052
+        const char *min_clones_s = g_hash_table_lookup(rsc_first->meta, XML_RSC_ATTR_INCARNATION_MIN);
13f052
+        if (min_clones_s) {
13f052
+            /* if clone min is set, we require at a minimum X number of instances
13f052
+             * to be runnable before allowing dependencies to be runnable. */
13f052
+            min_required_before = crm_parse_int(min_clones_s, "0");
13f052
+        }
13f052
     }
13f052
 
13f052
     cons_weight = pe_order_optional;
13f052
@@ -368,22 +376,31 @@ unpack_simple_rsc_order(xmlNode * xml_obj, pe_working_set_t * data_set)
13f052
         cons_weight |= get_flags(id, kind, action_first, action_then, FALSE);
13f052
     }
13f052
 
13f052
-    if (require_all == FALSE) {
13f052
+    /* If there is a minimum number of instances that must be runnable before
13f052
+     * the 'then' action is runnable, we use a pseudo action as an intermediate step
13f052
+     * start min number of clones -> pseudo action is runnable -> dependency runnable. */
13f052
+    if (min_required_before) {
13f052
         GListPtr rIter = NULL;
13f052
         char *task = crm_concat(CRM_OP_RELAXED_CLONE, id, ':');
13f052
         action_t *unordered_action = get_pseudo_op(task, data_set);
13f052
         free(task);
13f052
 
13f052
+        /* require the pseudo action to have "min_required_before" number of
13f052
+         * actions to be considered runnable before allowing the pseudo action
13f052
+         * to be runnable. */ 
13f052
+        unordered_action->required_runnable_before = min_required_before;
13f052
         update_action_flags(unordered_action, pe_action_requires_any);
13f052
 
13f052
         for (rIter = rsc_first->children; id && rIter; rIter = rIter->next) {
13f052
             resource_t *child = rIter->data;
13f052
-
13f052
+            /* order each clone instance before the pseudo action */
13f052
             custom_action_order(child, generate_op_key(child->id, action_first, 0), NULL,
13f052
                                 NULL, NULL, unordered_action,
13f052
                                 pe_order_one_or_more | pe_order_implies_then_printed, data_set);
13f052
         }
13f052
 
13f052
+        /* order the "then" dependency to occur after the pseudo action only if
13f052
+         * the pseudo action is runnable */ 
13f052
         order_id = custom_action_order(NULL, NULL, unordered_action,
13f052
                        rsc_then, generate_op_key(rsc_then->id, action_then, 0), NULL,
13f052
                        cons_weight | pe_order_runnable_left, data_set);
13f052
diff --git a/pengine/graph.c b/pengine/graph.c
13f052
index 9cfede6..3d832f0 100644
13f052
--- a/pengine/graph.c
13f052
+++ b/pengine/graph.c
13f052
@@ -29,7 +29,6 @@
13f052
 #include <allocate.h>
13f052
 #include <utils.h>
13f052
 
13f052
-gboolean update_action(action_t * action);
13f052
 void update_colo_start_chain(action_t * action);
13f052
 gboolean rsc_update_action(action_t * first, action_t * then, enum pe_ordering type);
13f052
 
13f052
@@ -261,8 +260,16 @@ graph_update_action(action_t * first, action_t * then, node_t * node, enum pe_ac
13f052
                                                 pe_action_runnable, pe_order_one_or_more);
13f052
 
13f052
         } else if (is_set(flags, pe_action_runnable)) {
13f052
-            if (update_action_flags(then, pe_action_runnable)) {
13f052
-                changed |= pe_graph_updated_then;
13f052
+            /* alright. a "first" action is considered runnable, incremente
13f052
+             * the 'runnable_before' counter */
13f052
+            then->runnable_before++;
13f052
+
13f052
+            /* if the runnable before count for then exceeds the required number
13f052
+             * of "before" runnable actions... mark then as runnable */
13f052
+            if (then->runnable_before >= then->required_runnable_before) {
13f052
+                if (update_action_flags(then, pe_action_runnable)) {
13f052
+                    changed |= pe_graph_updated_then;
13f052
+                }
13f052
             }
13f052
         }
13f052
         if (changed) {
13f052
@@ -456,6 +463,18 @@ update_action(action_t * then)
13f052
                      pe_action_pseudo) ? "pseudo" : then->node ? then->node->details->uname : "");
13f052
 
13f052
     if (is_set(then->flags, pe_action_requires_any)) {
13f052
+        /* initialize current known runnable before actions to 0
13f052
+         * from here as graph_update_action is called for each of
13f052
+         * then's before actions, this number will increment as
13f052
+         * runnable 'first' actions are encountered */
13f052
+        then->runnable_before = 0;
13f052
+
13f052
+        /* for backwards compatibility with previous options that use
13f052
+         * the 'requires_any' flag, initalize required to 1 if it is
13f052
+         * not set. */ 
13f052
+        if (then->required_runnable_before == 0) {
13f052
+            then->required_runnable_before = 1;
13f052
+        }
13f052
         clear_bit(then->flags, pe_action_runnable);
13f052
         /* We are relying on the pe_order_one_or_more clause of
13f052
          * graph_update_action(), called as part of the:
13f052
diff --git a/pengine/native.c b/pengine/native.c
13f052
index b93f8da..7d5f602 100644
13f052
--- a/pengine/native.c
13f052
+++ b/pengine/native.c
13f052
@@ -2817,8 +2817,7 @@ native_create_probe(resource_t * rsc, node_t * node, action_t * complete,
13f052
 }
13f052
 
13f052
 static void
13f052
-native_start_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_stonith,
13f052
-                         pe_working_set_t * data_set)
13f052
+native_start_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
13f052
 {
13f052
     node_t *target = stonith_op ? stonith_op->node : NULL;
13f052
 
13f052
@@ -2893,14 +2892,24 @@ find_fence_target_node_actions(GListPtr search_list, const char *key, node_t *fe
13f052
 }
13f052
 
13f052
 static void
13f052
-native_stop_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_stonith,
13f052
-                        pe_working_set_t * data_set)
13f052
+native_stop_constraints(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
13f052
 {
13f052
     char *key = NULL;
13f052
     GListPtr gIter = NULL;
13f052
     GListPtr action_list = NULL;
13f052
+
13f052
+    action_t *start = NULL;
13f052
     resource_t *top = uber_parent(rsc);
13f052
 
13f052
+    key = start_key(rsc);
13f052
+    action_list = find_actions(rsc->actions, key, NULL);
13f052
+    if(action_list) {
13f052
+        start = action_list->data;
13f052
+    }
13f052
+
13f052
+    g_list_free(action_list);
13f052
+    free(key);
13f052
+
13f052
     key = stop_key(rsc);
13f052
     action_list = find_fence_target_node_actions(rsc->actions, key, stonith_op->node, data_set);
13f052
     free(key);
13f052
@@ -2932,7 +2941,7 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_sto
13f052
         update_action_flags(action, pe_action_runnable);
13f052
         update_action_flags(action, pe_action_implied_by_stonith);
13f052
 
13f052
-        {
13f052
+        if(start == NULL || start->needs > rsc_req_quorum) {
13f052
             enum pe_ordering flags = pe_order_optional;
13f052
             action_t *parent_stop = find_first_action(top->actions, NULL, RSC_STOP, NULL);
13f052
 
13f052
@@ -3032,7 +3041,8 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_sto
13f052
             crm_trace("here - 1");
13f052
             update_action_flags(action, pe_action_pseudo);
13f052
             update_action_flags(action, pe_action_runnable);
13f052
-            if (is_stonith == FALSE) {
13f052
+
13f052
+            if (start == NULL || start->needs > rsc_req_quorum) {
13f052
                 order_actions(stonith_op, action, pe_order_preserve|pe_order_optional);
13f052
             }
13f052
         }
13f052
@@ -3044,8 +3054,6 @@ native_stop_constraints(resource_t * rsc, action_t * stonith_op, gboolean is_sto
13f052
 void
13f052
 rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, pe_working_set_t * data_set)
13f052
 {
13f052
-    gboolean is_stonith = FALSE;
13f052
-
13f052
     if (rsc->children) {
13f052
         GListPtr gIter = NULL;
13f052
 
13f052
@@ -3063,11 +3071,11 @@ rsc_stonith_ordering(resource_t * rsc, action_t * stonith_op, pe_working_set_t *
13f052
     }
13f052
 
13f052
     /* Start constraints */
13f052
-    native_start_constraints(rsc, stonith_op, is_stonith, data_set);
13f052
+    native_start_constraints(rsc, stonith_op, data_set);
13f052
 
13f052
     /* Stop constraints */
13f052
     if (stonith_op) {
13f052
-        native_stop_constraints(rsc, stonith_op, is_stonith, data_set);
13f052
+        native_stop_constraints(rsc, stonith_op, data_set);
13f052
     }
13f052
 }
13f052
 
13f052
diff --git a/pengine/regression.sh b/pengine/regression.sh
13f052
index d184798..7f73f92 100755
13f052
--- a/pengine/regression.sh
13f052
+++ b/pengine/regression.sh
13f052
@@ -31,6 +31,20 @@ info Performing the following tests from $io_dir
13f052
 create_mode="false"
13f052
 
13f052
 echo ""
13f052
+do_test cloned_start_one  "order first clone then clone... first clone_min=2"
13f052
+do_test cloned_start_two  "order first clone then clone... first clone_min=2"
13f052
+do_test cloned_stop_one   "order first clone then clone... first clone_min=2"
13f052
+do_test cloned_stop_two   "order first clone then clone... first clone_min=2"
13f052
+do_test clone_min_interleave_start_one "order first clone then clone... first clone_min=2 and then has interleave=true"
13f052
+do_test clone_min_interleave_start_two "order first clone then clone... first clone_min=2 and then has interleave=true"
13f052
+do_test clone_min_interleave_stop_one  "order first clone then clone... first clone_min=2 and then has interleave=true"
13f052
+do_test clone_min_interleave_stop_two  "order first clone then clone... first clone_min=2 and then has interleave=true"
13f052
+do_test clone_min_start_one "order first clone then primitive... first clone_min=2"
13f052
+do_test clone_min_start_two "order first clone then primitive... first clone_min=2"
13f052
+do_test clone_min_stop_all  "order first clone then primitive... first clone_min=2"
13f052
+do_test clone_min_stop_one  "order first clone then primitive... first clone_min=2"
13f052
+do_test clone_min_stop_two  "order first clone then primitive... first clone_min=2"
13f052
+
13f052
 do_test simple1 "Offline     "
13f052
 do_test simple2 "Start       "
13f052
 do_test simple3 "Start 2     "
13f052
diff --git a/pengine/test10/bug-5186-partial-migrate.dot b/pengine/test10/bug-5186-partial-migrate.dot
13f052
index 033d41d..65f5616 100644
13f052
--- a/pengine/test10/bug-5186-partial-migrate.dot
13f052
+++ b/pengine/test10/bug-5186-partial-migrate.dot
13f052
@@ -66,13 +66,10 @@
13f052
 "stonith 'reboot' bl460g1n7" -> "clnDiskd1_stop_0" [ style = bold]
13f052
 "stonith 'reboot' bl460g1n7" -> "clnDiskd2_stop_0" [ style = bold]
13f052
 "stonith 'reboot' bl460g1n7" -> "clnPing_stop_0" [ style = bold]
13f052
-"stonith 'reboot' bl460g1n7" -> "grpStonith8_stop_0" [ style = bold]
13f052
 "stonith 'reboot' bl460g1n7" -> "prmDiskd1_stop_0 bl460g1n7" [ style = bold]
13f052
 "stonith 'reboot' bl460g1n7" -> "prmDiskd2_stop_0 bl460g1n7" [ style = bold]
13f052
 "stonith 'reboot' bl460g1n7" -> "prmDummy_stop_0 bl460g1n7" [ style = bold]
13f052
 "stonith 'reboot' bl460g1n7" -> "prmPing_stop_0 bl460g1n7" [ style = bold]
13f052
-"stonith 'reboot' bl460g1n7" -> "prmStonith8-1_stop_0 bl460g1n7" [ style = bold]
13f052
-"stonith 'reboot' bl460g1n7" -> "prmStonith8-2_stop_0 bl460g1n7" [ style = bold]
13f052
 "stonith 'reboot' bl460g1n7" -> "prmVM2_stop_0 bl460g1n7" [ style = bold]
13f052
 "stonith 'reboot' bl460g1n7" -> "stonith_complete" [ style = bold]
13f052
 "stonith 'reboot' bl460g1n7" [ style=bold color="green" fontcolor="black"]
13f052
diff --git a/pengine/test10/bug-5186-partial-migrate.exp b/pengine/test10/bug-5186-partial-migrate.exp
13f052
index 216d962..bc058ea 100644
13f052
--- a/pengine/test10/bug-5186-partial-migrate.exp
13f052
+++ b/pengine/test10/bug-5186-partial-migrate.exp
13f052
@@ -104,11 +104,7 @@
13f052
         <attributes CRM_meta_record_pending="true" CRM_meta_timeout="20000" />
13f052
       </pseudo_event>
13f052
     </action_set>
13f052
-    <inputs>
13f052
-      <trigger>
13f052
-        <crm_event id="75" operation="stonith" operation_key="stonith-bl460g1n7-reboot" on_node="bl460g1n7" on_node_uuid="3232261593"/>
13f052
-      </trigger>
13f052
-    </inputs>
13f052
+    <inputs/>
13f052
   </synapse>
13f052
   <synapse id="8">
13f052
     <action_set>
13f052
@@ -182,9 +178,6 @@
13f052
       <trigger>
13f052
         <pseudo_event id="45" operation="stop" operation_key="grpStonith8_stop_0"/>
13f052
       </trigger>
13f052
-      <trigger>
13f052
-        <crm_event id="75" operation="stonith" operation_key="stonith-bl460g1n7-reboot" on_node="bl460g1n7" on_node_uuid="3232261593"/>
13f052
-      </trigger>
13f052
     </inputs>
13f052
   </synapse>
13f052
   <synapse id="13">
13f052
@@ -229,9 +222,6 @@
13f052
       <trigger>
13f052
         <pseudo_event id="45" operation="stop" operation_key="grpStonith8_stop_0"/>
13f052
       </trigger>
13f052
-      <trigger>
13f052
-        <crm_event id="75" operation="stonith" operation_key="stonith-bl460g1n7-reboot" on_node="bl460g1n7" on_node_uuid="3232261593"/>
13f052
-      </trigger>
13f052
     </inputs>
13f052
   </synapse>
13f052
   <synapse id="16">
13f052
diff --git a/pengine/test10/bug-5186-partial-migrate.summary b/pengine/test10/bug-5186-partial-migrate.summary
13f052
index f848c97..5e62a23 100644
13f052
--- a/pengine/test10/bug-5186-partial-migrate.summary
13f052
+++ b/pengine/test10/bug-5186-partial-migrate.summary
13f052
@@ -35,18 +35,22 @@ Transition Summary:
13f052
 
13f052
 Executing cluster transition:
13f052
  * Resource action: prmVM2          stop on bl460g1n6
13f052
+ * Pseudo action:   grpStonith8_stop_0
13f052
+ * Pseudo action:   prmStonith8-2_stop_0
13f052
  * Fencing bl460g1n7 (reboot)
13f052
  * Pseudo action:   stonith_complete
13f052
  * Pseudo action:   prmDummy_stop_0
13f052
  * Pseudo action:   prmVM2_stop_0
13f052
- * Pseudo action:   grpStonith8_stop_0
13f052
- * Pseudo action:   prmStonith8-2_stop_0
13f052
+ * Pseudo action:   prmStonith8-1_stop_0
13f052
  * Pseudo action:   clnDiskd1_stop_0
13f052
  * Pseudo action:   clnDiskd2_stop_0
13f052
  * Pseudo action:   clnPing_stop_0
13f052
  * Resource action: prmDummy        start on bl460g1n6
13f052
  * Resource action: prmVM2          start on bl460g1n8
13f052
- * Pseudo action:   prmStonith8-1_stop_0
13f052
+ * Pseudo action:   grpStonith8_stopped_0
13f052
+ * Pseudo action:   grpStonith8_start_0
13f052
+ * Resource action: prmStonith8-1   start on bl460g1n6
13f052
+ * Resource action: prmStonith8-2   start on bl460g1n6
13f052
  * Pseudo action:   prmDiskd1_stop_0
13f052
  * Pseudo action:   clnDiskd1_stopped_0
13f052
  * Pseudo action:   prmDiskd2_stop_0
13f052
@@ -55,10 +59,6 @@ Executing cluster transition:
13f052
  * Pseudo action:   clnPing_stopped_0
13f052
  * Pseudo action:   all_stopped
13f052
  * Resource action: prmVM2          monitor=10000 on bl460g1n8
13f052
- * Pseudo action:   grpStonith8_stopped_0
13f052
- * Pseudo action:   grpStonith8_start_0
13f052
- * Resource action: prmStonith8-1   start on bl460g1n6
13f052
- * Resource action: prmStonith8-2   start on bl460g1n6
13f052
  * Pseudo action:   grpStonith8_running_0
13f052
  * Resource action: prmStonith8-1   monitor=10000 on bl460g1n6
13f052
  * Resource action: prmStonith8-2   monitor=3600000 on bl460g1n6
13f052
diff --git a/pengine/test10/bug-lf-2551.dot b/pengine/test10/bug-lf-2551.dot
13f052
index ed80e15..18bca44 100644
13f052
--- a/pengine/test10/bug-lf-2551.dot
13f052
+++ b/pengine/test10/bug-lf-2551.dot
13f052
@@ -56,7 +56,6 @@ digraph "g" {
13f052
 "stonith 'reboot' hex-9" -> "cmirrord:3_stop_0 hex-9" [ style = bold]
13f052
 "stonith 'reboot' hex-9" -> "dlm:3_stop_0 hex-9" [ style = bold]
13f052
 "stonith 'reboot' hex-9" -> "dummy1_stop_0 hex-9" [ style = bold]
13f052
-"stonith 'reboot' hex-9" -> "fencing-sbd_stop_0 hex-9" [ style = bold]
13f052
 "stonith 'reboot' hex-9" -> "o2cb:3_stop_0 hex-9" [ style = bold]
13f052
 "stonith 'reboot' hex-9" -> "ocfs2-1:3_stop_0 hex-9" [ style = bold]
13f052
 "stonith 'reboot' hex-9" -> "stonith_complete" [ style = bold]
13f052
diff --git a/pengine/test10/bug-lf-2551.exp b/pengine/test10/bug-lf-2551.exp
13f052
index 0af9010..d6266e1 100644
13f052
--- a/pengine/test10/bug-lf-2551.exp
13f052
+++ b/pengine/test10/bug-lf-2551.exp
13f052
@@ -18,11 +18,7 @@
13f052
         <attributes CRM_meta_record_pending="false" CRM_meta_timeout="240000"  sbd_device="/dev/disk/by-id/dm-uuid-part1-mpath-3600508b400105b5a0001500000250000"/>
13f052
       </pseudo_event>
13f052
     </action_set>
13f052
-    <inputs>
13f052
-      <trigger>
13f052
-        <crm_event id="249" operation="stonith" operation_key="stonith-hex-9-reboot" on_node="hex-9" on_node_uuid="hex-9"/>
13f052
-      </trigger>
13f052
-    </inputs>
13f052
+    <inputs/>
13f052
   </synapse>
13f052
   <synapse id="2">
13f052
     <action_set>
13f052
diff --git a/pengine/test10/bug-lf-2551.summary b/pengine/test10/bug-lf-2551.summary
13f052
index f8d861c..158eb73 100644
13f052
--- a/pengine/test10/bug-lf-2551.summary
13f052
+++ b/pengine/test10/bug-lf-2551.summary
13f052
@@ -107,6 +107,7 @@ Transition Summary:
13f052
  * Stop    vm-61	(hex-9)
13f052
 
13f052
 Executing cluster transition:
13f052
+ * Pseudo action:   fencing-sbd_stop_0
13f052
  * Resource action: dummy1          monitor=300000 on hex-8
13f052
  * Resource action: dummy1          monitor=300000 on hex-7
13f052
  * Fencing hex-9 (reboot)
13f052
@@ -114,7 +115,7 @@ Executing cluster transition:
13f052
  * Pseudo action:   load_stopped_hex-8
13f052
  * Pseudo action:   load_stopped_hex-7
13f052
  * Pseudo action:   load_stopped_hex-0
13f052
- * Pseudo action:   fencing-sbd_stop_0
13f052
+ * Resource action: fencing-sbd     start on hex-0
13f052
  * Pseudo action:   dummy1_stop_0
13f052
  * Pseudo action:   vm-03_stop_0
13f052
  * Pseudo action:   vm-06_stop_0
13f052
@@ -133,7 +134,6 @@ Executing cluster transition:
13f052
  * Pseudo action:   vm-57_stop_0
13f052
  * Pseudo action:   vm-61_stop_0
13f052
  * Pseudo action:   load_stopped_hex-9
13f052
- * Resource action: fencing-sbd     start on hex-0
13f052
  * Resource action: dummy1          start on hex-0
13f052
  * Pseudo action:   base-clone_stop_0
13f052
  * Resource action: dummy1          monitor=30000 on hex-0
13f052
diff --git a/pengine/test10/clone_min_interleave_start_one.dot b/pengine/test10/clone_min_interleave_start_one.dot
13f052
new file mode 100644
13f052
index 0000000..15ac9be
13f052
--- /dev/null
13f052
+++ b/pengine/test10/clone_min_interleave_start_one.dot
13f052
@@ -0,0 +1,50 @@
13f052
+ digraph "g" {
13f052
+"FAKE1-clone_running_0" [ style=bold color="green" fontcolor="orange"]
13f052
+"FAKE1-clone_start_0" -> "FAKE1-clone_running_0" [ style = bold]
13f052
+"FAKE1-clone_start_0" -> "FAKE1_start_0 c7auto1" [ style = bold]
13f052
+"FAKE1-clone_start_0" [ style=bold color="green" fontcolor="orange"]
13f052
+"FAKE1_monitor_10000 c7auto1" [ style=bold color="green" fontcolor="black"]
13f052
+"FAKE1_start_0 c7auto1" -> "FAKE1-clone_running_0" [ style = bold]
13f052
+"FAKE1_start_0 c7auto1" -> "FAKE1_monitor_10000 c7auto1" [ style = bold]
13f052
+"FAKE1_start_0 c7auto1" [ style=bold color="green" fontcolor="black"]
13f052
+"FAKE2-clone_running_0" -> "FAKE3-clone_start_0" [ style = dashed]
13f052
+"FAKE2-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
13f052
+"FAKE2-clone_start_0" -> "FAKE2-clone_running_0" [ style = dashed]
13f052
+"FAKE2-clone_start_0" -> "FAKE2:1_start_0 c7auto3" [ style = dashed]
13f052
+"FAKE2-clone_start_0" -> "FAKE2:2_start_0 c7auto1" [ style = dashed]
13f052
+"FAKE2-clone_start_0" -> "FAKE2_start_0 c7auto2" [ style = dashed]
13f052
+"FAKE2-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
13f052
+"FAKE2:1_monitor_10000 c7auto3" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE2:1_start_0 c7auto3" -> "FAKE2-clone_running_0" [ style = dashed]
13f052
+"FAKE2:1_start_0 c7auto3" -> "FAKE2:1_monitor_10000 c7auto3" [ style = dashed]
13f052
+"FAKE2:1_start_0 c7auto3" -> "FAKE3:1_start_0 c7auto3" [ style = dashed]
13f052
+"FAKE2:1_start_0 c7auto3" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE2:2_monitor_10000 c7auto1" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE2:2_start_0 c7auto1" -> "FAKE2-clone_running_0" [ style = dashed]
13f052
+"FAKE2:2_start_0 c7auto1" -> "FAKE2:2_monitor_10000 c7auto1" [ style = dashed]
13f052
+"FAKE2:2_start_0 c7auto1" -> "FAKE3:2_start_0 c7auto1" [ style = dashed]
13f052
+"FAKE2:2_start_0 c7auto1" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE2_monitor_10000 c7auto2" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE2_start_0 c7auto2" -> "FAKE2-clone_running_0" [ style = dashed]
13f052
+"FAKE2_start_0 c7auto2" -> "FAKE2_monitor_10000 c7auto2" [ style = dashed]
13f052
+"FAKE2_start_0 c7auto2" -> "FAKE3_start_0 c7auto2" [ style = dashed]
13f052
+"FAKE2_start_0 c7auto2" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE3-clone_running_0" [ style=dashed color="red" fontcolor="orange"]
13f052
+"FAKE3-clone_start_0" -> "FAKE3-clone_running_0" [ style = dashed]
13f052
+"FAKE3-clone_start_0" -> "FAKE3:1_start_0 c7auto3" [ style = dashed]
13f052
+"FAKE3-clone_start_0" -> "FAKE3:2_start_0 c7auto1" [ style = dashed]
13f052
+"FAKE3-clone_start_0" -> "FAKE3_start_0 c7auto2" [ style = dashed]
13f052
+"FAKE3-clone_start_0" [ style=dashed color="red" fontcolor="orange"]
13f052
+"FAKE3:1_monitor_10000 c7auto3" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE3:1_start_0 c7auto3" -> "FAKE3-clone_running_0" [ style = dashed]
13f052
+"FAKE3:1_start_0 c7auto3" -> "FAKE3:1_monitor_10000 c7auto3" [ style = dashed]
13f052
+"FAKE3:1_start_0 c7auto3" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE3:2_monitor_10000 c7auto1" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE3:2_start_0 c7auto1" -> "FAKE3-clone_running_0" [ style = dashed]
13f052
+"FAKE3:2_start_0 c7auto1" -> "FAKE3:2_monitor_10000 c7auto1" [ style = dashed]
13f052
+"FAKE3:2_start_0 c7auto1" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE3_monitor_10000 c7auto2" [ style=dashed color="red" fontcolor="black"]
13f052
+"FAKE3_start_0 c7auto2" -> "FAKE3-clone_running_0" [ style = dashed]
13f052
+"FAKE3_start_0 c7auto2" -> "FAKE3_monitor_10000 c7auto2" [ style = dashed]
13f052
+"FAKE3_start_0 c7auto2" [ style=dashed color="red" fontcolor="black"]
13f052
+}
13f052
diff --git a/pengine/test10/clone_min_interleave_start_one.exp b/pengine/test10/clone_min_interleave_start_one.exp
13f052
new file mode 100644
13f052
index 0000000..b6e0c5d
13f052
--- /dev/null
13f052
+++ b/pengine/test10/clone_min_interleave_start_one.exp
13f052
@@ -0,0 +1,51 @@
13f052
+<transition_graph cluster-delay="60s" stonith-timeout="60s" failed-stop-offset="INFINITY" failed-start-offset="INFINITY"  transition_id="0">
13f052
+  <synapse id="0">
13f052
+    <action_set>
13f052
+      <rsc_op id="11" operation="monitor" operation_key="FAKE1_monitor_10000" internal_operation_key="FAKE1:0_monitor_10000" on_node="c7auto1" on_node_uuid="1">
13f052
+        <primitive id="FAKE1" long-id="FAKE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
13f052
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_interval="10000" CRM_meta_name="monitor" CRM_meta_notify="false" CRM_meta_timeout="20000" />
13f052
+      </rsc_op>
13f052
+    </action_set>
13f052
+    <inputs>
13f052
+      <trigger>
13f052
+        <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto1" on_node_uuid="1"/>
13f052
+      </trigger>
13f052
+    </inputs>
13f052
+  </synapse>
13f052
+  <synapse id="1">
13f052
+    <action_set>
13f052
+      <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto1" on_node_uuid="1">
13f052
+        <primitive id="FAKE1" long-id="FAKE1:0" class="ocf" provider="heartbeat" type="Dummy"/>
13f052
+        <attributes CRM_meta_clone="0" CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_name="start" CRM_meta_notify="false" CRM_meta_timeout="20000" />
13f052
+      </rsc_op>
13f052
+    </action_set>
13f052
+    <inputs>
13f052
+      <trigger>
13f052
+        <pseudo_event id="12" operation="start" operation_key="FAKE1-clone_start_0"/>
13f052
+      </trigger>
13f052
+    </inputs>
13f052
+  </synapse>
13f052
+  <synapse id="2" priority="1000000">
13f052
+    <action_set>
13f052
+      <pseudo_event id="13" operation="running" operation_key="FAKE1-clone_running_0">
13f052
+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
13f052
+      </pseudo_event>
13f052
+    </action_set>
13f052
+    <inputs>
13f052
+      <trigger>
13f052
+        <rsc_op id="10" operation="start" operation_key="FAKE1_start_0" internal_operation_key="FAKE1:0_start_0" on_node="c7auto1" on_node_uuid="1"/>
13f052
+      </trigger>
13f052
+      <trigger>
13f052
+        <pseudo_event id="12" operation="start" operation_key="FAKE1-clone_start_0"/>
13f052
+      </trigger>
13f052
+    </inputs>
13f052
+  </synapse>
13f052
+  <synapse id="3">
13f052
+    <action_set>
13f052
+      <pseudo_event id="12" operation="start" operation_key="FAKE1-clone_start_0">
13f052
+        <attributes CRM_meta_clone_max="3" CRM_meta_clone_node_max="1" CRM_meta_globally_unique="false" CRM_meta_notify="false" CRM_meta_timeout="20000" />
13f052
+      </pseudo_event>
13f052
+    </action_set>
13f052
+    <inputs/>
13f052
+  </synapse>
13f052
+</transition_graph>
13f052
diff --git a/pengine/test10/clone_min_interleave_start_one.scores b/pengine/test10/clone_min_interleave_start_one.scores
13f052
new file mode 100644
13f052
index 0000000..03de018
13f052
--- /dev/null
13f052
+++ b/pengine/test10/clone_min_interleave_start_one.scores
13f052
@@ -0,0 +1,67 @@
13f052
+Allocation scores:
13f052
+clone_color: FAKE1-clone allocation score on c7auto1: 0
13f052
+clone_color: FAKE1-clone allocation score on c7auto2: -INFINITY
13f052
+clone_color: FAKE1-clone allocation score on c7auto3: -INFINITY
13f052
+clone_color: FAKE1:0 allocation score on c7auto1: 0
13f052
+clone_color: FAKE1:0 allocation score on c7auto2: -INFINITY
13f052
+clone_color: FAKE1:0 allocation score on c7auto3: -INFINITY
13f052
+clone_color: FAKE1:1 allocation score on c7auto1: 0
13f052
+clone_color: FAKE1:1 allocation score on c7auto2: -INFINITY
13f052
+clone_color: FAKE1:1 allocation score on c7auto3: -INFINITY
13f052
+clone_color: FAKE1:2 allocation score on c7auto1: 0
13f052
+clone_color: FAKE1:2 allocation score on c7auto2: -INFINITY
13f052
+clone_color: FAKE1:2 allocation score on c7auto3: -INFINITY
13f052
+clone_color: FAKE2-clone allocation score on c7auto1: 0
13f052
+clone_color: FAKE2-clone allocation score on c7auto2: 0
13f052
+clone_color: FAKE2-clone allocation score on c7auto3: 0
13f052
+clone_color: FAKE2:0 allocation score on c7auto1: 0
13f052
+clone_color: FAKE2:0 allocation score on c7auto2: 0
13f052
+clone_color: FAKE2:0 allocation score on c7auto3: 0
13f052
+clone_color: FAKE2:1 allocation score on c7auto1: 0
13f052
+clone_color: FAKE2:1 allocation score on c7auto2: 0
13f052
+clone_color: FAKE2:1 allocation score on c7auto3: 0
13f052
+clone_color: FAKE2:2 allocation score on c7auto1: 0
13f052
+clone_color: FAKE2:2 allocation score on c7auto2: 0
13f052
+clone_color: FAKE2:2 allocation score on c7auto3: 0
13f052
+clone_color: FAKE3-clone allocation score on c7auto1: 0
13f052
+clone_color: FAKE3-clone allocation score on c7auto2: 0
13f052
+clone_color: FAKE3-clone allocation score on c7auto3: 0
13f052
+clone_color: FAKE3:0 allocation score on c7auto1: 0
13f052
+clone_color: FAKE3:0 allocation score on c7auto2: 0
13f052
+clone_color: FAKE3:0 allocation score on c7auto3: 0
13f052
+clone_color: FAKE3:1 allocation score on c7auto1: 0
13f052
+clone_color: FAKE3:1 allocation score on c7auto2: 0
13f052
+clone_color: FAKE3:1 allocation score on c7auto3: 0
13f052
+clone_color: FAKE3:2 allocation score on c7auto1: 0
13f052
+clone_color: FAKE3:2 allocation score on c7auto2: 0
13f052
+clone_color: FAKE3:2 allocation score on c7auto3: 0
13f052
+native_color: FAKE1:0 allocation score on c7auto1: 0
13f052
+native_color: FAKE1:0 allocation score on c7auto2: -INFINITY
13f052
+native_color: FAKE1:0 allocation score on c7auto3: -INFINITY
13f052
+native_color: FAKE1:1 allocation score on c7auto1: -INFINITY
13f052
+native_color: FAKE1:1 allocation score on c7auto2: -INFINITY
13f052
+native_color: FAKE1:1 allocation score on c7auto3: -INFINITY
13f052
+native_color: FAKE1:2 allocation score on c7auto1: -INFINITY
13f052
+native_color: FAKE1:2 allocation score on c7auto2: -INFINITY
13f052
+native_color: FAKE1:2 allocation score on c7auto3: -INFINITY
13f052
+native_color: FAKE2:0 allocation score on c7auto1: 0
13f052
+native_color: FAKE2:0 allocation score on c7auto2: 0
13f052
+native_color: FAKE2:0 allocation score on c7auto3: 0
13f052
+native_color: FAKE2:1 allocation score on c7auto1: 0
13f052
+native_color: FAKE2:1 allocation score on c7auto2: -INFINITY
13f052
+native_color: FAKE2:1 allocation score on c7auto3: 0
13f052
+native_color: FAKE2:2 allocation score on c7auto1: 0
13f052
+native_color: FAKE2:2 allocation score on c7auto2: -INFINITY
13f052
+native_color: FAKE2:2 allocation score on c7auto3: -INFINITY
13f052
+native_color: FAKE3:0 allocation score on c7auto1: 0
13f052
+native_color: FAKE3:0 allocation score on c7auto2: 0
13f052
+native_color: FAKE3:0 allocation score on c7auto3: 0
13f052
+native_color: FAKE3:1 allocation score on c7auto1: 0
13f052
+native_color: FAKE3:1 allocation score on c7auto2: -INFINITY
13f052
+native_color: FAKE3:1 allocation score on c7auto3: 0
13f052
+native_color: FAKE3:2 allocation score on c7auto1: 0
13f052
+native_color: FAKE3:2 allocation score on c7auto2: -INFINITY
13f052
+native_color: FAKE3:2 allocation score on c7auto3: -INFINITY
13f052
+native_color: shooter allocation score on c7auto1: 0
13f052
+native_color: shooter allocation score on c7auto2: 0
13f052
+native_color: shooter allocation score on c7auto3: 0
13f052
diff --git a/pengine/test10/clone_min_interleave_start_one.summary b/pengine/test10/clone_min_interleave_start_one.summary
13f052
new file mode 100644
13f052
index 0000000..b15f68a
13f052
--- /dev/null
13f052
+++ b/pengine/test10/clone_min_interleave_start_one.summary
13f052
@@ -0,0 +1,39 @@
13f052
+
13f052
+Current cluster status:
13f052
+Online: [ c7auto1 c7auto2 c7auto3 ]
13f052
+
13f052
+ shooter	(stonith:fence_phd_kvm):	Started c7auto1
13f052
+ Clone Set: FAKE1-clone [FAKE1]
13f052
+     Stopped: [ c7auto1 c7auto2 c7auto3 ]
13f052
+ Clone Set: FAKE2-clone [FAKE2]
13f052
+     Stopped: [ c7auto1 c7auto2 c7auto3 ]
13f052
+ Clone Set: FAKE3-clone [FAKE3]
13f052
+     Stopped: [ c7auto1 c7auto2 c7auto3 ]
13f052
+
13f052
+Transition Summary:
13f052
+ * Start   FAKE1:0	(c7auto1)
13f052
+ * Start   FAKE2:0	(c7auto2 - blocked)
13f052
+ * Start   FAKE2:1	(c7auto3 - blocked)
13f052
+ * Start   FAKE2:2	(c7auto1 - blocked)
13f052
+ * Start   FAKE3:0	(c7auto2 - blocked)
13f052
+ * Start   FAKE3:1	(c7auto3 - blocked)
13f052
+ * Start   FAKE3:2	(c7auto1 - blocked)
13f052
+
13f052
+Executing cluster transition:
13f052
+ * Pseudo action:   FAKE1-clone_start_0
13f052
+ * Resource action: FAKE1           start on c7auto1
13f052
+ * Pseudo action:   FAKE1-clone_running_0
13f052
+ * Resource action: FAKE1           monitor=10000 on c7auto1
13f052
+
13f052
+Revised cluster status:
13f052
+Online: [ c7auto1 c7auto2 c7auto3 ]
13f052
+
13f052
+ shooter	(stonith:fence_phd_kvm):	Started c7auto1
13f052
+ Clone Set: FAKE1-clone [FAKE1]
13f052
+     Started: [ c7auto1 ]
13f052
+     Stopped: [ c7auto2 c7auto3 ]
13f052
+ Clone Set: FAKE2-clone [FAKE2]
13f052
+     Stopped: [ c7auto1 c7auto2 c7auto3 ]
13f052
+ Clone Set: FAKE3-clone [FAKE3]
13f052
+     Stopped: [ c7auto1 c7auto2 c7auto3 ]
13f052
+
13f052
diff --git a/pengine/test10/clone_min_interleave_start_one.xml b/pengine/test10/clone_min_interleave_start_one.xml
13f052
new file mode 100644
13f052
index 0000000..fbe99de
13f052
--- /dev/null
13f052
+++ b/pengine/test10/clone_min_interleave_start_one.xml
13f052
@@ -0,0 +1,155 @@
13f052
+<cib crm_feature_set="3.0.10" validate-with="pacemaker-2.3" epoch="15" num_updates="0" admin_epoch="0" cib-last-written="Fri Jul 17 11:44:12 2015" update-origin="c7auto1" update-client="crm_resource" update-user="root" have-quorum="1" dc-uuid="3">
13f052
+  <configuration>
13f052
+    <crm_config>
13f052
+      <cluster_property_set id="cib-bootstrap-options">
13f052
+        <nvpair id="cib-bootstrap-options-have-watchdog" name="have-watchdog" value="false"/>
13f052
+        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.12-ef86b49"/>
13f052
+        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="corosync"/>
13f052
+        <nvpair id="cib-bootstrap-options-cluster-name" name="cluster-name" value="phd"/>
13f052
+      </cluster_property_set>
13f052
+    </crm_config>
13f052
+    <nodes>
13f052
+      <node id="1" uname="c7auto1"/>
13f052
+      <node id="2" uname="c7auto2"/>
13f052
+      <node id="3" uname="c7auto3"/>
13f052
+    </nodes>
13f052
+    <resources>
13f052
+      <primitive class="stonith" id="shooter" type="fence_phd_kvm">
13f052
+        <instance_attributes id="shooter-instance_attributes"/>
13f052
+        <operations>
13f052
+          <op id="shooter-monitor-interval-60s" interval="60s" name="monitor"/>
13f052
+        </operations>
13f052
+      </primitive>
13f052
+      <clone id="FAKE1-clone">
13f052
+        <primitive class="ocf" id="FAKE1" provider="heartbeat" type="Dummy">
13f052
+          <instance_attributes id="FAKE1-instance_attributes"/>
13f052
+          <operations>
13f052
+            <op id="FAKE1-start-timeout-20" interval="0s" name="start" timeout="20"/>
13f052
+            <op id="FAKE1-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
13f052
+            <op id="FAKE1-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
13f052
+          </operations>
13f052
+        </primitive>
13f052
+        <meta_attributes id="FAKE1-clone-meta">
13f052
+          <nvpair id="FAKE1-clone-min" name="clone-min" value="2"/>
13f052
+        </meta_attributes>
13f052
+        <meta_attributes id="FAKE1-clone-meta_attributes"/>
13f052
+      </clone>
13f052
+      <clone id="FAKE2-clone">
13f052
+        <primitive class="ocf" id="FAKE2" provider="heartbeat" type="Dummy">
13f052
+          <instance_attributes id="FAKE2-instance_attributes"/>
13f052
+          <operations>
13f052
+            <op id="FAKE2-start-timeout-20" interval="0s" name="start" timeout="20"/>
13f052
+            <op id="FAKE2-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
13f052
+            <op id="FAKE2-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
13f052
+          </operations>
13f052
+        </primitive>
13f052
+        <meta_attributes id="FAKE2-clone-meta">
13f052
+          <nvpair id="FAKE2-interleave" name="interleave" value="true"/>
13f052
+        </meta_attributes>
13f052
+      </clone>
13f052
+      <clone id="FAKE3-clone">
13f052
+        <primitive class="ocf" id="FAKE3" provider="heartbeat" type="Dummy">
13f052
+          <instance_attributes id="FAKE3-instance_attributes"/>
13f052
+          <operations>
13f052
+            <op id="FAKE3-start-timeout-20" interval="0s" name="start" timeout="20"/>
13f052
+            <op id="FAKE3-stop-timeout-20" interval="0s" name="stop" timeout="20"/>
13f052
+            <op id="FAKE3-monitor-interval-10" interval="10" name="monitor" timeout="20"/>
13f052
+          </operations>
13f052
+        </primitive>
13f052
+        <meta_attributes id="FAKE3-clone-meta">
13f052
+          <nvpair id="FAKE3-interleave" name="interleave" value="true"/>
13f052
+        </meta_attributes>
13f052
+      </clone>
13f052
+    </resources>
13f052
+    <constraints>
13f052
+      <rsc_order first="FAKE1-clone" first-action="start" id="order-FAKE1-clone-FAKE2-clone-mandatory" then="FAKE2-clone" then-action="start"/>
13f052
+      <rsc_order first="FAKE2-clone" first-action="start" id="order-FAKE2-clone-FAKE3-clone-mandatory" then="FAKE3-clone" then-action="start"/>
13f052
+      <rsc_location id="cli-ban-FAKE1-clone-on-c7auto3" rsc="FAKE1-clone" role="Started" node="c7auto3" score="-INFINITY"/>
13f052
+      <rsc_location id="cli-ban-FAKE1-clone-on-c7auto2" rsc="FAKE1-clone" role="Started" node="c7auto2" score="-INFINITY"/>
13f052
+    </constraints>
13f052
+  </configuration>
13f052
+  <status>
13f052
+    <node_state id="3" uname="c7auto3" in_ccm="true" crmd="online" crm-debug-origin="do_update_resource" join="member" expected="member">
13f052
+      <lrm id="3">
13f052
+        <lrm_resources>
13f052
+          <lrm_resource id="shooter" type="fence_phd_kvm" class="stonith">
13f052
+            <lrm_rsc_op id="shooter_last_0" operation_key="shooter_monitor_0" operation="monitor" crm-debug-origin="do_update_resource" crm_feature_set="3.0.10" transition-key="8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" transition-magic="0:7;8:4:7:9d56c099-7067-4908-9f99-89a103eff57b" on_node="c7auto3" call-id="5" rc-code="7" op-status="0" interval="0" last-run="1437146780" last-rc-change="1437146780" exec-time="998" queue-time="0" op-digest="f2317cad3d54cec5d7d7aa7d0bf35cf8"/>
13f052
+          </lrm_resource>
13f052
+          <lrm_resource id="FAKE1" type="Dummy" class="ocf" provider="heartbeat">
13f052
+            <lrm_rsc_op id="FAKE1_last_0" operation_key="FAKE1_stop_0" operation="stop" crm-debug-origin="do_update_resource" crm