commit c8136d5f0abe785223f4ea809073d6566aa36b58

Author: David Vossel <dvossel@redhat.com>

Date:   Wed Sep 25 18:14:13 2013 -0400

    RemoteLXC port

diff --git a/configure.ac b/configure.ac

index 6c80891..cfc1b1f 100644

--- a/configure.ac

+++ b/configure.ac

@@ -1084,6 +1084,10 @@ CRM_CONFIG_DIR="${localstatedir}/lib/pacemaker/cib"

 AC_DEFINE_UNQUOTED(CRM_CONFIG_DIR,"$CRM_CONFIG_DIR", Where to keep configuration files)

 AC_SUBST(CRM_CONFIG_DIR)

+CRM_CONFIG_CTS="${localstatedir}/lib/pacemaker/cts"

+AC_DEFINE_UNQUOTED(CRM_CONFIG_CTS,"$CRM_CONFIG_CTS", Where to keep cts stateful data)

+AC_SUBST(CRM_CONFIG_CTS)

+

 CRM_LEGACY_CONFIG_DIR="${localstatedir}/lib/heartbeat/crm"

 AC_DEFINE_UNQUOTED(CRM_LEGACY_CONFIG_DIR,"$CRM_LEGACY_CONFIG_DIR", Where Pacemaker used to keep configuration files)

 AC_SUBST(CRM_LEGACY_CONFIG_DIR)

@@ -1790,7 +1794,8 @@ cts/Makefile					        	\

 	cts/CTSvars.py						\

 	cts/LSBDummy						\

 	cts/benchmark/Makefile					\

-		cts/benchmark/clubench				\

+	cts/benchmark/clubench				\

+	cts/lxc_autogen.sh						\

 cib/Makefile							\

 crmd/Makefile							\

 pengine/Makefile						\

diff --git a/cts/CTSlab.py b/cts/CTSlab.py

index 58506c3..f7e183c 100755

--- a/cts/CTSlab.py

+++ b/cts/CTSlab.py

@@ -95,6 +95,7 @@ class LabEnvironment(CtsLab):

         #self["valgrind-opts"] = """--trace-children=no --num-callers=25 --gen-suppressions=all --suppressions="""+CTSvars.CTS_home+"""/cts.supp"""

         self["experimental-tests"] = 0

+        self["container-tests"] = 0

         self["valgrind-tests"] = 0

         self["unsafe-tests"] = 1

         self["loop-tests"] = 1

@@ -148,6 +149,7 @@ def usage(arg, status=1):

     print "\t [--no-unsafe-tests]          dont run tests that are unsafe for use with ocfs2/drbd"

     print "\t [--valgrind-tests]           include tests using valgrind"

     print "\t [--experimental-tests]       include experimental tests"

+    print "\t [--container-tests]          include pacemaker_remote tests that run in lxc container resources"

     print "\t [--oprofile 'node list']     list of cluster nodes to run oprofile on]"

     print "\t [--qarsh]                    use the QARSH backdoor to access nodes instead of SSH"

     print "\t [--seed random_seed]"

@@ -427,6 +429,9 @@ if __name__ == '__main__':

        elif args[i] == "--experimental-tests":

            Environment["experimental-tests"] = 1

+       elif args[i] == "--container-tests":

+           Environment["container-tests"] = 1

+

        elif args[i] == "--set":

            skipthis=1

            (name, value) = args[i+1].split('=')

diff --git a/cts/CTStests.py b/cts/CTStests.py

index b5dd69a..19f6ef4 100644

--- a/cts/CTStests.py

+++ b/cts/CTStests.py

@@ -73,6 +73,7 @@ class CTSTest:

         self.is_loop = 0

         self.is_unsafe = 0

         self.is_experimental = 0

+        self.is_container = 0

         self.is_valgrind = 0

         self.benchmark = 0  # which tests to benchmark

         self.timer = {}  # timers

@@ -205,6 +206,8 @@ class CTSTest:

             return 0

         elif self.is_experimental and not self.CM.Env["experimental-tests"]:

             return 0

+        elif self.is_container and not self.CM.Env["container-tests"]:

+            return 0

         elif self.CM.Env["benchmark"] and self.benchmark == 0:

             return 0

@@ -2473,4 +2476,118 @@ def TestList(cm, audits):

             result.append(bound_test)

     return result

+###################################################################

+class RemoteLXC(CTSTest):

+###################################################################

+    def __init__(self, cm):

+        CTSTest.__init__(self,cm)

+        self.name="RemoteLXC"

+        self.start = StartTest(cm)

+        self.startall = SimulStartLite(cm)

+        self.num_containers = 2

+        self.is_container = 1

+        self.failed = 0

+        self.fail_string = ""

+

+    def start_lxc_simple(self, node):

+

+        rc = self.CM.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -v &>/dev/null")

+        if rc == 1:

+            return self.skipped()

+

+        # restore any artifacts laying around from a previous test.

+        self.CM.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -R &>/dev/null")

+

+        # generate the containers, put them in the config, add some resources to them

+        pats = [ ]

+        watch = self.create_watch(pats, 120)

+        watch.setwatch()

+        pats.append("process_lrm_event: LRM operation lxc1_start_0.*confirmed.*ok")

+        pats.append("process_lrm_event: LRM operation lxc2_start_0.*confirmed.*ok")

+        pats.append("process_lrm_event: LRM operation lxc-ms_start_0.*confirmed.*ok")

+        pats.append("process_lrm_event: LRM operation lxc-ms_promote_0.*confirmed.*ok")

+

+        self.CM.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -g -a -m -s -c %d &>/dev/null" % self.num_containers)

+        self.set_timer("remoteSimpleInit")

+        watch.lookforall()

+        self.log_timer("remoteSimpleInit")

+        if watch.unmatched:

+            self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))

+            self.failed = 1

+

+    def cleanup_lxc_simple(self, node):

+

+        pats = [ ]

+        # if the test failed, attempt to clean up the cib and libvirt environment

+        # as best as possible 

+        if self.failed == 1:

+            # restore libvirt and cib

+            self.CM.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -R &>/dev/null")

+            self.CM.rsh(node, "crm_resource -C -r container1 &>/dev/null")

+            self.CM.rsh(node, "crm_resource -C -r container2 &>/dev/null")

+            self.CM.rsh(node, "crm_resource -C -r lxc1 &>/dev/null")

+            self.CM.rsh(node, "crm_resource -C -r lxc2 &>/dev/null")

+            self.CM.rsh(node, "crm_resource -C -r lxc-ms &>/dev/null")

+            time.sleep(20)

+            return

+

+        watch = self.create_watch(pats, 120)

+        watch.setwatch()

+

+        pats.append("process_lrm_event: LRM operation container1_stop_0.*confirmed.*ok")

+        pats.append("process_lrm_event: LRM operation container2_stop_0.*confirmed.*ok")

+

+        self.CM.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -p &>/dev/null")

+        self.set_timer("remoteSimpleCleanup")

+        watch.lookforall()

+        self.log_timer("remoteSimpleCleanup")

+

+        if watch.unmatched:

+            self.fail_string = "Unmatched patterns: %s" % (repr(watch.unmatched))

+            self.failed = 1

+

+        # cleanup libvirt

+        self.CM.rsh(node, "/usr/share/pacemaker/tests/cts/lxc_autogen.sh -R &>/dev/null")

+

+    def __call__(self, node):

+        '''Perform the 'RemoteLXC' test. '''

+        self.incr("calls")

+

+        ret = self.startall(None)

+        if not ret:

+            return self.failure("Setup failed, start all nodes failed.")

+

+        self.start_lxc_simple(node)

+        self.cleanup_lxc_simple(node)

+

+        self.CM.debug("Waiting for the cluster to recover")

+        self.CM.cluster_stable()

+

+        if self.failed == 1:

+            return self.failure(self.fail_string)

+

+        return self.success()

+

+    def errorstoignore(self):

+        '''Return list of errors which should be ignored'''

+        return [ """Updating failcount for ping""",

+                 """LogActions: Recover ping""",

+                 """LogActions: Recover lxc-ms""",

+                 """LogActions: Recover container""",

+                 # The orphaned lxc-ms resource causes an expected transition error

+                 # that is a result of the pengine not having knowledge that the 

+                 # ms resource used to be a clone.  As a result it looks like that 

+                 # resource is running in multiple locations when it shouldn't... But in

+                 # this instance we know why this error is occurring and that it is expected.

+                 """Calculated Transition .* /var/lib/pacemaker/pengine/pe-error""",

+                 """Resource lxc-ms .* is active on 2 nodes attempting recovery""",

+                 """Unknown operation: fail""",

+                 """notice: operation_finished: ping-""",

+                 """notice: operation_finished: container""",

+                 """notice: operation_finished: .*_monitor_0:.*:stderr""",

+                 """(ERROR|error): sending stonithRA op to stonithd failed.""",

+                ]

+

+AllTestClasses.append(RemoteLXC)

+

 # vim:ts=4:sw=4:et:

diff --git a/cts/Makefile.am b/cts/Makefile.am

index e01ac10..cb86db8 100644

--- a/cts/Makefile.am

+++ b/cts/Makefile.am

@@ -40,6 +40,7 @@ cts_DATA	=	README cts.supp

 cts_SCRIPTS	=	cluster_test		\

 			CTSlab.py		\

+			lxc_autogen.sh	\

 			LSBDummy		\

 			$(top_srcdir)/fencing/fence_dummy

diff --git a/cts/lxc_autogen.sh.in b/cts/lxc_autogen.sh.in

new file mode 100644

index 0000000..1d334c4

--- /dev/null

+++ b/cts/lxc_autogen.sh.in

@@ -0,0 +1,362 @@

+#!/bin/bash

+

+containers="2"

+download=0

+share_configs=0

+# different than default libvirt network in case this is run nested in a KVM instance

+addr="192.168.123.1"

+restore=0

+restore_pcmk=0

+restore_all=0

+generate=0

+key_gen=0

+cib=0

+add_master=0

+verify=0

+working_dir="@CRM_CONFIG_CTS@/lxc"

+curdir=$(pwd)

+

+function helptext() {

+	echo "lxc_autogen.sh - A tool for generating libvirt lxc containers for testing purposes."

+	echo ""

+	echo "Usage: lxc-autogen [options]"

+	echo ""

+	echo "Options:"

+	echo "-g, --generate         Generate libvirt lxc environment in the directory this script is run from."

+	echo "-k, --key-gen          Generate local pacemaker remote key only."

+	echo "-r, --restore-libvirt  Restore the default network, and libvirt config to before this script ran."

+	echo "-p, --restore-cib      Remove cib entries this script generated."

+	echo "-R, --restore-all      Restore both libvirt and cib plus clean working directory. This will leave libvirt xml files though so rsc can be stopped properly."

+	echo ""

+	echo "-a, --add-cib          Add remote-node entries for each lxc instance into the cib"

+	echo "-m, --add-master       Add master resource shared between remote-nodes"

+	echo "-d, --download-agent   Download and install the latest VirtualDomain agent."

+	echo "-s, --share-configs    Copy container configs to all other known cluster nodes, (crm_node -l)"

+	echo "-c, --containers       Specify the number of containers to generate, defaults to $containers. Used with -g"

+	echo "-n, --network          What network to override default libvirt network to. Example: -n 192.168.123.1. Used with -g"

+	echo "-v, --verify           Verify environment is capable of running lxc"

+	echo ""

+	exit $1

+}

+

+while true ; do

+	case "$1" in

+	--help|-h|-\?) helptext 0;;

+	-c|--containers) containers="$2"; shift; shift;;

+	-d|--download-agent) download=1; shift;;

+	-s|--share-configs) share_configs=1; shift;;

+	-n|--network) addr="$2"; shift; shift;;

+	-r|--restore-libvirt) restore=1; shift;;

+	-p|--restore-cib) restore_pcmk=1; shift;;

+	-R|--restore-all)

+		restore_all=1

+		restore=1

+		restore_pcmk=1

+		shift;;

+	-g|--generate) generate=1; shift;;

+	-k|--key-gen) key_gen=1; shift;;

+	-a|--add-cib) cib=1; shift;;

+	-m|--add-master) add_master=1; shift;;

+	-v|--verify) verify=1; shift;;

+	"") break;;

+	*) helptext 1;;

+	esac

+done

+

+if [ $verify -eq 1 ]; then

+	# verify virsh tool is available and that 

+	# we can connect to lxc driver.

+	virsh -c lxc:/// list --all > /dev/null 2>&1

+	if [ $? -ne 0 ]; then

+		echo "Could not connect 'virsh -c lxc:///' check that libvirt lxc driver is installed"

+		exit 1

+	fi

+

+	cat /etc/selinux/config  | grep -e "SELINUX.*=.*permissive" -e "SELINUX.*=.*enforcing" > /dev/null 2>&1

+	if [ $? -ne 0 ]; then

+		echo "/etc/selinux/config must have SELINUX set to permissive or enforcing mode."

+		exit 1

+	fi

+

+	ps x > /tmp/lxc-autogen-libvirt-test.txt

+	grep "libvirtd" /tmp/lxc-autogen-libvirt-test.txt

+	if [ $? -ne 0 ]; then

+		rm -f /tmp/lxc-autogen-libvirt-test.txt

+		echo "libvirtd isn't up."

+		exit 1

+	fi

+	rm -f /tmp/lxc-autogen-libvirt-test.txt

+

+	which rsync > /dev/null 2>&1

+	if [ $? -ne 0 ]; then

+		echo "rsync is required"

+	fi

+

+fi

+

+#strip last digits off addr

+tmp="s/\.$(echo "$addr" | tr '.' ' ' | awk '{print $4}')$//g"

+addr=$(echo $addr | sed -e ${tmp})

+

+set_network()

+{

+	rm -f cur_network.xml

+	cat << END >> cur_network.xml

+<network>

+  <name>default</name>

+  <uuid>41ebdb84-7134-1111-a136-91f0f1119225</uuid>

+  <forward mode='nat'/>

+  <bridge name='virbr0' stp='on' delay='0' />

+  <mac address='52:54:00:A8:12:35'/>

+  <ip address='$addr.1' netmask='255.255.255.0'>

+    <dhcp>

+      <range start='$addr.2' end='$addr.254' />

+    </dhcp>

+  </ip>

+</network>

+END

+

+	ls restore_default.xml > /dev/null 2>&1

+	if [ $? -ne 0 ]; then

+		virsh net-dumpxml default > restore_default.xml

+	fi

+	virsh net-destroy default

+	virsh net-undefine default

+	virsh net-define cur_network.xml

+	virsh net-start default

+}

+

+generate_key()

+{

+	#generate pacemaker remote key

+	ls /etc/pacemaker/authkey > /dev/null 2>&1

+	if [ $? != 0 ]; then

+			mkdir -p /etc/pacemaker

+			dd if=/dev/urandom of=/etc/pacemaker/authkey bs=4096 count=1

+	fi

+}

+

+generate()

+{

+	set_network

+

+	# Generate libvirt domains in xml

+	for (( c=1; c <= $containers; c++ ))

+	do

+		rm -rf lxc$c-filesystem

+		mkdir -p lxc$c-filesystem/var/run/

+		mkdir -p lxc$c-filesystem/usr/var/run

+		rm -f lxc$c.xml

+		cat << END >> lxc$c.xml

+<domain type='lxc'>

+  <name>lxc$c</name>

+  <memory>102400</memory>

+  <os>

+    <type>exe</type>

+    <init>$working_dir/lxc$c-filesystem/launch-helper</init>

+  </os>

+  <devices>

+    <console type='pty'/>

+    <filesystem type='mount'>

+      <source dir='$working_dir/lxc$c-filesystem/var/run'/>

+      <target dir='/var/run'/>

+    </filesystem>

+    <filesystem type='mount'>

+      <source dir='$working_dir/lxc$c-filesystem/usr/var/run'/>

+      <target dir='/usr/var/run'/>

+    </filesystem>

+    <interface type='network'>

+      <mac address='52:54:00:$c$c:$(($RANDOM % 9))$(($RANDOM % 9)):$(($RANDOM % 9))$c'/>

+      <source network='default'/>

+    </interface>

+  </devices>

+</domain>

+END

+		rm -f container$c.cib

+		cat << END >> container$c.cib

+      <primitive class="ocf" id="container$c" provider="heartbeat" type="VirtualDomain">

+        <instance_attributes id="container$c-instance_attributes">

+          <nvpair id="container$c-instance_attributes-force_stop" name="force_stop" value="true"/>

+          <nvpair id="container$c-instance_attributes-hypervisor" name="hypervisor" value="lxc:///"/>

+          <nvpair id="container$c-instance_attributes-config" name="config" value="$working_dir/lxc$c.xml"/>

+          <nvpair id="container$c-instance_attributes-remote-node" name="remote-node" value="lxc$c"/>

+        </instance_attributes>

+        <utilization id="container$c-utilization">

+          <nvpair id="container$c-utilization-cpu" name="cpu" value="1"/>

+          <nvpair id="container$c-utilization-hv_memory" name="hv_memory" value="100"/>

+        </utilization>

+        <meta_attributes id="container$c-meta_attributes">

+          <nvpair id="container$c-meta_attributes-remote-node" name="remote-node" value="lxc$c"/>

+        </meta_attributes>

+      </primitive>

+END

+

+		rm -f lxc-ms$c.cib

+		cat << END >> lxc-ms.cib

+      <master id="lxc-ms-master">

+        <primitive class="ocf" id="lxc-ms" provider="pacemaker" type="Stateful">

+          <instance_attributes id="lxc-ms-instance_attributes"/>

+          <operations>

+            <op id="lxc-ms-monitor-interval-10s" interval="10s" name="monitor"/>

+          </operations>

+        </primitive>

+        <meta_attributes id="lxc-ms-meta_attributes">

+          <nvpair id="lxc-ms-meta_attributes-master-max" name="master-max" value="1"/>

+          <nvpair id="lxc-ms-meta_attributes-clone-max" name="clone-max" value="$containers"/>

+        </meta_attributes>

+      </master>

+END

+

+		rm -f lxc$c-filesystem/launch-helper

+		cat << END >> lxc$c-filesystem/launch-helper

+#!/bin/bash

+ifconfig eth0 $addr.10$c

+route add 0.0.0.0 gw $addr.1 eth0

+hostname lxc$c

+/usr/sbin/pacemaker_remoted

+END

+		chmod 711 lxc$c-filesystem/launch-helper

+

+		cat << END >> /etc/hosts

+$addr.10$c     lxc$c

+END

+	done

+}

+

+apply_cib_master()

+{

+	cibadmin -Q > cur.cib

+	export CIB_file=cur.cib

+

+	cibadmin -o resources -C -x lxc-ms.cib

+	for tmp in $(ls lxc*.xml); do

+		tmp=$(echo $tmp | sed -e 's/\.xml//g')

+		echo "<rsc_location id=\"lxc-ms-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-ms\" score=\"INFINITY\"/>" > tmp_constraint

+		cibadmin -o constraints -C -x tmp_constraint

+		echo "<rsc_location id=\"lxc-ping-location-${tmp}\" node=\"${tmp}\" rsc=\"ping-1\" score=\"-INFINITY\"/>" > tmp_constraint

+		cibadmin -o constraints -C -x tmp_constraint > /dev/null 2>&1

+		rm -f tmp_constraint

+	done

+	unset CIB_file

+

+	cibadmin --replace --xml-file cur.cib

+	rm -f cur.cib

+}

+

+apply_cib_entries()

+{

+	node=$(crm_node -n)

+

+	cibadmin -Q > cur.cib

+	export CIB_file=cur.cib

+	for tmp in $(ls container*.cib); do

+		cibadmin -o resources -C -x $tmp

+

+		tmp=$(echo $tmp | sed -e 's/\.cib//g')

+		crm_resource -M -r $tmp -H $node

+	done

+	unset CIB_file

+

+	cibadmin --replace --xml-file cur.cib

+	rm -f cur.cib

+}

+

+restore_cib()

+{

+	node=$(crm_node -n)

+	cibadmin -Q > cur.cib

+	export CIB_file=cur.cib

+

+	for tmp in $(ls lxc*.xml); do

+		tmp=$(echo $tmp | sed -e 's/\.xml//g')

+		echo "<rsc_location id=\"lxc-ms-location-${tmp}\" node=\"${tmp}\" rsc=\"lxc-ms\" score=\"INFINITY\"/>" > tmp_constraint

+		cibadmin -o constraints -D -x tmp_constraint

+		echo "<rsc_location id=\"lxc-ping-location-${tmp}\" node=\"${tmp}\" rsc=\"ping-1\" score=\"-INFINITY\"/>" > tmp_constraint

+		cibadmin -o constraints -D -x tmp_constraint

+		rm -f tmp_constraint

+	done

+	cibadmin -o resources -D -x lxc-ms.cib

+

+	for tmp in $(ls container*.cib); do

+		tmp=$(echo $tmp | sed -e 's/\.cib//g')

+		crm_resource -U -r $tmp -H $node

+		crm_resource -D -r $tmp -t primitive

+	done

+	unset CIB_file

+

+	cibadmin --replace --xml-file cur.cib

+	rm -f  cur.cib 

+}

+

+restore_libvirt()

+{

+	for tmp in $(ls lxc*.xml); do

+		tmp=$(echo $tmp | sed -e 's/\.xml//g')

+		virsh -c lxc:/// destroy $tmp > /dev/null 2>&1

+		virsh -c lxc:/// undefine $tmp > /dev/null 2>&1

+

+		sed -i.bak "/...\....\....\..* ${tmp}/d" /etc/hosts

+		echo "$tmp destroyed"

+	done

+

+	ls restore_default.xml > /dev/null 2>&1

+	if [ $? -eq 0 ]; then

+		virsh net-destroy default > /dev/null 2>&1

+		virsh net-undefine default > /dev/null 2>&1

+		virsh net-define restore_default.xml

+		virsh net-start default

+		if [ $? -eq 0 ]; then

+			echo "default network restored"

+		fi

+	fi

+	rm -f restore_default.xml > /dev/null 2>&1 

+}

+

+distribute_configs()

+{

+	local node

+	local id

+	while read id node

+	do

+		rsync -ave 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' $working_dir/lxc*.xml $node:/$working_dir

+		rsync -ave 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' $working_dir/lxc*-filesystem $node:/$working_dir

+	done < <(crm_node -l)

+}

+

+mkdir -p $working_dir

+cd $working_dir

+

+if [ $download -eq 1 ]; then

+	wget https://raw.github.com/ClusterLabs/resource-agents/master/heartbeat/VirtualDomain

+	chmod 755 VirtualDomain

+	mv -f VirtualDomain /usr/lib/ocf/resource.d/heartbeat/VirtualDomain

+fi

+if [ $restore_pcmk -eq 1 ]; then

+	restore_cib

+fi

+if [ $restore -eq 1 ]; then

+	restore_libvirt

+fi

+if [ $key_gen -eq 1 ]; then

+	generate_key

+fi

+if [ $generate -eq 1 ]; then

+	if [ $key_gen -eq 0]; then

+		generate_key

+	fi

+	generate

+fi

+if [ $cib -eq 1 ]; then

+	apply_cib_entries

+fi

+if [ $add_master -eq 1 ]; then

+	apply_cib_master

+fi

+if [ $share_configs -eq 1 ]; then

+	distribute_configs

+fi

+if [ $restore_all -eq 1 ]; then

+	ls | grep -v "lxc.\.xml" | xargs rm -rf

+fi

+

+cd $curdir