#!/usr/bin/python
import threading
import time
import os, commands
import getopt, sys
import sys
import re
import json
import ujson
import os,sys,socket,shutil
import logging
import platform
import testsum

try:
	import argparse
	import pexpect
	import setproctitle
except ImportError as e:
	module = str(e)[16:]
	sys.exit("we require the python module %s " % module)

#hosts = ['east', 'west', 'road', 'nic', 'north']
r_init = threading.Event()
i_ran = threading.Event()
n_init = threading.Event()
result_file_lock = threading.Lock()

def rsync_file_to_dir (src, dst):
	cmd="/usr/bin/rsync -q -aP %s %s/" % (src, dst)
	logging.debug("%s %s", os.getcwd(), cmd)
	try:
		os.system(cmd)
	except:
		logging.exception("EXCEPTION ? cmd %s , cwd %s", os.getcwd(), cmd)

class guest (threading.Thread):
	def __init__(self, hostname, role, testname, args, install=None, compile=None):
		threading.Thread.__init__(self)
		self.hostname = hostname
		self.role = role
		self.testname = testname
		self.status = "NEW"
		self.bootwait = args.bootwait
		self.stoponerror = args.stoponerror

		self.reboot = True
		if args.noreboot:
			self.reboot = None

	def run(self):
		self.start = time.time()
		e = self.boot_n_grab_console()
		self.status = "INIT"
		if e:
			self.clean_abort();
		else:
			e = self.run_test()

		if not e:
			e = "end"
		self.log_line('OUTPUT/RESULT', e)
		logging.info("%s done %s ran %s sec %s", self.hostname, self.testname,
				(time.time() - self.start), e)

	def boot_n_grab_console(self):
		e = self.connect_to_kvm()
		if e:
			self.clean_abort();
			return e


	def clean_abort(self):
	   # we are aborting: set all waiting events end of show.
	   logging.debug("%s set all events to abort", self.hostname)
	   r_init.set()
	   n_init.set()
	   i_ran.set()


	def run_test(self):

		# now on we MUST match the entire prompt,
		# or else we end up sending too soon and getting mangling!

		prompt = "\[root@%s %s\]# " % (self.hostname, self.testname)
		logging.debug ("%s role %s running test %s prompt %s",
				self.hostname, self.role, self.testname, prompt.replace("\\",""))

		child = self.child
		timer = 120
		ret = True

		cmd = "cd /testing/pluto/%s " % (self.testname)
		child.sendline(cmd)
		try:
			child.expect (prompt, searchwindowsize=100,timeout=timer)
		except:
			err = "failed [%s] test directory" %(cmd)
			logging.error("%s", self.hostname, err)
			return err

		if self.role == "initiator":
			start = time.time()
			logging.info("%s wait for responder and maybe nic to initialize",
					self.hostname)
			n_init.wait()
			logging.debug("%s wait for responder to initialize", self.hostname)
			r_init.wait()
			logging.info("%s initiator is ready (waited %s)", self.hostname,
					(time.time() - start))

		output_file = "OUTPUT/%s.console.verbose.txt" % (self.hostname)
		f = open(output_file, 'w')
		child.logfile = f

		self.status = "INIT-RUN"
		cmd = "./%sinit.sh" %  (self.hostname)
		e = read_exec_shell_cmd(child, cmd, prompt, timer, self.hostname)

		if (self.role == "responder"):
			r_init.set()

		if (self.role == "nic"):
			n_init.set()

		if e:
			f.close
			return e

		cmd = "./%srun.sh" %  (self.hostname)
		if os.path.exists(cmd):
			e = read_exec_shell_cmd( child, cmd, prompt, timer, self.hostname)
			i_ran.set()
			if e:
				f.close
				return e
		else:
			start = time.time()
			print("waiting for initiator to finish run")
			i_ran.wait()
			print("initiator is ran waited %s" % (time.time() - start))

		cmd = "./final.sh"
		if os.path.exists(cmd):
			e =  read_exec_shell_cmd( child, cmd, prompt, timer, self.hostname)
			if e:
				f.close
				return e
		f.close

	def log_line(self, filename, msg):

		if not self.testname:
			return

		logline = dict ()
		#output_file = "OUTPUT/RESULT"
		logline ["epoch"] = time.time()
		logline ["hostname"]  =  self.hostname
		logline ["testname"]  =  self.testname
		logline ["msg"] = msg
		logline ["runtime"] = time.time() - self.start
		logline ["time"] = time.strftime("%Y-%m-%d %H:%M", time.localtime())

		result_file_lock.acquire()
		f = open(filename, 'a')
		f.write(ujson.dumps(logline, ensure_ascii=True,  double_precision=2))
		f.write("\n")
		f.close
		result_file_lock.release()


	def connect_to_kvm(self):

		prompt = "\[root@%s " % (self.hostname)

		vmlist = commands.getoutput("sudo virsh list")
		running = False
		for line in vmlist.split("\n")[2:]:
			try:
				num,host,state = line.split()
				if host == self.hostname and state == "running":
					running = True
					print("Found %s running already" % self.hostname)
					continue
			except:
				pass

		bootwait = self.bootwait
		pause = bootwait

		if bootwait > 15:
			pause = 15

		if not running:
			done = False
			v_start = ''
			tries =  bootwait;
			while not done and tries != 0:
				if os.path.isfile("OUTPUT/stop-tests-now"):
					return "aborting: found OUTPUT/stop-tests-now"

				print("Booting %s %s/%s" % (self.hostname,tries, bootwait))
				v_start = commands.getoutput("sudo virsh start %s" % self.hostname)
				logging.info(v_start)
				re_e = re.search(r'error:', v_start, re.I )
				if re_e:
					tries -= 1
					time.sleep(1)
				else:
					done = True

					# just abort this test
			if not done:
				v_start = "KVMERROR %s " % self.hostname + v_start
				logging.error(v_start)
				self.log_line('OUTPUT/stop-tests-now', v_start)
				if self.stoponerror:
					# the whole show ends here
					self.log_line('../stop-tests-now', v_start)
					return v_start

			time.sleep(pause)
		elif self.reboot:
			commands.getoutput("sudo virsh reboot %s" % self.hostname)
			print("Rebooting %s - pausing %s seconds" % (self.hostname,pause))
			time.sleep(pause)

		print("Taking %s console by force" % self.hostname)
		cmd = "sudo virsh console --force %s" % self.hostname
		timer = 120
		child = pexpect.spawn(cmd)
		child.delaybeforesend = 0.1
		self.child = child
		# child.logfile = sys.stdout
		# don't match full prompt, we want it to work regardless cwd

		done = False
		tries =  bootwait - pause + 1

		print("Waiting on %s login: %s" % (self.hostname, prompt))
		while not done and tries != 0:
			if os.path.isfile("OUTPUT/stop-tests-now"):
				return "aborting: found OUTPUT/stop-tests-now"
			try:
				child.sendline ('')
				print("%s [%s] waiting on login: or %s" % (self.hostname,
					tries, prompt))
				res = child.expect (['login: ', prompt], timeout=3)
				if res == 0:
					print("%s sending login name root" % self.hostname)
					child.sendline ('root')
					print("%s found, expecting password prompt" % self.hostname)
					child.expect ('Password:', timeout=1)
					print("%s found, sending password" % self.hostname)
					child.sendline ('swan')
					print("%s waiting on root shell prompt %s" % (self.hostname, prompt))
					child.expect ('root.*', timeout=1)
					print("got prompt %s" % prompt.replace("\\",""))
					done = True
				elif res == 1:
					print('----------------------------------------------')
					print(' Already logged in as root on %s' % prompt.replace("\\",""))
					print('----------------------------------------------')
					done = True
			except:
				tries -= 1
				time.sleep(1)

		if not done:
			err = 'KVMERROR console is not answering: abort test'
			logging.error("%s %s %s",self.hostname, err, self.testname)
			self.log_line('OUTPUT/stop-tests-now', err)

			if self.stoponerror:
				logging.error("stop")
				self.log_line('../stop-tests-now', err)
			return err

		child.sendline ('TERM=dumb; export TERM; unset LS_COLORS')
		res = child.expect (['login: ', prompt], timeout=3)
		child.setecho(False) ## this does not seems to work
		child.sendline("stty sane")
		res = child.expect (['login: ', prompt], timeout=3)
		child.sendline("stty -onlcr")
		res = child.expect (['login: ', prompt], timeout=3)
		child.sendline("stty -echo")
		res = child.expect (['login: ', prompt], timeout=3)

# end of class

def get_results(r):
	if os.path.exists(r):
		f = open(r, 'r')
		for line in f:
			x = ujson.loads(line)
			if "result" in x and "testname"  in x:
				return x

def shut_down_hosts(args, test_hosts):

	running = []
	all_hosts = list(DEFAULTCONFIG['swanhosts'])
	all_hosts.extend(DEFAULTCONFIG['regualrhosts'])

	if args.noreboot:
		logging.debug("all hosts [%s] shutdown list", ' '.join(map(str, all_hosts)))
		logging.debug("remove [%s] from shutdown list", ' '.join(map(str, test_hosts)))
		for t in test_hosts:
			for h in all_hosts:
				if h is t:
					all_hosts.remove(t)

	logging.debug("shutdown list [%s] ", ' '.join(map(str, all_hosts)))

	vmlist = commands.getoutput("sudo virsh list")
	for line in vmlist.split("\n")[2:]:
		try:
			num,host,state = line.split()
			for h in all_hosts:
				if h == host:
					running.append(host)
					cmd = "sudo virsh shutdown %s" % host
					logging.debug("Found %s %s shutodwn send %s", host, state, cmd)
					shut = commands.getoutput(cmd)
		except:
			pass

	tries = args.shutdownwait
	while len(running) and tries != 0:
		logging.info("Found %s guests [%s] running. Wait upto %d seconds to shutdown", len(running),
				' '.join(map(str, running)), tries)

		del running[:]
		try:
			vmlist = commands.getoutput("sudo virsh list")
			for line in vmlist.split("\n")[2:]:
				try:
					num,host,state = line.split()
					for h in all_hosts:
						if h == host:
							running.append(host)
				except:
					pass
		except:
			pass
		tries -= 1
		time.sleep(1)

	if len(running):
		e = "KVMERROR not able to shutdown %s guests: [%s] abort" % (len(running), ' '.join(map(str, running)))
		logging.error (e)
		return e

def orient (t):
	test_hosts = []
	if not os.path.exists("eastrun.sh"):
		t["responder"] = "east"
	else:
		logging.error ("ABORT can't identify RESPONDER: no %s/eastinit.sh" % os.getcwd())

	if os.path.exists("nicinit.sh"):
		t["nic"] = "nic"
		test_hosts.append(t["nic"])

	tcpdump_devs = ["swan12"]
	tcpdump_filter = "not stp and not port 22"

	if os.path.exists("westrun.sh"):
		t["initiator"] = "west"
	elif os.path.exists("roadrun.sh"):
		t["initiator"] = "road"
		tcpdump_devs.append("swan13")
	elif os.path.exists("northrun.sh"):
		t["initiator"] = "north"
		tcpdump_devs.append("swan13")
	else:
		logging.error("ABORT can't identify INITIATOR in directory %s" % os.getcwd())

	test_hosts.append(t["initiator"])
	test_hosts.append(t["responder"])
	cmds = []
	for iface in tcpdump_devs:
		pcap_file =  'OUTPUT/' + iface + '.pcap'
		cmd = "/sbin/tcpdump -s 0 -w %s -n -i %s %s &" % (pcap_file, iface, tcpdump_filter)
		logging.debug(cmd)
		cmds.append(cmd)

	return cmds, test_hosts

def read_exec_shell_cmd(ex, filename, prompt, timer, hostname = ""):

	if os.path.exists(filename):
		logging.debug("%s execute commands from file %s", hostname, filename)
		f_cmds = open(filename, "r")
		for line in f_cmds:
			if os.path.isfile("OUTPUT/stop-tests-now"):
				return "aborting: found OUTPUT/stop-tests-now"

			line = line.strip()
			# We need the lines with # for the cut --- tuc sections
			# if line and not line[0] == '#':
			if line:
				print("%s: %s" % (prompt.replace("\\",""), line))
				ex.sendline(line)
				try:
					ex.expect(prompt,timeout=timer, searchwindowsize=100)
				except:
					err = "#%s timedout send line: %s" % (prompt,line)
					logging.error("%s try sending CTRL+c and continue",err)
					ex.sendcontrol('c')
					ex.sendline(err)
					# in the old days the function would return here.
					#f_cmds.close
					#return err
					err = ''

		f_cmds.close

	else:
		# not a file name but a command: send it as it is.
		print(filename)
		ex.sendline(filename)
		try:
			ex.expect (prompt,timeout=timer, searchwindowsize=100)
		except:
			err = "%s failed to send command: %s" % (prompt, filename)
			logging.debug("%s %s", hostname, err)
			return	err

# kill any lingering tcpdumps
def kill_zombie_tcpdump(signal=1):
	pids = commands.getoutput("pidof tcpdump")
	for pid in (pids.split()):
		logging.info("killing tcpdump process %s signal = %s", pid, signal)
		try:
			os.kill(int(pid),signal)
		except OSError as e:
			logging.error("killing tcpdump process %s %s", pid, str(e))

# kill all hanging previous of instances of this script.
def kill_zombies(proctitle):
	me = os.getpid()
	zombie_pids = commands.getoutput("pidof %s" % proctitle)
	for pid in ( zombie_pids.split() ):
		if int(pid) != int(me):
			logging.info ("killing %s pid %s from [%s] my pid %s", proctitle, pid, zombie_pids, me)
			os.kill(int(pid),9)
	kill_zombie_tcpdump(signal=9)

def init_output_dir():
	output_dir = "%s/OUTPUT" % os.getcwd()

	if os.path.isdir(output_dir):
		shutil.rmtree(output_dir)
	os.mkdir(output_dir, 0o777)

def sanitize(cmd):
	sanity = commands.getoutput(cmd)
	#logging.info ("sanitizer output %s", sanity.replace("\n", " "))
	logging.info ("sanitizer output %s", sanity)
	return sanity

def split_sanity(sanity):
	if sanity:
		for line in sanity.split("\n")[2:]:
			try:
				key,name,value = line.split()
				if key == "result":
					return value
			except:
				pass

def re_write_result(args, testname, sanity, result = "FAILED"):

	changed = False

	if sanity:
		result = split_sanity(sanity)
	else:
		return
	output_file = "OUTPUT/RESULT"
	if not os.path.exists(output_file):
		return

	lines = []
	f = open(output_file, 'r')
	for line in f:
		x = None
		try:
			x = ujson.loads(line)
		except:
			lines.append(line)
			contine

		if x and "result" in x and "testname" in x:
			if x["result"] == result:
				logging.debug("testcase %s result didn't change %s",testname, result)
				continue

			changed = True
			logging.info("testcase %s result new '%s' old '%s'",testname, x["result"],  result)
			x[result] = result
			x["resanitized"] = time.strftime("%Y-%m-%d %H:%M", time.localtime())
			lines.append(ujson.dumps(x, ensure_ascii=True, double_precision=2))

	f.close


	if not changed:
		return False

	f = open(output_file, 'w')

	for line in lines:
		f.write(line)
		f.write("\n")

	f.close
	return changed

def write_result(args, start, testname, sanity, result = 'FAILED', e = None, testexpect = ''):

	if sanity:
		result = split_sanity(sanity)

	logline = dict ()
	output_file = "OUTPUT/RESULT"
	f = open(output_file, 'a')
	logline ["epoch"] = time.time()
	logline ["testname"]  =  testname
	logline ["result"] = result
	logline ["time"] = time.strftime("%Y-%m-%d %H:%M", time.localtime())
	logline ["runtime"] = time.time() - start
	logline ["node"] = args.node
	if testexpect:
		logline ["expect"] = testexpect

	if e:
		logline ["error"] = e
	f.write(ujson.dumps(logline, ensure_ascii=True, double_precision=2))
	f.write("\n")
	f.close

DEFAULTCONFIG =  {
	'resultsdir' : '/home/build/results',
	'sanitizer' : "../../utils/sanitize.sh",
	'bootwait' : 109,
	'swanhosts' : ['east', 'west', 'road', 'north'],
	'regualrhosts' : ['nic'],
	'shutdownwait': 63,
	'newrun' : None,
	'stoponerror' : None,
	'retry' : 5,
	'node' : platform.node(),
	'testlist' : "TESTLIST"
}

def cmdline():
	parser = argparse.ArgumentParser(description='swantest arguments.')

	parser.add_argument("--retry",
			default=DEFAULTCONFIG['retry'], type=int,
			help="retry when there is console error.")

	parser.add_argument("--node", default=DEFAULTCONFIG['node'], help="Default node name.")

	parser.add_argument("--resanitize", default=False, action="store_true",
			help="re-sanitize results, run it from ~/libreswan/testing/pluto directory")

	parser.add_argument("--graphs", default=False, action="store_true",
			help="generate graphs and JSON tables.")

	parser.add_argument("--stop", default=False, action="store_true",
			help="stop tests now. call from testing/pluto directory")

	parser.add_argument("--stoponerror",
			default=DEFAULTCONFIG['stoponerror'], action="store_true",
			help="Stop on kvm errors. Default coninues")

	parser.add_argument("--newrun",
			default=DEFAULTCONFIG['newrun'], action="store_true",
			help="overwrite the results in %s. Default %s" % (DEFAULTCONFIG['resultsdir'], DEFAULTCONFIG['newrun']))

	parser.add_argument("--resultsdir",
						default=DEFAULTCONFIG['resultsdir'],
						help="test results directory %s" % DEFAULTCONFIG['resultsdir'])
	parser.add_argument("--sanitizer",
						default=DEFAULTCONFIG['sanitizer'],
						help="sanitizer script. %s" % DEFAULTCONFIG['sanitizer'])
	parser.add_argument("--testlist",
						default=DEFAULTCONFIG['testlist'],
						help="read kvmpluto tests from  %s. in the repositiry testing/pluto/TESTLIST." % DEFAULTCONFIG['testlist'])

	parser.add_argument("-v", "--verbose",
						default=1, type=int,
						help="increase verbosity: 0 = only warnings, 1 = info, 2 = debug. Default info")
	parser.add_argument("--bootwait",
						default=DEFAULTCONFIG['bootwait'], type=int,
						help="seconds to reboot guest. Default %s seconds" % DEFAULTCONFIG['bootwait'])
	parser.add_argument("--shutdownwait",
						default=DEFAULTCONFIG['shutdownwait'], type=int,
						help="seconds to wait for guest to shutdown. Default %s seconds" % DEFAULTCONFIG['shutdownwait'])
	parser.add_argument('--testname', '-t',
						help='The name of the test to run.')
	parser.add_argument('--noreboot',
						default=None, action="store_true",
						help='Dont reboot vm. Default reboot')
	parser.add_argument('--leavezombies',
						default=None, action="store_true",
						help='leave other instances running. Default kill all other swantest and lingering tcpdump')

	args = parser.parse_args()

	proctitle="swantest"
	if not args.leavezombies:
		kill_zombies(proctitle)

	setproctitle.setproctitle(proctitle)
	logger =  logging.getLogger()
	if args.verbose == 0:
		logger.setLevel(logging.WARN)
	elif args.verbose == 1:
		logger.setLevel(logging.INFO)
	elif args.verbose == 2:
		logger.setLevel(logging.DEBUG)

	return args

def kvm_error(r_file):
	if not os.path.exists(r_file):
		return  False
	f = open(r_file, 'r')
	for line in f:
		x = json.loads(line)
		try:
			re_e = re.search(r'KVMERROR', x['error'], re.I )
			if re_e:
				logging.info(line)
				f.close
				return line
		except:
			pass
	f.close
	return False


def do_test(args, start='', testexpect=''):

	if not start:
		start = time.time()

	args = cmdline()

	testname = ''
	if args.testname:
		testname =	args.testname
	else:
		testname = os.path.basename(os.getcwd())

	logging.info("***** KVM PLUTO RUNNING test %s *******", testname)

	t = dict()
	tcpdump_cmds, test_hosts = orient(t)

	e = shut_down_hosts(args, test_hosts)

	init_output_dir()

	if e:
		write_result(args, start, testname, None, "abort", e)
		kill_zombie_tcpdump()
		# we can't call exit(1) "make check" will abort then
		return e

	for cmd in tcpdump_cmds:
		os.system(cmd)

	r_init.clear()
	i_ran.clear()
	n_init.clear()

	# Create new threads
	th_responder = guest(t["responder"], "responder", testname, args)
	if "nic" in t:
		th_nic = guest(t["nic"], "nic", testname, args)
	th_initiator = guest(t["initiator"], "initiator", testname, args)

	# Start new Threads
	th_responder.start()
	if "nic" in t:
		th_nic.start()
	else:
		n_init.set()

	th_initiator.start()

	th_responder.join()
	th_initiator.join()
	if "nic" in t:
		th_nic.join()

	kill_zombie_tcpdump()

	s = sanitize(args.sanitizer)
	write_result(args, start, testname, s, testexpect = testexpect)


def skip_testlist_line(args, line, verbose=False):
	

	if line[0] == '#' or re.match('^[ \t]*$', line):
		# skip the comments
		return True

	try:
		testtype, testdir, testexpect = line.split()
	except:
		logging.error("****** skipping malformed: %s", line)
		return True

	if testtype == "skiptest":
		if verbose:
			logging.error("****** %s", line)
		return True
	elif testtype  != "kvmplutotest":
		if verbose:
			logging.error("****** skipping test %s: yet to be migrated to kvm style ", testdir)
		return True
	elif not os.path.exists(testdir):
		if verbose:
			logging.error("**** Skipping missing %s/%s *****",
				os.getcwd(), testdir)
		return True

	return False

def resanitize(args):
	r = args.testlist
	logging.info("re sanitize the existing results checking for %s. Typically, this should start from ~/libreswan/testing/pluto", r)
	if not os.path.exists(r):
		logging.error("ABORT re sanitize missing %s. Did it start from right place", r)
		return None

	output_dir = setup_result_dir(args)

	rsync_file_to_dir(args.testlist, output_dir)
	f = open(r, 'r')
	for line in f:
		logging.debug("%s", line)

		if skip_testlist_line(args,line, verbose=False):
			continue
		
		testtype, testdir, testexpect = line.split()
		rdif = testdir  + '/OUTPUT/'

		r1 = testdir  + '/OUTPUT/RESULT'

		x = get_results(r1)
		if not x :
			# there is no result ?? 
			continue
		
		if x and (x["result"] == 'failed'):
			logging.debug("resanitize now %s %s",testdir, testexpect )

			## chdir start
			os.chdir(testdir)

			s = sanitize(args.sanitizer)
			a = re_write_result(args=args, testname = testdir, sanity=s)
			if a:
				logging.info("new result %s", a)

			#for h in DEFAULTCONFIG['swanhosts']:
			#	f = "OUTPUT/%s.console.diff"%h
			#	if  os.path.exists(f):
			#		cmd = "egrep '^[+-]' %s | egrep -v '^(\+\+\+|---)' | sort -u | cmp -s - ../strongswan-star-diff"%f
			#		(status, output) =  commands.getstatusoutput(cmd);
			#		if not status:
			#			print "cp %s/OUTPUT/%s.console.txt %s/ "%(testdir,h,testdir)
			#			print "cat %s/%s"%(testdir,f)

			os.chdir("..")
			## chdir end

		#copy the test  to new result directory
		if os.path.exists(output_dir):
			rsync_file_to_dir(testdir, output_dir);
		else:
			logging.info("missing output dir %s. can not copy resanitized results",output_dir)

	testsum.read_dirs(args, output_dir)

def do_test_list(args, start, tried, output_dir):
	if not os.path.exists(args.testlist):
		return None
	ran =  1;

	logging.info("** run kvmpluto tests from file %s **" , args.testlist)

	if not os.path.exists(args.resultsdir):
		try:
			os.mkdir(args.resultsdir, 0o755)
		except:
			logging.error("failed to create directory %s", args.resultsdir)
	else:
		logging.info("testresult directory %s exist", args.resultsdir)

	s = 'stop-tests-now'
	if os.path.exists(s) and (tried == 0):
		os.unlink(s)
		logging.debug("removing existing %s" % s)

	rsync_file_to_dir(args.testlist, output_dir)
	f = open(args.testlist, 'r')
	for line in f:
		logging.debug("%s", line)
		if os.path.exists(s):
			logging.error("* stop all tests now. Found %s *", s)
			return True

		if skip_testlist_line(args, line, verbose=True):
			continue
		
		testtype, testdir, testexpect = line.split()

		r_file = output_dir + '/' + testdir  + '/OUTPUT/RESULT'
		if os.path.exists(r_file) and tried and (tried < args.retry):
			if kvm_error(r_file):
				logging.info("result [%s] KVMERROR. delete previous run, retry %s/%s", r_file, tried, args.retry)
				o_dir = output_dir + '/' + testdir
				shutil.rmtree(o_dir)
				try:
					os.mkdir(o_dir, 0o755)
				except:
					logging.error("failed to create directory %s", output_dir)

			else:
				continue
		elif os.path.exists(r_file):
			# output already exists: skip test
			continue
		else:
			logging.debug("%s is not there; run this test", r_file)

		## chdir start
		os.chdir(testdir)
		if os.path.exists('stop-tests-now'):
			logging.error("****** skip test %s found stop-tests-now *****", testdir)
			os.chdir("..")
			## chdir end
			continue

		logging.debug("****** next test %s *****", testdir)
		do_test(args, testexpect = testexpect )
		ran =  ran + 1
		os.chdir("..")
		## chdir end

		if os.path.exists(output_dir):
			rsync_file_to_dir(testdir, output_dir)
		else:
			logging.info("missing output dir %s. can not copy results",output_dir)

	f.close
	return ran

def setup_result_dir(args):
	date_dir = time.strftime("%Y-%m-%d", time.localtime())
	output_dir = args.resultsdir

	if (args.node):
		output_dir = args.resultsdir + '/' + args.node

	if not os.path.isdir(output_dir):
		try:
			logging.debug("create directory %s", output_dir)
			os.mkdir(output_dir, 0o777)
		except:
			logging.error("failed to create directory %s", output_dir)
	else:
			logging.debug("directory %s exists", output_dir)

	if args.node:
		output_dir = output_dir + '/' + date_dir + '-' + args.node
	else:
		output_dir = output_dir + '/' + date_dir

	ipsecversion = ''
	try:
		# inside 'make check' IPSECVERSION is set
		ipsecversion =  os.environ["IPSECVERSION"]
	except:
		pass

	if not ipsecversion:
		# Lets assume the path is <LIBRESWANSRC>/testing/utis 
		cwd =  os.getcwd()
		dn =   os.path.normpath(os.path.dirname(os.path.realpath(__file__)))
		os.chdir(dn)
		os.chdir("../../")
		cmd = "make showversion"
		ipsecversion = commands.getoutput(cmd)
		ipsecversion = ipsecversion.strip()
		try:
			os.chdir(cwd)
		except:
			pass
		logging.debug("cwd = %s , swantest = %s , %s, IPSECVERSION %s", cwd, dn, cmd, ipsecversion)
	else :
		logging.debug("IPSECVERSION %s was set in env", ipsecversion)

	if ipsecversion:
		output_dir = output_dir + '-' + ipsecversion

	if (args.resanitize):
		output_dir =  output_dir + "-resanitized"

	if args.newrun and os.path.exists(output_dir):
		logging.info("newrun, remove results [%s]", output_dir)
		if os.path.islink(output_dir):
			os.unlink(output_dir)
		elif os.path.isdir(output_dir):
			shutil.rmtree(output_dir)
		else:
			os.unlink(output_dir)
	try:
		os.mkdir(output_dir, 0o777)
	except:
		logging.error("failed to create directory %s", output_dir)

	logging.info("results will be in %s", output_dir)
	return output_dir

def gen_graphs_tables(args):
	testsum.read_dirs(args)

def gen_stop_tests_now(args):
	if not os.path.exists(args.testlist):
		return None

	s = "stop-tests-now"
	f = open(s, 'w')
	f.write("stop\n")
	f.close

def main():
	start = time.time()
	args = cmdline()
	tried = 0
	ran = 2 # used for summery generation.

	if args.graphs:
		gen_graphs_tables(args)
		return
	elif args.stop:
		gen_stop_tests_now(args)
		return
	elif args.resanitize:
		resanitize(args)
		return

	output_dir = setup_result_dir(args)
	if do_test_list(args, start, tried, output_dir): #try if there is a TESTLIST
		while (tried < args.retry):
			if ran > 1:
				gen_graphs_tables(args)
			tried = 1 + tried
			logging.info("retry %s %s/%s ", args.testlist, tried, args.retry)
			ran = do_test_list(args, start, tried, output_dir)

	else:
		do_test(args, start = start)  # no TESTLIST. Let's try a single test

if __name__ == "__main__":
	main()
