#!/bin/bash

#

# This Source Code Form is subject to the terms of the Mozilla Public

# License, v. 2.0. If a copy of the MPL was not distributed with this

# file, You can obtain one at http://mozilla.org/MPL/2.0/.

#

#

# Copyright (c) 2014, Joyent, Inc.

#

#

# catest: a simple testing tool and framework. See usage below for details.

#

shopt -s xpg_echo

#

# Global configuration

#

cat_arg0=$(basename $0)			# canonical name of "catest"

cat_outbase="catest.$$"			# output directory name

cat_tstdir="test"			# test directory

#

# Options and arguments

#

cat_tests=""				# list of tests (absolute paths)

opt_a=false				# run all tests

opt_c=false				# colorize test results

opt_k=false				# keep output of successful tests

opt_o="/var/tmp"			# parent directory for output directory

opt_t=					# TAP format output file

opt_S=false				# Non-strict mode for js tests

#

# Current state

#

cat_outdir=				# absolute path to output directory

cat_tapfile=				# absolute path of TAP output file

cat_ntests=				# total number of tests

cat_nfailed=0				# number of failed tests run

cat_npassed=0				# number of successful tests run

cat_nrun=0				# total number of tests run

#

# Allow environment-specific customizations.

#

[[ -f $(dirname $0)/catest_init.sh ]] && . $(dirname $0)/catest_init.sh

#

# fail MSG: emits the given error message to stderr and exits non-zero.

#

function fail

{

	echo "$cat_arg0: $@" >&2

	[[ -n $cat_tapfile ]] && echo "Bail out! $@" >> $cat_tapfile

	exit 1

}

#

# usage [MSG]: emits the given message, if any, and a usage message, then exits.

#

function usage

{

	[[ $# -ne 0 ]] && echo "$cat_arg0: $@\n" >&2

	cat <<USAGE >&2

Usage: $cat_arg0 [-k] [-c] [-o dir] [-t file] test1 ...

       $cat_arg0 [-k] [-c] [-o dir] [-t file] -a

In the first form, runs specified tests.  In the second form, runs all tests

found under "$cat_tstdir" of the form "tst*.<ext>" for supported extensions.

TESTS

    Tests are just files to be executed by some interpreter. In most cases, a

    test succeeds if it exits successfully and fails otherwise.  You can also

    specify the expected stdout of the test in a file with the same name as the

    test plus a ".out" suffix, in which case the test will also fail if the

    actual output does not match the expected output.

    Supported interpreter extensions are "sh" (bash) and "js" (node).

    This framework does not provide per-test setup/teardown facilities, but

    test files can do whatever they want, including making use of common

    libraries for setup and teardown.

TEST OUTPUT

    Summary output is printed to stdout.  TAP output can be emitted with "-t".

    Per-test output is placed in a new temporary directory inside the directory

    specified by the -o option, or /var/tmp if -o is not specified.

    Within the output directory will be a directory for each failed test which

    includes a README describing why the test failed (e.g., exited non-zero), a

    copy of the test file itself, the actual stdout and stderr of the test, and

    the expected stdout of the test (if specified).

    If -k is specified, the output directory will also include a directory for

    each test that passed including the stdout and stderr from the test.

The following options may be specified:

	-a 		Runs all tests under $cat_tstdir

			(ignores other non-option arguments)

	-c		Color code test result messages

	-h		Output this message

	-k		Keep output from all tests, not just failures

	-o directory	Specifies the output directory for tests

			(default: /var/tmp)

	-S		Turn off strict mode for tests

	-t file		Emit summary output in TAP format

USAGE

	exit 2

}

#

# abspath FILE: emits a canonical, absolute path to the given file or directory.

#

function abspath

{

	local dir=$(dirname $1) base=$(basename $1)

	if [[ $base = ".." ]]; then

		cd "$dir"/.. > /dev/null || fail "abspath '$1': failed to chdir"

		pwd

		cd - > /dev/null || fail "abspath '$1': failed to chdir back"

	else

		cd "$dir" || fail "abspath '$1': failed to chdir"

		echo "$(pwd)/$base"

		cd - > /dev/null || fail "abspath '$1': failed to chdir back"

	fi

}

#

# cleanup_test TESTDIR "success" | "failure": cleans up the output directory

#     for this test

#

function cleanup_test

{

	local test_odir="$1" result=$2

	local newdir

	if [[ $result = "success" ]]; then

		newdir="$(dirname $test_odir)/success.$cat_npassed"

	else

		newdir="$(dirname $test_odir)/failure.$cat_nfailed"

	fi

	mv "$test_odir" "$newdir"

	echo $newdir

}

#

# emit_failure TEST ODIR REASON: indicate that a test has failed

#

function emit_failure

{

	local test_label=$1 odir=$2 reason=$3

	if [[ $cat_tapfile ]]; then

		echo "not ok $(($cat_nrun+1)) $test_label" >> $cat_tapfile

	fi

	echo "${TRED}FAILED.${TCLEAR}"

	echo "$test_path failed: $reason" > "$odir/README"

	[[ -n "$odir" ]] && echo ">>> failure details in $odir\n"

	((cat_nfailed++))

}

#

# emit_pass TEST: indicate that a test has passed

#

function emit_pass

{

	local test_label=$1

	if [[ $cat_tapfile ]]; then

		echo "ok $((cat_nrun+1)) $test_label" >> $cat_tapfile

	fi

	echo "${TGREEN}success.${TCLEAR}"

	((cat_npassed++))

}

#

# Executes a single test

#

# Per-test actions:

# - Make a directory for that test

# - cd into that directory and exec the test

# - Redirect standard output and standard error to files

# - Tests return 0 to indicate success, non-zero to indicate failure

#

function execute_test

{

	[[ $# -eq 1 ]] || fail "Missing test to execute"

	local test_path=$1

	local test_name=$(basename $1)

	local test_dir=$(dirname $1)

	local test_label=$(echo $test_path | sed -e s#^$SRC/##)

	local test_odir="$cat_outdir/test.$cat_nrun"

	local ext=${test_name##*.}

	local faildir

	local EXEC

	echo "Executing test $test_label ... \c "

	mkdir "$test_odir" >/dev/null || fail "failed to create test directory"

	cp "$test_path" "$test_odir"

	case "$ext" in

	"sh") 	EXEC=bash ;;

	"js") 	EXEC=node ;;

	*) 	faildir=$(cleanup_test "$test_odir" "failure")

		emit_failure "$test_label" "$faildir" "unknown file extension"

		return 0

		;;

	esac

	pushd "$test_dir" >/dev/null

	if [[ $opt_S ]]; then

		$EXEC $test_name -S >$test_odir/$$.out 2>$test_odir/$$.err

	else

		$EXEC $test_name >$test_odir/$$.out 2>$test_odir/$$.err

	fi

	execres=$?

	popd > /dev/null

	if [[ $execres != 0 ]]; then

		faildir=$(cleanup_test "$test_odir" "failure")

		emit_failure "$test_label" "$faildir" "test returned $execres"

		return 0

	fi

	if [[ -f $test_path.out ]] && \

	    ! diff $test_path.out $test_odir/$$.out > /dev/null ; then

		cp $test_path.out $test_odir/$test_name.out

		faildir=$(cleanup_test "$test_odir" "failure")

		emit_failure "$test_label" "$faildir" "stdout mismatch"

		return 0

	fi

	cleanup_test "$test_odir" "success" > /dev/null

	emit_pass "$test_label"

}

while getopts ":o:t:ackSh?" c $@; do

	case "$c" in

	a|c|k|S) 	eval opt_$c=true ;;

	o|t) 	eval opt_$c="$OPTARG" ;;

	h)	usage ;;

	:) 	usage "option requires an argument -- $OPTARG" ;;

	*) 	usage "invalid option: $OPTARG" ;;

	esac

done

#

# If configured to use terminal colors, record the escape sequences here.

#

if [[ $opt_c == "true" && -t 1 ]]; then

    TGREEN="$(tput setaf 2)"

    TRED="$(tput setaf 1)"

    TCLEAR="$(tput sgr0)"

fi

shift $((OPTIND-1))

[[ $# -eq 0 && $opt_a == "false" ]] && \

    usage "must specify \"-a\" or list of tests"

#

# Initialize paths and other environment variables.

#

export SRC=$(abspath $(dirname $0)/../..)

export PATH=$SRC/deps/ctf2json:$PATH

[[ -n $HOST ]] || export HOST=$(hostname)

#

# We create and set CATMPDIR as a place for the tests to store temporary files.

#

export CATMPDIR="/var/tmp/catest.$$_tmpfiles"

if [[ $opt_a = "true" ]]; then

	cat_tests=$(find $SRC/$cat_tstdir \

	    -name 'tst*.js' -o -name 'tst*.sh') || \

	    fail "failed to locate tests in $SRC/$cat_tstdir"

	cat_tests=$(sort <<< "$cat_tests")

else

	for t in $@; do

		[[ -f $t ]] || fail "cannot find test $t"

		cat_tests="$cat_tests $(abspath $t)"

	done

fi

mkdir -p "$opt_o/$cat_outbase"

cat_outdir=$(abspath $opt_o/$cat_outbase)

mkdir -p $CATMPDIR || fail "failed to create $CATMPDIR"

cat_ntests=$(echo $cat_tests | wc -w)

printf "Configuration:\n"

printf "    SRC:                         $SRC\n"

printf "    Output directory:            $cat_outdir\n"

printf "    Temp directory:              $CATMPDIR\n"

if [[ -n "$opt_t" ]]; then

    cat_tapfile=$(abspath $opt_t)

    printf "    TAP output:                  $cat_tapfile\n"

fi

printf "    Keep successful test output: $opt_k\n"

printf "    Found %d test(s) to run\n\n" $cat_ntests

#

# Validate parameters and finish setup.

#

[[ $cat_ntests -gt 0 ]] || fail "no tests found"

if [[ -n "$cat_tapfile" ]]; then

	echo "1..$(($cat_ntests))" > $cat_tapfile || \

	    fail "failed to emit TAP output"

fi

#

# Allow for environment-specific customizations.  These are optionally loaded

# by the catest_init.sh file sourced earlier.

#

if type catest_init > /dev/null 2>&1 && ! catest_init; then

	fail "catest_init failed"

fi

#

# Start the test run.

#

printf "===================================================\n\n"

for t in $cat_tests; do

	execute_test $t

	((cat_nrun++))

done

printf "\n===================================================\n\n"

printf "Results:\n"

printf "\tTests passed:\t%2d/%2d\n" $cat_npassed $cat_nrun

printf "\tTests failed:\t%2d/%2d\n" $cat_nfailed $cat_nrun

printf "\n===================================================\n"

if [[ $opt_k == "false" ]]; then

	echo "Cleaning up output from successful tests ... \c "

	rm -rf $cat_outdir/success.*

	rm -rf $CATMPDIR

	echo "done."

fi

exit $cat_nfailed