#
# common preliminary check procedures for QA scripts
#
# Copyright (c) 2014-2017 Red Hat.
# Copyright (c) 1997-2002 Silicon Graphics, Inc.  All Rights Reserved.
#

if [ ! -f localconfig ]
then
    ./mk.localconfig
fi

# need to know what version we're running
. ./localconfig

# if systemctl is present, then use this to start and stop services
#
if `which systemctl >/dev/null 2>&1`
then
    export PCPQA_SYSTEMD=yes
else
    export PCPQA_SYSTEMD=no
fi

# common routine for preventing a test from running on some platforms
# use: _notrun "Reason for not running this test is ..."
# 
_notrun()
{
    echo $@ >$here/$seq.notrun
    echo "$seq: [not run] `cat $here/$seq.notrun`"
    rm -f $here/$seq.pre-avc
    status=0
    exit $status
}

# common routine for failing a test in a standard way
# use: _fail "Reason for failing this test is ..."
#
_fail()
{
    echo FAIL: $@ 1>&2
    status=0
    exit $status
}

# calculate the size of a given fail, echo it on standard output
# 
_filesize()
{
    filename=$1

    if [ $PCP_PLATFORM = darwin ]
    then
	# Mac OS X format
	# stat(1) format
	# 234881026 5304024 -rwxr-xr-x 1 kenj kenj 0 2016 "May  4 14:00:42 2011" "Apr 27 20:14:16 2011" "Apr 27 20:14:16 2011" "Apr 27 20:14:16 2011" 4096 8 0 441
	stat $filename 2>&1 | $PCP_AWK_PROG '{ print $8 }'
    else
	# stat(1) format
	#   File: `441'
	#   Size: 2016      	Blocks: 8          IO Block: 4096   regular file
	# Device: 816h/2070d	Inode: 758237      Links: 1
	# Access: (0755/-rwxr-xr-x)  Uid: ( 1000/    kenj)   Gid: ( 1000/    kenj)
	# Access: 2011-05-09 06:52:58.000000000 +1000
	# Modify: 2011-02-27 14:42:30.000000000 +1100
	# Change: 2011-02-28 19:21:27.000000000 +1100
	stat $filename 2>&1 | $PCP_AWK_PROG '$1 == "Size:" { print $2 }'
    fi
}

# determine whether sufficient free space exists to run a test;
# passed in parameter is the size needed, in megabytes.
# 
_check_freespace()
{
    needspace=$1

    # Filesystem           1M-blocks Used Available Use% Mounted on
    # /dev/sda5                57349     24838     29621  46% /
    case "$PCP_PLATFORM"
    in
	darwin)
	    free=`df -m . | $PCP_AWK_PROG 'NR == 2 { print $4 }'`
	    ;;
	*)
	    free=`df -mP . | $PCP_AWK_PROG 'NR == 2 { print $4 }'`
	    ;;
    esac

    if [ -z "$free" ]
    then
	echo "Cannot determine free space (df -mP fails)"
    elif [ "$free" -lt "$needspace" ]
    then
	echo "Insufficient free space ($free MB, need $needspace MB)"
    fi
}

# save and restore configuration files in a way that labels which test
# was responsible in the event of a test abort, but also preserves the
# mode and ownership of the configuration file
#
_save_config()
{
    if [ -f "$1" ]
    then
	$sudo rm -f "$1.$seq"
	$sudo mv "$1" "$1.$seq"
	$sudo cp "$1.$seq" "$1"
    elif [ -d "$1" ]
    then
	$sudo rm -rf "$1.$seq"
	$sudo mv "$1" "$1.$seq"
	$sudo cp -r "$1.$seq" "$1"
    else
	echo "Botch: _save_config: $1 does not exist"
    fi
}

_restore_config()
{
    if [ -f "$1.$seq" ]
    then
	$sudo rm -f "$1"
	$sudo mv "$1.$seq" "$1"
    elif [ -d "$1.$seq" ]
    then
	$sudo rm -rf "$1"
	$sudo mv "$1.$seq" "$1"
    else
	echo "Warning: _restore_config: $1.$seq does not exist"
    fi
}

# systemd config changes
#
_stop_auto_restart()
{
    if [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$1.service ]
    then
	if grep '^Restart=always' $PCP_SYSTEMDUNIT_DIR/$1.service >/dev/null
	then
	    _save_config $PCP_SYSTEMDUNIT_DIR/$1.service
	    sed -e 's/Restart=always/Restart=no/' <$PCP_SYSTEMDUNIT_DIR/$1.service >$tmp.tmp
	    $sudo cp $tmp.tmp $PCP_SYSTEMDUNIT_DIR/$1.service
	    if `which systemctl >/dev/null 2>&1`
	    then
		$sudo systemctl daemon-reload
	    else
		$sudo kill -HUP 1
	    fi
	fi
    fi
}

_restore_auto_restart()
{
    if [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$1.service.$seq ]
    then
	_restore_config $PCP_SYSTEMDUNIT_DIR/$1.service
	if `which systemctl >/dev/null 2>&1`
	then
	    $sudo systemctl daemon-reload
	else
	    $sudo kill -HUP 1
	fi
    fi
}

# Wrapper for PCP daemon init scripts ... use systemctl if available,
# else run the PCP scripts directly
#
# Based on similar logic in the $PCP_RC_DIR/pcp script, but note this
# one has to do the $sudo and handle "pcp" as the special non-service
# case ...
#
# Usage: _service [-v] <service> [on|off|restart|...]
#
_service()
{
    if [ -n "$1" -a "$1" = "-v" ]
    then
	_verbose=true
	shift
    else
	_verbose=false
    fi
    _do_systemctl=false
    if [ "$PCPQA_SYSTEMD" = no ]
    then
	# don't ever use systemctl
	$_verbose && echo "_service: no systemctl, PCPQA_SYSTEMD=$PCPQA_SYSTEMD"
    else
	which systemctl >/dev/null 2>&1 && _do_systemctl=true
	if $_do_systemctl
	then
	    case "$1"
	    in
		pcp)
		    if [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/pmcd.service -a -f $PCP_SYSTEMDUNIT_DIR/pmlogger.service -a -f $PCP_RC_DIR/pcp ]
		    then
			:
		    else
			$_verbose && echo "_service: no systemctl, need pmcd service, pmlogger servce and pcp rc script"
			_do_systemctl=false
		    fi
		    ;;
		*)
		    if [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$1.service ]
		    then
			:
		    else
			$_verbose && echo "_service: no systemctl, need $1 service"
			_do_systemctl=false
		    fi
		    ;;
	    esac
	else
	    $_verbose && echo "_service: no systemctl executable"
	fi
    fi

    if $_do_systemctl
    then
	# smells like systemctl is the go ...
	#
	case "$1"
	in
	    pcp)
		# not a systemd service but a wrapper for the pmcd and pmlogger
		# services, so a little different ...
		#
		if [ "$2" = start -o "$2" = restart ]
		then
		    $_verbose && echo "_service: using systemctl $2 $pmcd.service then pmlogger.service"
		    $sudo systemctl reset-failed pmcd.service 2>>$here/$seq.full
		    $sudo systemctl $2 pmcd.service
		    $sudo systemctl reset-failed pmlogger.service 2>>$here/$seq.full
		    $sudo systemctl $2 pmlogger.service
		else
		    $_verbose && echo "_service: using systemctl $2 pmlogger.service then pmcd.service"
		    $sudo systemctl $2 pmlogger.service
		    $sudo systemctl $2 pmcd.service
		    if [ "$2" = stop ]
		    then
			# stop pmlogger_check ... it gets started as a
			# by-product of start pmlogger, but is not stopped
			# by stop pmlogger (there is a long and complicated
			# good reason for this)
			#
			$_verbose && echo "_service: also stop pmlogger_check"
			$sudo systemctl stop pmlogger_check.timer pmlogger_check.service
		    fi
		fi
		;;
	    *)
		$_verbose && echo "_service: using systemctl $2 $1.service"
		if [ "$2" = start -o "$2" = restart ]
		then
		    $sudo systemctl reset-failed $1.service 2>>$here/$seq.full
		fi
		$sudo systemctl $2 $1.service
		if [ "$1" = pmlogger -a "$2" = stop ]
		then
		    # stop pmlogger_check ... see note above
		    #
		    $_verbose && echo "_service: also stop pmlogger_check"
		    $sudo systemctl stop pmlogger_check.timer pmlogger_check.service
		fi
		;;
	esac
    elif [ -f $PCP_RC_DIR/$1 ]
    then
	$_verbose && echo "_service: using $PCP_RC_DIR/$1 $2"
	$sudo $PCP_RC_DIR/$1 $2
    else
	echo "_service: $1 is not a systemctl service nor a $PCP_RC_DIR script!"
	status=0
	exit $status
    fi
}

# check that a given agent (argument 1) is up and running;
# indicated by return status and agent warnings to stdout.
#
_check_agent() 
{
    agent=$1
    quiet=$2
    sts=0

    [ X"$quiet" = "X" ] && quiet=false

    pminfo -f $agent 2>&1 | \
        $PCP_AWK_PROG '
BEGIN           { value=0; metric=0; warn=0 }
NF==0           { next }
/ value /       { value++; next }
/^'$agent'/         { metric++
                  if ($0 !~ /:/) next
		}
		{ warn++ }
END             { if (warn) printf "%d warnings, ",warn
		  printf "%d metrics and %d values\n",metric,value
		}' > $tmp.fetch 2>&1

    pminfo -v $agent 2>&1 | LC_COLLATE=POSIX sort >$tmp.verify
    $quiet || cat $tmp.fetch
    if grep warnings $tmp.fetch > /dev/null 2>&1
    then
	# X warnings, Y metrics and Z values
	num_warn=`sed -n -e '/warnings/s/ .*//p' < $tmp.fetch`
	if [ "$num_warn" -gt 0 ]
	then
	    echo "Warnings when fetching $agent metrics:"
	    cat $tmp.verify
	    sts=1
	fi
    else
    	num_warn=0
    fi

    return $sts
}

#
# prepare and cleanup pmda routines work together to ensure
# pmcd+pmda state is left as it was at the start of a test.
#
_prepare_pmda()
{
    __agent=$1
    __names=$2

    iam=$__agent
    [ "X$__names" = "X" ] && __names=$iam
    done_clean=false
    install_on_cleanup=false
    pminfo $__names >/dev/null 2>&1 && install_on_cleanup=true
    echo "_prepare_pmda($__agent,$__names) install_on_cleanup=$install_on_cleanup" >>$here/$seq.full
    # copy the pmcd config file to restore state later.
    _save_config $PCP_PMCDCONF_PATH
    unset ROOT MAKEFLAGS
}

_cleanup_pmda()
{
    __agent=$1
    __iopts=$2

    if $done_clean
    then
	echo "_cleanup_pmda($__agent,$__iopts) called twice?" >>$here/$seq.full
    else
	_restore_config $PCP_PMCDCONF_PATH
	_service pcp restart | _filter_pcp_start
	_restore_auto_restart pmcd
	_wait_for_pmcd
	_wait_for_pmlogger
	if $install_on_cleanup
	then
	    [ "X$__iopts" = "X" ] && __iopts=/dev/null
	    ( cd $PCP_PMDAS_DIR/$__agent; $sudo ./Install <$__iopts >/dev/null 2>&1 )
	    echo "_cleanup_pmda($__agent,$__iopts) reinstall PMDA" >>$here/$seq.full
	else
	    ( cd $PCP_PMDAS_DIR/$__agent; $sudo ./Remove >/dev/null 2>&1 )
	    echo "_cleanup_pmda($__agent,$__iopts) remove PMDA" >>$here/$seq.full
	fi
	_wait_for_pmcd
	_wait_for_pmlogger
	done_clean=true
    fi
    $sudo rm -f $tmp.*
}

#
# create a Linux /proc/stat-alike file for a given number of CPUs
#
_make_proc_stat()
{
    __path=$1
    __ncpu=$2

    echo cpu 0 0 0 0 0 0 0 0 0 > $__path
    __cpu=0
    while [ $__cpu -lt $__ncpu ]
    do
	echo cpu$__cpu 0 0 0 0 0 0 0 0 0 >> $__path
	__cpu=`expr $__cpu + 1`
    done
}

_make_helptext()
{
    __pmda=$1
    __home=`pwd`

    if [ -f $PCP_PMDAS_DIR/$__pmda/help.dir ]
    then
	# ./Install already run, or QA stumbled past to run newhelp
	if [ -f $PCP_PMDAS_DIR/$__pmda/help -a $PCP_PMDAS_DIR/$__pmda/help -nt $PCP_PMDAS_DIR/$__pmda/help.dir ]
	then
	    :
	else
	    return 0
	fi
    fi

    cd $PCP_PMDAS_DIR/$__pmda
    $sudo $PCP_BINADM_DIR/newhelp -n root help >$tmp.out 2>&1
    if [ -f $PCP_PMDAS_DIR/$__pmda/help.dir ]
    then
	:
    else
	cat $tmp.out
	echo "Arrgh ... failed to create help.dir for $pmda PMDA"
	return 1
    fi
    cd $__home
    return 0
}

#
# Get all IPv6 connection strings for a hosts interfaces, excluding loopback
#
_host_to_ipv6addrs()
{
    # extract string from lines like:
    # inst [2 or "br0"] value "fe80::d217:c2ff:fea7:4ae9/64"
    #
    pminfo -f -h "$1" network.interface.ipv6_addr \
    | sed \
	-e '/^$/d' \
	-e '/^network/d' \
	-e '/"lo"/d' \
	-e 's/.* or "//' \
	-e 's/"] value "/ /' \
	-e 's/\/[0-9][0-9]*"//g' \
    | while read iface addr
    do
	case "$addr"
	in
	    fe80::*)
		echo "$addr%$iface"
		;;
	    *)
		echo $addr
		;;
	esac
    done
}

_host_to_ipaddr()
{
    perl -MSocket -e '
	my $packed_ip = gethostbyname("'$1'");
	if (defined $packed_ip) {
	    my $ip_address = inet_ntoa($packed_ip);
	    print $ip_address;
	}'
}

_ipaddr_to_host()
{
    perl -MSocket -e '
	my $iaddr = inet_aton("'$1'");
	if (defined $iaddr) {
	    my $name  = gethostbyaddr($iaddr, AF_INET);
	    print $name;
	}'
}

_host_to_fqdn()
{
    ans=''
    if which nslookup >/dev/null 2>&1
    then
	ans=`nslookup $1 2>&1 | sed -n -e '/^Name:[ 	][ 	]*/{
s///p
q
}'`
    elif which host >/dev/null 2>&1
    then
	ans=`host $1 2>&1 | sed -n -e '/ has address /{
s/ .*//p
q
}'`
    fi

    if [ -z "$ans" ]
    then
	# no working DNS ... just go with what we were given
	#
	ans=$1
    fi
    echo "$ans"
}

_ipv6_localhost()
{
    if [ ! -f /etc/hosts ]
    then
	echo >&2 "Arrgh ... cannot find /etc/hosts to determine IPv6 localhost name"
	return
    fi
    for sniff in localhost6 ip6-localhost ipv6-localhost
    do
	if sed -e '/^#/d' -e 's/$/ /' /etc/hosts | grep -q "[ 	]$sniff "
	then
	    echo $sniff
	    return
	fi
    done
    echo >&2 "Arrgh ... cannot determine IPv6 localhost name from /etc/hosts"
    return
}

_domain_name()
{
    if which domainname >/dev/null 2>&1
    then
	__domainname=`domainname`
    else
	__domainname=`hostname -d`
    fi
    [ -z "$__domainname" ] && __domainname=localdomain
    [ "$__domainname" = "(none)" ] && __domainname=localdomain
    [ "$__domainname" = "(null)" ] && __domainname=localdomain
    echo "$__domainname"
}

_machine_id()
{
    cat /etc/machine-id 2> /dev/null || echo localmachine
}

# Check metric ($1) availability from pmcd on host ($2, defaults to
# a local: pminfo connection) ... abort if no value(s) available
#
_check_metric()
{
    if pminfo -h ${2-local:} -f $1 2>&1 | grep " value " >/dev/null
    then
	:
    else
	echo "Check failed for metric \"$1\" at host \"${2-local:}\" ... is PMDA installed?"
	status=0
	exit $status
    fi
}

# Check for metric availability from local pmcd and _notrun if not
# available.
# Use this check for metrics that are requireed by a qa test but are
# not universally available, e.g. kernel metrics across different
# platforms.
#
_need_metric()
{
    numval=`pmprobe $1 | cut -f 2 -d ' '`
    case "$numval"
    in
	-*)
	    _notrun "metric \"$1\" not available"
	    ;;
	[0-9]*)
	    ;;
	*)
	    echo "_need_metric: unexpected output: `pmprobe $1`"
	    ;;
    esac
}

# wait_for_pmcd [maxdelay [host]]
#
_wait_for_pmcd()
{
    # 20 seconds default seems like a reasonable max time to get going
    #debug# set -x
    _can_wait=${1-20}
    if [ $# -eq 2 -a -n "$2" ]
    then
	_host="$2"
	_host_opt="-h $2"
    else
	_host=localhost
	_host_opt=''
    fi
    _i=0
    _dead=true
    #debug# ping -c 2 $_host
    #debug# pcp -h $_host
    #debug# pcp -h `hostname`
    rm -f $tmp._wait_for_pmcd.err $tmp._wait_for_pmcd.out
    while [ $_i -lt $_can_wait ]
    do
	echo "-- $_i --" >>$tmp._wait_for_pmcd.err
	echo "-- $_i --" >>$tmp._wait_for_pmcd.out
	#debug# pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd $_host_opt pmcd.numclients
	_sts=`pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd $_host_opt pmcd.numclients 2>>$tmp._wait_for_pmcd.err | tee -a $tmp._wait_for_pmcd.out | $PCP_AWK_PROG '{print $2}'`
	if [ "${_sts:-0}" -gt 0 ]
	then
	    # numval really > 0, we're done
	    #
	    _dead=false
	    break
	fi
	sleep 1
	_i=`expr $_i + 1`
    done
    if $_dead
    then
	date
	echo "Arrgghhh ... pmcd at $_host failed to start after $_can_wait seconds"
	echo "=== failing pmprobes stdout ==="
	cat $tmp._wait_for_pmcd.out
	echo "=== failing pmprobes stderr ==="
	cat $tmp._wait_for_pmcd.err
	echo "... and now"
	echo "+ pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd $_host_opt pmcd.numclients"
	pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd $_host_opt pmcd.numclients
	case $_host
	in
	    localhost|unix:|local:|`hostname`)
		# these are all local PMCD's
		#
		echo "=== pmcd.log ==="
		cat $PCP_LOG_DIR/pmcd/pmcd.log

		echo "likely looking processes ..."
		ps "$PCP_PS_ALL_FLAGS" | egrep "[p]m|[P]ID"
		;;
	    *)
		# assume a remote PMCD
		#
		ssh pcpqa@$_host "sh -c '. \$PCP_DIR/etc/pcp.env; echo; echo "=== pmcd.log ==="; [ -f \$PCP_LOG_DIR/pmcd/pmcd.log ] && cat \$PCP_LOG_DIR/pmcd/pmcd.log; [ -f \$PCP_LOG_DIR/pmcd.log ] && cat \$PCP_LOG_DIR/pmcd.log; echo; echo likely looking processes ...; ( ps \$PCP_PS_ALL_FLAGS | egrep \"[p]m|[P]ID\" )'" </dev/null
		;;
	esac
	status=0
	exit $status
    fi
}

# wait_for_pmcd_stop [maxdelay]
#
_wait_for_pmcd_stop()
{
    # 20 seconds default seems like a reasonble max time to get shutdown
    #debug# set -x
    _can_wait=${1-20}
    _i=0
    _stopped=false
    rm -f $tmp._wait_for_pmcd_stop.err $tmp._wait_for_pmcd_stop.out
    while [ $_i -lt $_can_wait ]
    do
	echo "-- $_i --" >>$tmp._wait_for_pmcd_stop.out.all
	echo "-- $_i --" >>$tmp._wait_for_pmcd_stop.err.all
	#debug# pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd pmcd.numclients
	pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd pmcd.numclients >$tmp._wait_for_pmcd_stop.out 2>$tmp._wait_for_pmcd_stop.err
	if grep 'pmprobe: Cannot connect to PMCD' $tmp._wait_for_pmcd_stop.err >/dev/null
	then
	    # we're done
	    #
	    _stopped=true
	    break
	fi
	cat $tmp._wait_for_pmcd_stop.out >>$tmp._wait_for_pmcd_stop.out.all
	cat $tmp._wait_for_pmcd_stop.err >>$tmp._wait_for_pmcd_stop.err.all
	sleep 1
	_i=`expr $_i + 1`
    done
    if $_stopped
    then
	:
    else
	date
	echo "Arrgghhh ... pmcd failed to stop after $_can_wait seconds"
	$PCP_PS_PROG $PCP_PS_ALL_FLAGS | egrep '[p]mcd|[P]ID'
	echo "=== successful pmprobes stdout ==="
	cat $tmp._wait_for_pmcd_stop.out.all
	echo "=== successful pmprobes stderr ==="
	cat $tmp._wait_for_pmcd_stop.err.all
	echo "... and now"
	echo "+ pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd pmcd.numclients"
	pmprobe -Dpdu,context -n $PCP_VAR_DIR/pmns/root_pmcd pmcd.numclients
	echo "=== pmcd.log ==="
	cat $PCP_LOG_DIR/pmcd/pmcd.log
	status=0
	exit $status
    fi
}

# wait_for_pmlogger [pid logfile [maxdelay]]
#
_wait_for_pmlogger()
{
    # 20 seconds default seems like a reasonable max time to get going
    _maxdelay=20

    case $#
    in
	0)
	    _pid="-P"
	    _sudo="$sudo -u pcp"
	    dir_hostname=`hostname || echo localhost`
	    _logfile="$PCP_ARCHIVE_DIR/$dir_hostname/pmlogger.log" 
            _iam="primary"
	    ;;
	2)
	    _pid=$1
	    _sudo=''
	    _logfile=$2
            _iam="pid=$_pid"
	    ;;
	3)
	    _pid=$1
	    _sudo=''
	    _logfile=$2
	    _maxdelay=$3
            _iam="pid=$_pid"
	    ;;
	*)
	    echo "_wait_for_pmlogger(): wrong number of arguments"
	    status=0
	    exit $status
	    ;;
    esac

    #debug# set -x
    _i=0
    _dead=true
    rm -f $tmp._wait_for_pmlogger.pmlc
    while [ $_i -lt $_maxdelay ]
    do
	ps $PCP_PS_ALL_FLAGS | egrep "([p]mlogger|[P]ID)" >>$tmp._wait_for_pmlogger.pmlc
	echo "pmlc $_pid" >>$tmp._wait_for_pmlogger.pmlc
	# May need sudo -u pcp here in case we're doing credentials checks
	# in pmlogger and this is the primary pmlogger (started by user
	# pcp) ... $_sudo contains the right secret sauce.
	#
	# For hard debugging, add -Dpdu,context,desperate to pmlc invocation
	# below.
	#
	if $_sudo pmlc $_pid </dev/null 2>&1 \
		| tee -a $tmp._wait_for_pmlogger.pmlc \
		| egrep "Connection refused|Transport endpoint is not connected" >/dev/null
	then
	    sleep 1
	    _i=`expr $_i + 1`
        else
	    # pmlogger socket has been set up ...
	    _dead=false
	    # give pmlogger a chance to detect that pmlc has gone away
	    # so the port is free
	    sleep 1
	    break
	fi
    done
    if $_dead
    then
	date
	echo "Arrgghhh ... pmlogger ($_iam) failed to start after $_maxdelay seconds"
	echo "at `date`."
	echo "pmlogger log ($_logfile) ..."
	cat $_logfile
	echo

	#  If pmlc could not connect to pmlogger, report details
	if [ -s $tmp._wait_for_pmlogger.pmlc ]
	then
	    echo "pmlc output ..."
	    cat $tmp._wait_for_pmlogger.pmlc
	    echo
	fi

	#  If pmlogger could not connect to PMCD, find which host it was
	#  connecting to, and get the pmcd.log file from that host.
	cat $_logfile | $PCP_AWK_PROG '/pmlogger: Cannot connect to PMCD on host/' \
	  | sed -e '	s/pmlogger: Cannot connect to PMCD on host "//g' \
	  -e '	s/": .*//g' >$tmp._wait_for_pmlogger.host
	if [ -s $tmp._wait_for_pmlogger.host ]
	then
	    _pmcdhost=`cat $tmp._wait_for_pmlogger.host`
	    echo "pmcd log ($_pmcdhost:$PCP_LOG_DIR/pmcd/pmcd.log) ..."
	    if [ -r /hosts/$_pmcdhost$PCP_LOG_DIR/pmcd/pmcd.log ]
	    then
	    	cat /hosts/$_pmcdhost$PCP_LOG_DIR/pmcd/pmcd.log
	    elif [ -r /hosts/$_pmcdhost$PCP_LOG_DIR/pmcd.log ]
	    then
	    	cat /hosts/$_pmcdhost$PCP_LOG_DIR/pmcd.log
	    else
	    	if [ "`hostname | sed -e 's/\..*//'`" != $_pmcdhost ]
	    	then
	    	    if scp -q $_pmcdhost:$PCP_LOG_DIR/pmcd/pmcd.log \
	    	      $tmp._wait_for_pmlogger.pmcdlog
	    	    then
	    	    	cat $tmp._wait_for_pmlogger.pmcdlog
	    	    elif scp -q $_pmcdhost:$PCP_LOG_DIR/pmcd.log \
	    	      $tmp._wait_for_pmlogger.pmcdlog
	    	    then
	    	    	cat $tmp._wait_for_pmlogger.pmcdlog
	    	    fi
	    	else
	    	    cat $PCP_LOG_DIR/pmcd/pmcd.log
	    	fi
	    fi
	fi
	base=`cat $_logfile | sed -n '/^Archive basename:[ 	]*/s///p'`
	if [ -n "$base" -a -f $base.0 ]
	then
	    echo "archive created ..."
	    pmdumplog -l $base.0
	    nres=`pmdumplog $base.0 | grep '^[0-9]' | wc -l | sed -e 's/ //g'`
	    echo "archive contains $nres records"
	else
	    echo "archive not created"
	fi
	echo
	echo "local pmlogger map ..."
	for map in $PCP_TMP_DIR/pmlogger/*
	do
	    if [ "`echo $map`" = "$PCP_TMP_DIR"'/pmlogger/*' ]
	    then
		echo "No files in $PCP_TMP_DIR/pmlogger !?"
	    else
		ls -l $map
		cat $map
	    fi
	done
	echo
	echo "Likely looking processes ..."
	ps $PCP_PS_ALL_FLAGS | egrep '([p]m)|([P]ID)'
	status=0
	exit $status
    fi
    rm -f $tmp._wait_for_pmlogger.*
}

# Start a pmlogger with appropriate privs that it can create
# portmap files in PCP_TMP_DIR (requires pcp or root account)
#
# Couple of side-effects to be aware of:
# Returns with $pid containing the backgrounded pmlogger PID;
# Creates the pmlogger archive, logfile, etc as root/pcp, not
# as the user running QA (so, tmp cleanup needs to use sudo).
#
_start_up_pmlogger()
{
    cat >$tmp.cmd <<End-of-File
#!/bin/sh
pmlogger $@ &
echo pid=\$!
End-of-File

    __user=root
    id pcp >/dev/null 2>&1 && __user=pcp

    $sudo -u $__user sh $tmp.cmd $@ >$tmp.pid
    eval `cat $tmp.pid`
}

# wait for a pmlogger process to end that is not our child
# (else we could just use shell wait command).  Instead we
# must keep tabs on the PID file and bail when its gone.
#
_wait_pmlogger_end()
{
    #debug# set -x
    if [ -z "$1" ]
    then
	echo "Usage botch: _wait_pmlogger_end() needs pid arg"
	status=0
	exit $status
    fi
    _pid=$1

    _i=0
    while true
    do
        if [ -f "$PCP_TMP_DIR/pmlogger/$_pid" ]
	then
	    _i=`expr $_i + 1`
	    if [ "$_i" -ge 100 ]
	    then
		echo "_wait_pmlogger_end: failed to see $PCP_TMP_DIR/pmlogger/$_pid removed after 100 iterations"
		#debug# set +x
		return
	    fi
	    pmsleep 0.1
	else
	    # now there is an ugly race condition at the end of pmlogger
	    # ... once the control file has been removed (as per the test
	    # above) the pmlogger log file is not complete until the atexit()
	    # code is run ... so hang around waiting for the process to
	    # go away
	    #
	    _i=0
	    while true
	    do
		if sudo kill -s 0 $_pid 2>/dev/null
		then
		    # process still exists
		    _i=`expr $_i + 1`
		    if [ "$_i" -ge 100 ]
		    then
			# may exist, but is defunct so we should treat this
			# as "ended" ...
			#
			$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
			| $PCP_AWK_PROG >$tmp.pmlogger.defunct '
$2 == '$_pid' && $0 ~ /defunct/	{ print $3 }'
			if [ -s "$tmp.pmlogger.defunct" ]
			then
			    echo "_wait_pmlogger_end: pid $_pid is defunct" >>$here/$seq.full
			    if which pstree >/dev/null 2>&1
			    then
				pstree -s -A -pua $_pid >>$here/$seq.full 2>&1
			    else
				$PCP_PS_PROG $PCP_PS_ALL_FLAGS -p $_pid `cat $tmp.pmlogger.defunct`
			    fi
			else
			    echo "_wait_pmlogger_end: failed to see pid $_pid exit after 100 iterations"
			fi
			#debug# set +x
			return
		    fi
		    pmsleep 0.1
		else
		    # process has really exited
		    #debug# set +x
		    return
		fi
	    done
	fi
    done
}

# Wait for pmcd to end
#
_wait_pmcd_end()
{
    #debug# set -x

    _pid=`cat $PCP_RUN_DIR/pmcd.pid 2>/dev/null`
    if [ -z "$_pid" ]
    then
	# no $PCP_RUN_DIR/pmcd.pid file, try ps(1) the hard way
	#
	_pid=`$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
	| sed -n -e '/\[p]mcd /{
s/^[^ 	]*//
s/^[ 	]*//
s/[ 	].*//
p
}'`
    fi
    if [ -z "$_pid" ]
    then
	#debug# set +x
	return
    fi

    _i=0
    while true
    do
	if sudo kill -s 0 $_pid 2>/dev/null
	then
	    # process still exists
	    _i=`expr $_i + 1`
	    if [ "$_i" -ge 100 ]
	    then
		echo "_wait_pmcd_end: failed to see pid $_pid exit after 100 iterations"
		#debug# set +x
		return
	    fi
	    pmsleep 0.1
	else
	    # process has really exited
	    #debug# set +x
	    return
	fi
    done
}

# Wait for at primary pmie to start ...
#
_wait_for_pmie()
{
    #debug# set -x

    _i=0
    while true
    do
	if [ ! -f $PCP_RUN_DIR/pmie.pid ]
	then
	    # process not started yet ...
	    _i=`expr $_i + 1`
	    if [ "$_i" -ge 100 ]
	    then
		echo "_wait_for_pmie: failed to see pmie start after 100 iterations"
		#debug# set +x
		return
	    fi
	    pmsleep 0.1
	else
	    # process has really started ...
	    #debug# set +x
	    return
	fi
    done
}

# Wait for primary pmie to end
#
_wait_pmie_end()
{
    #debug# set -x

    _pidlist=`cat $PCP_RUN_DIR/pmie.pid 2>/dev/null`
    if [ -z "$_pidlist" ]
    then
	# no $PCP_RUN_DIR/pmie.pid file, try ps(1) the hard way ...
	# Note, this will wait for ALL pmie's running on the local machine.
	#
	_pidlist=`$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
	| sed -n -e '/\[p]mie /{
s/^[^ 	]*//
s/^[ 	]*//
s/[ 	].*//
p
}'`
    fi
    if [ -z "$_pidlist" ]
    then
	#debug# set +x
	return
    fi

    _i=0
    while true
    do
	rm -f $tmp.some_pmie_running
	for _pid in $_pidlist
	do
	    if sudo kill -s 0 $_pid 2>/dev/null
	    then
		# process still exists
		touch $tmp.some.pmie
	    fi
	done
	if [ -f $tmp.some.pmie ]
	then
	    _i=`expr $_i + 1`
	    if [ "$_i" -ge 100 ]
	    then
		echo "_wait_pmie_end: failed to see pid(s) $_pidlist exit after 100 iterations"
		$PCP_PS_PROG $PCP_PS_ALL_FLAGS | egrep '[P]ID|[p]mie'
		#debug# set +x
		break
	    fi
	    pmsleep 0.1
	else
	    # all processes have exited
	    #debug# set +x
	    break
	fi
    done
}

# Wait for pmproxy to start up on given port
# Usage: _wait_for_pmproxy [port [logfile]]
#
_wait_for_pmproxy()
{
    case $#
    in
	0)
	    _port=44322
	    _logfile="$PCP_ARCHIVE_DIR/pmproxy/pmproxy.log" 
	    ;;
	1)
	    _port="$1"
	    _logfile="$PCP_ARCHIVE_DIR/pmproxy/pmproxy.log" 
	    ;;
	2)
	    _port="$1"
	    _logfile="$2"
	    ;;
	*)
	    echo "_wait_for_pmproxy(): wrong number of arguments"
	    status=0
	    exit $status
	    ;;
    esac

    # 20 seconds default seems like a reasonable max time to get going
    _maxdelay=20

    _count=0
    while ! $PCP_BINADM_DIR/telnet-probe -c localhost $_port
    do
	_count=`expr $_count + 1`
	if [ $_count -ge $_maxdelay ]
	then
	    echo "pmproxy failed to start on port $_port after $_maxdelay seconds"
	    echo "likely looking processes ..."
	    $PCP_PS_PROG $PCP_PS_ALL_FLAGS | egrep '[P]ID|[p]mproxy'
	    if [ -f "$_logfile" ]
	    then
		echo "pmproxy logfile ($_logfile) ..."
		cat "$_logfile"
	    else
		echo "pmproxy logfile ($_logfile) not created"
	    fi
	    status=0
	    exit $status
	fi
	sleep 1
    done
}

# ensure primary logger allows pmlc state changes
#
_writable_primary_logger()
{
    _configfile="$PCP_VAR_DIR/config/pmlogger/config.default"

    echo "_open_primary_logger checking config file: $_configfile" >>$here/$seq.full

    # first, move aside the default configuration file
    _save_config "$_configfile"

    # next, create a new default configuration file
    PMLOGCONF="$PCP_BINADM_DIR/pmlogconf"
    test -f "$_configfile" || \
	$sudo $PMLOGCONF -q -h localhost "$_configfile" >>$here/$seq.full 2>&1

    # finally, open it up to local access from pmlc
    sed <"$_configfile" >$tmp.sed-i \
	-e 's/^allow localhost.*$/allow localhost : all;/g' \
	-e 's/^allow local:.*$/allow local:* : all;/g'
    $sudo cp $tmp.sed-i "$_configfile"
    rm -f $tmp.sed-i
}

# restore primary logger pmlc access restrictions
#
_restore_primary_logger()
{
    _configfile="$PCP_VAR_DIR/config/pmlogger/config.default"

    # if there was an initial configuration, put it back
    if test -f "$_configfile.$seq"
    then
	_restore_config "$_configfile"
    else
	# otherwise, close out access from local pmlc once more
	if test -f "$_configfile"
	then
	    sed <"$_configfile" >$tmp.sed-i \
		-e 's/^allow localhost.*$/allow localhost : enquire;/g' \
		-e 's/^allow local:.*$/allow local:* : enquire;/g'
	    $sudo cp $tmp.sed-i "$_configfile"
	    rm -f $tmp.sed-i
	fi
    fi
}

# restore pmlogger control file(s) to ensure only primary logger
# would be started (as in a default install) - i.e. remove farm.
#
_restore_pmlogger_control()
{
    for _file in `find $PCP_ETC_DIR/pcp/pmlogger -type f`
    do
	$PCP_AWK_PROG '/^\$/ || /^#/ || (/PCP_ARCHIVE_DIR/ && $2 == "y") {print}' \
		$_file >$tmp.new
	$sudo mv $tmp.new $_file
    done
}

# purify support
#
# typical usage:
#
# At the top before outputting $seq.out but after setting $seq ...
#   _check_purify prog
#
# Main code...
#   _setup_purify prog
#   _run_purify [arg1] [arg2]
#

# initial setup for running purify
# creates purified binary in $_purify_dir
#
_setup_purify()
{
    # Need to change these to match Purify setup locally, if you
    # have Purify!
    #
    LM_LICENSE_FILE=27000@snort.melbourne.sgi.com
    RSU_LICENSE_MAP=/usr/Rational/config/LICENSE_MAP
    export LM_LICENSE_FILE RSU_LICENSE_MAP

    rm -f $here/$seq.full
    _path=$1
    _prog=`echo $_path | sed -e 's#.*/##'`
    _pure_prog="$_prog.pure"
    _purify_dir=$tmp.purify

    rm -rf $_purify_dir
    mkdir $_purify_dir
    cp $_path $_purify_dir
    _here=`pwd`
    cd $_purify_dir

    cat >.purify<<EOF
suppress umr _write
suppress miu
EOF
    unset PURIFYOPTIONS PUREOPTIONS
    purify -log-file=stderr $_prog >out 2>&1
    if [ ! -x $_prog.pure ]
    then
	cat out
	echo "Hmm ... purify failed to create $_prog.pure"
	status=0
	exit $status
    fi
    cd $_here
}

_run_purify()
{
    _here=`pwd`
    cd $_purify_dir
    $_purify_dir/$_pure_prog "$@" > $tmp._purify.out 2>&1
    cat $tmp._purify.out >>$_here/$seq.full
    if grep -i expired $tmp._purify.out >/dev/null; then
	cat $tmp._purify.out
    else
	_filter_purify < $tmp._purify.out
    fi
    cd $_here
}

_filter_purify()
{
    $PCP_AWK_PROG '
state == 0 && /License successfully checked out/	{ state = 1; next }
state == 0 && /Purify checking enabled/			{ state = 1; next }
state == 1						{ print }' \
    | sed \
	-e 's/pid [0-9][0-9]*/pid PID/g' \
	-e "s;$_purify_dir;TMP;g" \
	-e '/reserved for Purify internal use/d' \
	-e 's/suppressed chunks/suppressed blocks/g' \
    | $PCP_AWK_PROG -v extra="$PURIFY_FILTER_EXTRA" '
/Purify instrumented/		{ skip = 0 }
/bytes leaked\./		{ print "..."; skip = 1; next }
skip == 1			{ next }
				{ print }
/Purify Heap Analysis/		{ print "..."; skip = 1 }
/Basic memory usage \(including Purify/		{ print "..."; skip = 1 }
extra != "" && /Current file descriptors/	{ print "..."; skip = 1 }' \
    | (if [ "$PURIFY_FILTER_EXTRA" ]
    then sed -e 's/in use: [0-9][0-9]*/in use: N/'
    else cat -
    fi)
}

_check_purify()
{
    [ $# -eq 1 ] || _notrun "_check_purify needs executable as argument"
    _path=$1
    which purify >/dev/null 2>&1
    [ $? -eq 0 ] || _notrun "No purify binary found"
}

_check_series()
{
    which pmseries >/dev/null 2>&1 || \
	_notrun "pmseries command line utility not installed"
    which redis-cli >/dev/null 2>&1 || \
	_notrun "Redis command line utility not installed"
    which redis-server >/dev/null 2>&1 || \
	_notrun "Redis server utility not installed"
    redis-cli ping >/dev/null
    [ $? -eq 0 ] || _notrun "Redis server not running locally"
    eval `redis-cli info server |
	grep redis_version: |
	sed -e 's/:/=/g' -e 's/\..*//g'`
    [ $redis_version -ge 5 ] || _notrun "Redis $redis_version too old"
}

_check_search()
{
    which pmsearch >/dev/null 2>&1 || \
	_notrun "pmsearch command line utility not installed"
    redis-cli module list | grep -q ^ft$ || \
	_notrun "redisearch module is not loaded in Redis server"
}

_find_redis_modules()
{
    # scan through a list of known locations for Redis modules
    if sudo test -d "$PCP_LIB_DIR/redis/modules"
    then
	echo "$PCP_LIB_DIR/redis/modules"
    else
	echo "$PCP_LIB32_DIR/redis/modules"
    fi
}

_check_display()
{
    # note: non-X systems (MacOSX, Windows) must pass here unchallenged
    if [ -z "$PCPQA_CLOSE_X_SERVER" ]
    then
	_notrun "\$PCPQA_CLOSE_X_SERVER not set in ./common.config"
	# NOTREACHED
    fi
    which xdpyinfo >/dev/null 2>&1
    if [ $? -eq 0 ]
    then
	DISPLAY=$PCPQA_CLOSE_X_SERVER xdpyinfo >/dev/null 2>&1 || \
	    _notrun "Failed sanity check on DISPLAY $PCPQA_CLOSE_X_SERVER"
    fi
    export DISPLAY=$PCPQA_CLOSE_X_SERVER

    # hackery for warning:
    # QStandardPaths: XDG_RUNTIME_DIR not set, defaulting to '/tmp/...'
    mkdir -p "$tmp/xdg-runtime"
    chmod 0700 "$tmp/xdg-runtime"
    export XDG_RUNTIME_DIR="$tmp/xdg-runtime"
}

_clean_display()
{
    test -d "$tmp/xdg-runtime" && $sudo rm -fr "$tmp/xdg-runtime"
}

# valgrind support
#
# typical usage:
#
# At the top before outputting $seq.out but after setting $seq ...
#   _check_valgrind
#
# Main code...
#   _run_valgrind app [arg1 [arg2] ... ]
# or
#   _run_helgrind [--sudo] app [arg1 [arg2] ... ]
#

_check_valgrind()
{
    which valgrind >/dev/null 2>&1
    [ $? -eq 0 ] || _notrun "No valgrind binary found"
    case `hostname -s`
    in
	vm10)
	    _notrun "valgrind broken on vm10"
	    ;;
    esac
}

# Note: because we may be avoiding leaks in system libraries
# via the suppressions, there is potential indeterminism for
# the following case:
# -All heap blocks were freed -- no leaks are possible
# +LEAK SUMMARY:
# +definitely lost: 0 bytes in 0 blocks
# +indirectly lost: 0 bytes in 0 blocks
#
# Note
#	last 6 sed expressions are for broken valgrind(1) on
#	Debian strech circa 2016
#	one before that is for broken valgrind(1) on Arch Linux
#	circa Nov 2019
# 
_filter_valgrind()
{
    noleak="LEAK SUMMARY:\ndefinitely lost: 0 bytes in 0 blocks\nindirectly lost: 0 bytes in 0 blocks"
    sed \
	-e 's/^==*[1-9][0-9]*==* *//' \
	-e '/^$/d' \
	-e '/^Copyright (/d' \
	-e '/^Using Valgrind-/d' \
	-e '/^Parent PID:/d' \
	-e '/^HEAP SUMMARY:/d' \
	-e '/^in use at exit:/d' \
	-e '/^total heap usage:/d' \
	-e '/^possibly lost:/d' \
	-e '/^still reachable:/d' \
	-e '/^suppressed:/d' \
	-e '/^Reachable blocks (those to which a pointer was found)/d' \
	-e '/^To see them, rerun with:/d' \
	-e '/^For lists of detected and suppressed errors,/d' \
	-e '/^For counts of detected and suppressed errors,/d' \
	-e '/^ERROR SUMMARY:/s/ (suppressed: [^)]*)/ .../' \
	-e "s/^All heap blocks were freed -- .*/$noleak/g" \
	-e '/warning: addVar: in range .* outside all rx mapped areas/d' \
	-e '/WARNING: Serious error when reading debug info/d' \
	-e '/When reading debug info from /d' \
	-e '/Ignoring non-Dwarf2\/3\/4 block in .debug_info/d' \
	-e '/Last block truncated in .debug_info; ignoring/d' \
	-e '/parse_CU_Header: is neither DWARF2 nor DWARF3 nor DWARF4/d' \
	-e '/DW_FORM_GNU_strp_alt used, but no alternate .debug_str/d' \
	-e '/^parse DIE/,/confused by the above DIE/d' \
    # end
}

_filter_helgrind()
{
    sed \
	-e 's/^==*[1-9][0-9]*==* *//' \
	-e '/^$/d' \
	-e '/^Parent PID:/d' \
	-e '/^Copyright (/d' \
	-e '/^Using Valgrind-/d' \
	-e '/^Command: /d' \
	-e '/^For lists of detected and suppressed errors,/d' \
	-e '/^For counts of detected and suppressed errors,/d' \
	-e '/^Use --history-level=approx or =none/d' \
	-e '/^the cost of reduced accuracy of conflicting-access/d' \
	-e '/^ERROR SUMMARY:/s/ (suppressed: [^)]*)/ .../' \
    # end
}

_run_valgrind()
{
    if [ x"$1" = "x--sudo" ]
    then
	_valgrind_prefix=$sudo
	shift
    else
	_valgrind_prefix=''
    fi
    # extract version number I.J.K ... ignore anything after that,
    # e.g. .SVN or .SVN-Debian for Debian-based distros
    #
    __version=`valgrind --version | sed -e 's/valgrind-//' -e 's/\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/'`
    if [ -f $here/valgrind-suppress-$__version ]
    then
	__extra="--suppressions=$here/valgrind-suppress-$__version"
	echo "Warning: using extra $__extra" >>$here/$seq.full
    else
	__extra=''
	echo "Warning: no extra suppressions found for valgrind version $__version" >>$here/$seq.full
    fi
    $_valgrind_prefix valgrind \
	--leak-check=full --read-var-info=yes --gen-suppressions=all \
	--suppressions=$here/valgrind-suppress $__extra \
	--log-file=$tmp._valgrind \
	    "$@" 2>$tmp._valgrind.err >$tmp._valgrind.out
    echo "=== std out ==="
    $sudo cat $tmp._valgrind.out
    echo "=== std err ==="
    $sudo cat $tmp._valgrind.err
    echo "=== valgrind report ===" >>$here/$seq.full
    $sudo cat $tmp._valgrind >>$here/$seq.full
    echo "=== filtered valgrind report ==="
    $sudo cat $tmp._valgrind | _filter_valgrind
}

# Alternate invocation, using valgrind-silence as an assertion.
# Produces no stdout/stderr from valgrind - except in case of errors!
# Uses same suppression logic as _run_valgrind.
#
# Usage:                    $_valgrind_clean_assert CMD ARGS ...
# just as you'd have used   CMD ARGS ...
# alone, or                 |filtered
# or                        & backgrounded
#
# It works even if valgrind is not available (expanding to nothing).
#
_valgrind_clean_assert=
if which valgrind >/dev/null 2>&1; then
    # Note:
    # 	$here and $seq may not be set here, and this is not a shell
    # 	procedure, so skip any echoing
    #
    __version=`valgrind --version | sed -e 's/valgrind-//'`
    if [ -f $here/valgrind-suppress-$__version ]
    then
	__extra="--suppressions=$here/valgrind-suppress-$__version"
    else
	__extra=''
    fi
    valgrind --help >$tmp.valgrind.help 2>&1
    grep -q .--vgdb= <$tmp.valgrind.help && __extra="$__extra --vgdb=no"
    valgrind --help 2>&1 | grep -q show-leak-kinds
    if grep -q .--show-leak-kinds= <$tmp.valgrind.help
    then
	__extra="$__extra --show-leak-kinds=definite"
    else
	if grep -q .--show-possibly-lost= <$tmp.valgrind.help
	then
	    __extra="$__extra --show-reachable=no --show-possibly-lost=no"
	else
	    __extra="$__extra --show-reachable=no"
	fi
    fi
    $sudo rm -f $tmp.valgrind.help
    _valgrind_clean_assert="valgrind -q \
	     --leak-check=full --read-var-info=yes --gen-suppressions=all \
	     --suppressions=$here/valgrind-suppress $__extra \
	     --log-fd=1"
fi

_run_helgrind()
{
    $sudo rm -f $tmp.valgrind.err $tmp.valgrind.out $tmp.valgrind.log
    touch $tmp.valgrind.err $tmp.valgrind.out
    if [ x"$1" = "x--sudo" ]
    then
	$sudo touch $tmp.valgrind.log
	_helgrind_prefix=$sudo
	shift
    else
	touch $tmp.valgrind.log
	_helgrind_prefix=''
    fi
    __version=`valgrind --version | sed -e 's/valgrind-//'`
    if [ -f $here/helgrind-suppress-$__version ]
    then
	__extra="--suppressions=$here/helgrind-suppress-$__version"
	echo "Warning: using extra $__extra" >>$here/$seq.full
    else
	__extra=''
	echo "Warning: no extra suppressions found for helgrind version $__version" >>$here/$seq.full
    fi

    $_helgrind_prefix valgrind \
	--tool=helgrind --gen-suppressions=all \
	--suppressions=$here/helgrind-suppress $__extra \
	--log-file=$tmp.valgrind.log \
	    "$@" 2>$tmp.valgrind.err >$tmp.valgrind.out
    echo "=== valgrind report ===" >>$here/$seq.full
    $sudo cat $tmp.valgrind.log >>$here/$seq.full
    echo "=== filtered valgrind report ==="
    $sudo cat $tmp.valgrind.log | _filter_helgrind
}

#
# Checks that given_value is in range of correct_value +/- tolerance.
# Tolerance can be an absolute value or a percentage of the correct value
# (see examples with tolerances below).
# Outputs suitable message to stdout if it's not in range.
#
# A verbose option, -v, may be used as the LAST argument
# 
# e.g. 
# foo: 0.0298 = 0.03 +/- 5%
# _within_tolerance "foo" 0.0298 0.03 5%  
# 
# foo: 0.0298 = 0.03 +/- 0.01
# _within_tolerance "foo" 0.0298 0.03 0.01
#
# foo: 0.0298 = 0.03 -0.01 +0.002
# _within_tolerance "foo" 0.0298 0.03 0.01 0.002
#
# foo: verbose output of 0.0298 = 0.03 +/- 5% 
# _within_tolerance "foo" 0.0298 0.03 5% -v 

_within_tolerance()
{
  _name=$1
  _given_val=$2
  _correct_val=$3
  _mintol=$4
  _maxtol=$_mintol
  _verbose=0
  _debug=false

  # maxtol arg is optional
  # verbose arg is optional
  if [ $# -ge 5 ]
  then 
     if [ "$5" = "-v" ]
     then
	_verbose=1
     else
        _maxtol=$5
     fi
  fi
  if [ $# -ge 6 ]
  then
     [ "$6" = "-v" ] && _verbose=1
  fi

  # find min with or without %
  _mintolerance=`echo $_mintol | sed -e 's/%//'` 
  if [ $_mintol = $_mintolerance ]
  then 
      _min=`echo "scale=5; $_correct_val-$_mintolerance" | bc`
  else
      _min=`echo "scale=5; $_correct_val-$_mintolerance*0.01*$_correct_val" | bc`
  fi

  # find max with or without %
  _maxtolerance=`echo $_maxtol | sed -e 's/%//'` 
  if [ $_maxtol = $_maxtolerance ]
  then 
      _max=`echo "scale=5; $_correct_val+$_maxtolerance" | bc`
  else
      _max=`echo "scale=5; $_correct_val+$_maxtolerance*0.01*$_correct_val" | bc`
  fi

  $_debug && echo "min = $_min"
  $_debug && echo "max = $_max"

  cat <<EOF > $tmp._bc.1
scale=5;
if ($_min <= $_given_val) 1;
if ($_min > $_given_val) 0; 
EOF

  cat <<EOF > $tmp._bc.2
scale=5;
if ($_given_val <= $_max) 1;
if ($_given_val > $_max) 0;
EOF

  _above_min=`bc < $tmp._bc.1`
  _below_max=`bc < $tmp._bc.2`

  _in_range=`expr $_above_min \& $_below_max` 

  # fix up min, max precision for output
  # can vary for 5.3, 6.2
  _min=`echo $_min | sed -e 's/0*$//'` # get rid of trailling zeroes
  _max=`echo $_max | sed -e 's/0*$//'` # get rid of trailling zeroes

  if [ $_in_range -eq 1 ] 
  then
	[ $_verbose -eq 1 ] && echo $_name is in range
	return 0
  else
	[ $_verbose -eq 1 ] && echo $_name has value of $_given_val
	[ $_verbose -eq 1 ] && echo $_name is NOT in range $_min .. $_max	
	return 1
  fi
}

# disable systemd timers and / or comment pmlogger_check and pmsnap entries in
# the crontab file (also cron.pmcheck and cron.pmsnap entries for backwards
# compatibility)
# Usage: _remove_job_scheduler cron_backup systemd_state sudo
#
# cron_backup - where to keep the old crontab file
# systemd_state - where to keep systemd timer state
# sudo - location of sudo
#
_remove_job_scheduler()
{
    rc_cron_backup=${1:-crontab}
    rc_systemd_state=${2:-systemd.state}
    rc_sudo=${3:-sudo}

    $rc_sudo rm -f $rc_cron_backup $rc_systemd_state

    if systemctl cat pmie_daily.timer >/dev/null 2>&1; then
	for i in pmie.service pmie_daily.timer pmie_check.timer pmlogger_daily.timer \
		pmlogger_daily_report.timer pmlogger_daily_report-poll.timer \
		pmlogger_daily-poll.timer pmlogger_check.timer ; do
	    $rc_sudo systemctl is-active "$i" > /dev/null || continue
	    $rc_sudo systemctl stop $i >/dev/null
	    echo "$i" >> $rc_systemd_state
	done
    fi

    which crontab >/dev/null 2>&1 || return	# systemd timers only

    if $rc_sudo crontab -l 2>/dev/null >$rc_cron_backup
    then
	# Need to cull lines like this, so we end up whith just the
	# real crontab entries (if any)
	#
	# DO NOT EDIT THIS FILE - edit the master and reinstall.
	# (/dev/null installed on Sun Feb  2 07:30:33 2020)
	# (Cronie version 4.2)
	sed <$rc_cron_backup >$tmp.__cron \
	    -e '/^# DO NOT EDIT THIS FILE/d' \
	    -e '/^# (.* installed on /d' \
	    -e '/^# (Cronie/d' \
	# end
	cp $tmp.__cron $rc_cron_backup
    else
	# error, remove the backup file so no changes are made
	rm -f $rc_cron_backup
    fi

    if [ -s $rc_cron_backup ]
    then
	$rc_sudo cat $rc_cron_backup \
	| sed \
	    -e 's/^[^#].*pmlogger_check/#&/' \
	    -e 's/^[^#].*pmsnap/#&/' \
	    -e 's/^[^#].*cron.pmcheck/#&/' \
	    -e 's/^[^#].*cron.pmsnap/#&/' \
	| $rc_sudo crontab > /dev/null 2>&1
    fi
}

# restore systemd timers and / or crontab back to original state
# Usage: _restore_job_scheduler cron_backup systemd_state sudo
#
# cron_backup - where to keep the old crontab file
# systemd_state - where to keep systemd timer state
# sudo - location of sudo
#
_restore_job_scheduler()
{
    rc_cron_backup=${1:-crontab}
    rc_systemd_state=${2:-systemd.state}
    rc_sudo=${3:-sudo}

    if [ -s $rc_systemd_state ]; then
	for i in $(cat $rc_systemd_state); do
	    $rc_sudo systemctl start $i >/dev/null
	done
	rm -f $rc_systemd_state
    fi

    if [ -s $rc_cron_backup ]
    then
	$rc_sudo rm -f rc_cron_out rc_cron_check rc_cron_diff
	if $rc_sudo crontab $rc_cron_backup >rc_cron_out 2>&1
	then
	    # check everything is OK
	    #
	    $rc_sudo crontab -l >rc_cron_check
	    sed -e '/^#/d' $rc_cron_backup >$rc_cron_backup.clean
	    sed -e '/^#/d' rc_cron_check >rc_cron_check.clean
	    if diff -u $rc_cron_backup.clean rc_cron_check.clean >rc_cron_diff 2>&1
	    then
		:
	    else
		echo "_restore_cron: Warning: could not restore crontab to original state"
		echo "               Unexpected differences ..."
		diff -u $rc_cron_backup rc_cron_check
	    fi
	    $rc_sudo rm -f rc_cron_check rc_cron_check.clean rc_cron_diff
	else
	    echo "_restore_cron: Warning: could not restore crontab to original state"
	    echo "               crontab(1) failed ..."
	    cat rc_cron_out
	fi
	$rc_sudo rm -f rc_cron_out rc_cron_check rc_cron_diff
    fi
}

# running QA within a container can mean no crontab setup;
# _check_job_scheduler() is a convenience routine checking the external
# dependencies of _remove_job_scheduler() and _restore_job_scheduler()
#
_check_job_scheduler()
{
    systemctl cat pmie_daily.timer >/dev/null 2>&1 && return
    which crontab >/dev/null 2>&1 || _notrun "No crontab binary found"
}

# get offset into an archive relative to the first pmResult
# past the preamble
#
# Usage: _arch_start archive [offset]
#
_arch_start()
{
    pmdumplog -z $1 \
    | $PCP_AWK_PROG '
/^[0-9][0-9]:[0-9][0-9]:/	{ if ($3 ~ /pmcd.pmlogger.host/) next
				  split($1, t, ":")
				  t[3] += '"${2-0}"'
				  while (t[3] < 0) {
				    t[3] += 60
				    t[2]--
				  }
				  while (t[3] > 60) {
				    t[3] -= 60
				    t[2]++
				  }
				  while (t[2] < 0) {
				    t[2] += 60
				    t[1]--
				  }
				  while (t[2] > 60) {
				    t[2] -= 60
				    t[1]++
				  }
				  while (t[1] < 0)
				    t[1] += 24
				  while (t[1] > 23)
				    t[1] -= 24
				  printf "@%02d:%02d:%06.3f",t[1],t[2],t[3]
				  exit
				}'
}

# get an unused ipc port ... returned on std out, empty for failure
# Usage: _get_port tcp|udp low_port high_port
#
_get_port()
{
    [ $# -ne 3 ] && return
    __proto=$1
    __port=$2
    while [ $__port -le "$3" ]
    do
	if $sudo fuser $__port/$__proto >/dev/null 2>&1
	then
	    :
	else
	    echo $__port
	    return
	fi
	__port=`expr $__port + 1`
    done
}

# get host endianness ("le" or "be")
_get_endian()
{
    check=`echo a | od -x 2>&1 | sed -e 's/^0[^ ]* *//' -e 's/ //g' -e '/^$/d'`
    case "$check"
    in
    0a61) echo le ;;
    610a) echo be ;;
    *)
	echo "Arrgh ... od -x returned something odd ($check)"
	echo a | od -x
	status=0
	exit $status
	;;
    esac
}

# get host word size ("32" or "64")
_get_word_size()
{
    machine=`uname -m`
    case "$machine"
    in
    i?86|athlon|ppc)
	size=32
	# but, apps are 64-bit on my Mac OS X
	[ $PCP_PLATFORM = darwin ] && size=64
	;;
    x86_64|ia64|ppc64|ppc64le|s390x|aarch64)
	size=64
	;;
    *)
	echo "uname gave machine type with unknown word size ($machine)"
	status=0
	exit $status
	;;
    esac
    echo $size
}

# _all_hostnames host - generate all hostnames (or IP addresses) for this host,
#			that map to some network interface, excluding loopback
#
_all_hostnames()
{
    touch $tmp._addr
    ssh pcpqa@$1 </dev/null netstat -in 2>/dev/null >$tmp._tmp
    if grep 'Network.*Address' $tmp._tmp >/dev/null
    then
	# This is the IRIX version of netstat -in, get IP addr from the
	# Address field
	# 
	# Name   Mtu   Network         Address            Ipkts Ierrs ...
	# ef0    1500  134.14.55.128   134.14.55.149  712168207    10 ...
	#                              134.14.55.159      
	#                              134.14.55.147      
	# ef2*   1500  none            none                   0     0 ...
	# lo0    32992 127             127.0.0.1       23628402     0 ...
	#
	$PCP_AWK_PROG <$tmp._tmp >$tmp._addr '
/^lo/									{ next }
NF >= 4 && $4 ~ /^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/ 	{ print $4 }
NF == 1 && $1 ~ /^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/	{ print $1 }
END									{ print "End-of-List" }'
    else
	ssh pcpqa@$1 </dev/null /sbin/ifconfig 2>/dev/null >$tmp._tmp
	if grep 'UP.*RUNNING' $tmp._tmp >/dev/null
	then
	    # This is the Linux version of ifconfig, get IP addr from the
	    # inet addr: line
	    #
	    # eth0      Link encap:Ethernet  HWaddr 00:90:27:98:EE:A8  
	    #	        inet addr:134.14.55.176  Bcast:134.14.55.255  ...
	    #	        UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
	    #	        ...
	    #
	    # lo        Link encap:Local Loopback  
	    #	        inet addr:127.0.0.1  Mask:255.0.0.0
	    #	        UP LOOPBACK RUNNING  MTU:16436  Metric:1
	    #	        ...
	    #
	    # Note addr: tag is not present in some ifconfig output
	    #
	    $PCP_AWK_PROG <$tmp._tmp '
/^lo/						{ skip=1; next }
skip == 1 && NF > 0				{ next }
skip == 1 					{ skip = 0 }
$1 == "inet" && $2 ~ /addr:/			{ print $2 }
$1 == "inet" && $2 ~ /^[0-9]/			{ print $2 }
END						{ print "End-of-List" }' \
	    | sed -e 's/addr://' >$tmp._addr
	else
	    # Nothing we can do really, as there is no way of passing
	    # an error back from here, other than returning an empty
	    # list
	    return
	fi
    fi
    cat $tmp._addr \
    | while read __ip
    do
	if [ "$__ip" = "End-of-List" ]
	then
	    echo
	    break
	fi
	# check that ip addr is reachable
	if ping -c 1 $__ip >/dev/null 2>&1
	then
	    __host=`_ipaddr_to_host $__ip`
	    if [ ! -z "$__host" ]
	    then
		$PCP_ECHO_PROG $PCP_ECHO_N ",$__host""$PCP_ECHO_C"
	    else
		$PCP_ECHO_PROG $PCP_ECHO_N ",$__ip""$PCP_ECHO_C"
	    fi
	fi
    done \
    | sed -e 's/^,//'
}

# _all_ipaddrs - generate all IP addresses for this host,
#			that map to some network interface, excluding
#			loopback, slip, ppp
#
# See _all_hostnames() above for comments on the method used.
#
_all_ipaddrs()
{
    touch $tmp._addr
    if [ "$1" = "localhost" ]
    then 
	netstat -in 2>/dev/null >$tmp._tmp
    else
	ssh pcpqa@$1 </dev/null netstat -in 2>/dev/null >$tmp._tmp
    fi
    if grep 'Network.*Address' $tmp._tmp >/dev/null
    then
	# this is the IRIX version of netstat -in, get IP addr from the
	# Address field
	#
	$PCP_AWK_PROG <$tmp._tmp >$tmp._addr '
/^lo/ || /^sl/ || /^pp/							{ next }
NF >= 4 && $4 ~ /^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/ 	{ print $4 }
NF == 1 && $1 ~ /^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/	{ print $1 }
END									{ print "End-of-List" }'
    else
        if [ "$1" = "localhost" ]
	then 
	    /sbin/ifconfig 2>/dev/null >$tmp._tmp
	else
	    ssh pcpqa@$1 </dev/null /sbin/ifconfig 2>/dev/null >$tmp._tmp
	fi
	if grep 'UP.*RUNNING' $tmp._tmp >/dev/null
	then
	    # This is the Linux version of ifconfig, get IP addr from the
	    # inet addr: line
	    #
# ppp0      Link encap:Point-to-Point Protocol
#           inet addr:134.14.52.219  P-t-P:134.14.52.189  Mask:255.255.255.255
#           UP POINTOPOINT RUNNING NOARP MULTICAST  MTU:1522  Metric:1
#           RX packets:50119 errors:0 dropped:0 overruns:0 frame:0
#           TX packets:47474 errors:0 dropped:0 overruns:0 carrier:0
#           collisions:0 txqueuelen:3
#           RX bytes:7017171 (6.6 Mb)  TX bytes:3952015 (3.7 Mb)
	    #
	    # Note addr: tag is not present in some ifconfig output,
	    #      AND UP comes first!
	    #
	    $PCP_AWK_PROG <$tmp._tmp '
/^[a-z]/					{ loopback = 0 }
/^lo/						{ loopback = 1; next }
$1 == "inet" && $2 ~ /addr:/			{ save = $2; next }
$1 == "inet" && $2 ~ /^[0-9]/			{ save = $2; next }
$1 == "TX"					{ if (loopback == 0 && save != 0)
						    print save
						  save = 0
						}
END						{ print "End-of-List" }' \
	    | sed -e 's/addr://' >$tmp._addr
	else
	    # Nothing we can do really, as there is no way of passing
	    # an error back from here, other than returning an empty
	    # list
	    return
	fi
    fi
    cat $tmp._addr \
    | while read __ip
    do
	if [ "$__ip" = "End-of-List" ]
	then
	    echo
	    break
	fi
	$PCP_ECHO_PROG $PCP_ECHO_N ",$__ip""$PCP_ECHO_C"
    done \
    | sed -e 's/^,//'
}

# fqdn for localhost
# 
_get_fqdn()
{
    _host_to_fqdn `hostname`
}

# Distro-specific filtering for init, rc scripts, chkconfig, et al
#
_filter_init_distro()
{
    if [ -f /etc/mandriva-release ]
    then
	# looks like this is a Mandriva bug ... see
	# http://mandriva.598463.n5.nabble.com/Bug-24409-initscripts-New-netfs-provides-local-fs-scripts-can-t-be-turned-off-td869820.html
	#
	sed \
	    -e '/Warning: netfs is needed by pcp in runlevel/d'
    else
	cat
    fi
}

# deal with chkconfig et al
# assumes $sudo is set correctly
# try very hard to _not_ emit messages unless serious errors encountered
#
_change_config()
{
    if [ $PCP_PLATFORM = linux ]
    then
	pat=$1
	[ "$1" = "verbose" ] && pat=""

	_have_systemctl=false
	which systemctl >/dev/null 2>&1 && _have_systemctl=true
	if [ -z "$pat" ]
	then
	    _have_service=true
	elif [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$pat.service ]
	then
	    _have_service=true
	else
	    _have_service=false
	fi
	if $_have_systemctl && $_have_service
	then
	    # Run with systemd whenever it is available now
	    #
	    case "$2"
	    in
		on)	act=enable ;;
		off)	act=disable ;;
		*)
		    echo "_change_config: Error: \$2=$2 not \"on\" or \"off\" as expected"
		    status=0
		    exit $status
		    ;;
	    esac
	    if [ -n "$pat" ]
	    then
		$sudo systemctl $act $pat.service >$tmp._tmp 2>$tmp._err
		[ $? -eq 0 ] || cat $tmp._tmp
	    fi
	elif which chkconfig >/dev/null 2>&1
	then
	    # Try the older RedHat and SuSE way ..
	    #
	    [ -n "$pat" ] && $sudo chkconfig $pat $2 2>&1 \
	    | sed \
		-e '/^insserv: warning: current start runlevel(s) .* of script .pcp./d' \
		-e '/^insserv: Service .* is missed in the runlevels /d' \
	    # end
	elif [ -x /usr/sbin/sysv-rc-conf ]
	then
	    # Try the Debian and Ubuntu way ..
	    #
	    [ -n "$pat" ] && $sudo /usr/sbin/sysv-rc-conf $pat $2
	elif which rc-update >/dev/null 2>&1
	then
	    # Try the Gentoo way ..
	    #
	    if [ -n "$pat" ]
	    then
		case $2
		in
		    on)		act=add ;;
		    off)	act=delete ;;
		    *)
			echo "_change_config: Error: \$2=$2 not \"on\" or \"off\" as expected"
			status=0
			exit $status
			;;
		esac
		$sudo rc-update $act $pat default >$tmp._tmp 2>&1
		[ $? -eq 0 ] || cat $tmp._tmp
	    fi
	elif [ -f /etc/slackware-version ]
	then
	    # Slackware uses BSD-style init, so no control
	    :
	else
	    # I have no clue!
	    #
	    echo "_change_config: Error: cannot change config \"$1 $2\""
	    status=0
	    exit $status
	fi
    elif [ $PCP_PLATFORM = solaris ]
    then
	# Try the Solaris way ..
	#
	if which svcadm >/dev/null 2>&1
	then
	    case $1 
	    in
		pmcd)       pat=pmcd ;;
		pmlogger)   pat=pmlogger ;;
		verbose)    pat="" ;;
		*)	    pat=$1 ;;
	    esac
	    if [ -n "$pat" ]
	    then
		if [ "$2" = on ]
		then
		    state=`svcs -l svc:/application/pcp/$pat | sed -n '/^state[ 	]/s///p'`
		    [ -n "$state" -a "$state" != "online" ] \
			&& $sudo svcadm clear svc:/application/pcp/$pat
		    $sudo svcadm enable svc:/application/pcp/$pat
		elif [ "$2" = off ]
		then
		    $sudo svcadm disable svc:/application/pcp/$pat
		else
		    echo "_change_config: Error: \$2=$2 not \"on\" or \"off\" as expected"
		    status=0
		    exit $status
		fi
	    fi
	else
	    echo "_change_config: Error: cannot find svcs for Solaris"
	    status=0
	    exit $status
	fi
    elif [ $PCP_PLATFORM = darwin ]
    then
	case $1 
	in
	    pmcd)	pat=PMCD ;;
	    pmlogger)	pat=PMLOGGER ;;
	    pmie)	pat=PMIE ;;
	    pmproxy)	pat=PMPROXY ;;
	    verbose)	pat="" ;;
	    *)		pat=$1 ;;
	esac
	if [ -n "$pat" ]
	then
	    state=`sed -n -e "/^$pat=/{"'
s/.*=//
s/-//g
p
}' /etc/hostconfig`
	    if [ -z "$state" ]
	    then
		echo "_change_config: Error: No $pat control line in /etc/hostconfig"
		echo "You need to add a $pat=-YES- line to this file"
		status=0
		exit $status
	    fi
	    if [ "$2" = "on" ]
	    then
		req_state=YES
	    elif [ "$2" = "off" ]
	    then
		req_state=NO
	    else
		echo "_change_config: Error: bad state ($2) should be on or off"
		status=0
		exit $status
	    fi
	    if [ "$state" != "$req_state" ]
	    then
		sed </etc/hostconfig >$tmp._state \
		    -e "/^$pat=/s/-.*/-$req_state-/"
		sudo cp $tmp._state /etc/hostconfig
	    fi
	fi
    elif [ $PCP_PLATFORM = freebsd ]
    then
	# no control for FreeBSD
	:
    elif [ $PCP_PLATFORM = netbsd ]
    then
	# no control for NetBSD
	:
    elif [ $PCP_PLATFORM = openbsd ]
    then
	# no control for OpenBSD
	:
    else
	# I have no idea what to do for this platform!
	#
	echo "_change_config: Error: cannot \"$1 $2\" for $PCP_PLATFORM"
	status=0
	exit $status
    fi 2>&1 \
    | _filter_init_distro
}

_get_config()
{
    if [ "$PCP_PLATFORM" = linux ]
    then
	case $1 
	in
	    pmlogger|pmcd) pat=$1 ;;
	    verbose) pat="" ;;
	    *)	pat=$1 ;;
	esac

	_have_systemctl=false
	which systemctl >/dev/null 2>&1 && _have_systemctl=true
	if [ -z "$pat" ]
	then
	    _have_service=true
	elif [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$pat.service ]
	then
	    _have_service=true
	else
	    _have_service=false
	fi
	if $_have_systemctl && $_have_service
	then
	    # Run with systemd whenever it is available now
	    #
	    if [ -z "$pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		if systemctl -q is-enabled $pat.service >/dev/null 2>&1
		then
		    echo on
		else
		    echo off
		fi
	    fi
	elif which chkconfig >/dev/null 2>&1
	then
	    # Try the older RedHat and SuSE way ..
	    #
	    if [ -z "$pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		if chkconfig $pat >$tmp.__tmp 2>&1
		then
		    # success from chkconfig is only useful if no output
		    # was generated ... in the latter case, grep the output
		    # for hints (this is for SuSE SLES9 in particular)
		    #
		    if [ -s $tmp.__tmp ]
		    then
			if grep ' on$' $tmp.__tmp >/dev/null
			then
			    echo on
			elif grep ' off$' $tmp.__tmp >/dev/null
			then
			    echo off
			else
			    echo off
			fi
		    else
			echo on
		    fi
		else
		    echo off
		fi
	    fi
	elif [ -x /usr/sbin/sysv-rc-conf ]
	then
	    # Try the Debian and Ubuntu way ..
	    #
	    if [ -z "$pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		if $sudo /usr/sbin/sysv-rc-conf $pat
		then
		    echo on
		else
		    echo off
		fi
	    fi
	elif which rc-update >/dev/null 2>&1
	then
	    # Try the Gentoo way ..
	    #
	    if [ -z "$pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		if rc-update show default | grep $pat >/dev/null
		then
		    echo on
		else
		    echo off
		fi
	    fi
	elif [ -f /etc/slackware-version ]
	then
	    # Slackware uses BSD-style init, so no control
	    :
	else
	    echo "_change_config: Error: don't know how to change config for Linux"
	    status=0
	    exit $status
	fi
    elif [ $PCP_PLATFORM = solaris ]
    then
	if which svcs >/dev/null 2>&1
	then
	    case $1 
	    in
		pmlogger|pmcd) pat=$1 ;;
		verbose) pat="" ;;
		*) pat=$1 ;;
	    esac
	    if [ -z "$pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		state=`svcs -H svc:/application/pcp/$pat | sed -e 's/[ 	].*//'`
		case "$state"
		in
		    online|maintenance)
			echo on
			;;
		    offline|disabled)
			echo off
			;;
		    *)
			echo "_change_config: Error: smf ($state) from svcs not expected"
			status=0
			exit $status
			;;
		esac
	    fi
	else
	    echo "_change_config: Error: cannot find svcs for Solaris"
	    status=0
	    exit $status
	fi
    elif [ $PCP_PLATFORM = darwin ]
    then
	case $1 
	in
	    pmcd)	pat=PMCD ;;
	    pmlogger)	pat=PMLOGGER ;;
	    pmie)	pat=PMIE ;;
	    verbose)	pat="" ;;
	    *)		pat=$1 ;;
	esac
	if [ -n "$pat" ]
	then
	    state=`sed -n -e "/^$pat=/{"'
s/.*=//
s/-//g
p
}' /etc/hostconfig`
	    if [ -z "$state" ]
	    then
		echo "_change_config: Error: No $pat control line in /etc/hostconfig" >&2
		echo "You need to add a $pat=-YES- line to this file" >&2
		status=0
		exit $status
	    fi
	    if [ "$state" = "YES" ]
	    then
		echo on
	    elif [ "$state" = "NO" ]
	    then
		echo off
	    else
		echo "_change_config: Error: bad state ($state) should be YES or NO" >&2
		status=0
		exit $status
	    fi
	fi
    elif [ $PCP_PLATFORM = freebsd ]
    then
	# no control for FreeBSD
	:
    elif [ $PCP_PLATFORM = netbsd ]
    then
	# no control for NetBSD
	:
    elif [ $PCP_PLATFORM = openbsd ]
    then
	# no control for OpenBSD
	:
    else
	echo "_change_config: Error: cannot \"$1 $2\" for $PCP_PLATFORM"
    fi
}

# This used to disable all system pmloggers running via the control
# file(s), but that was problematic and so now replaces the control
# file with a simple one that starts a primary pmlogger with /dev/null
# as the configuration file so pmlogger makes not requests to pmcd.
# 
# See _restore_loggers() below for the logic to re-instate the original
# control file(s).
#
_disable_loggers()
{
    [ -z "$PCP_PMLOGGERCONTROL_PATH" ] && \
		PCP_PMLOGGERCONTROL_PATH="$PCP_VAR_DIR/config/pmlogger/control"
    if [ -f $PCP_PMLOGGERCONTROL_PATH.$seq ]
    then
	echo "_disable_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.$seq already exists"
	date
	ls -l $PCP_PMLOGGERCONTROL_PATH.$seq
	status=0
	exit $status
    fi
    if [ -f $PCP_PMLOGGERCONTROL_PATH ]
    then
	$sudo mv $PCP_PMLOGGERCONTROL_PATH $PCP_PMLOGGERCONTROL_PATH.$seq
    else
	echo "_disable_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH missing"
    fi
    if [ -d $PCP_PMLOGGERCONTROL_PATH.$seq.d ]
    then
	echo "_disable_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.$seq.d already exists"
	date
	ls -ld $PCP_PMLOGGERCONTROL_PATH.$seq.d
	status=0
	exit $status
    fi
    if [ -d $PCP_PMLOGGERCONTROL_PATH.d ]
    then
	$sudo mv $PCP_PMLOGGERCONTROL_PATH.d $PCP_PMLOGGERCONTROL_PATH.$seq.d
    else
	echo "_disable_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.d missing"
    fi
    cat <<End-of-File >$tmp._tmp
# Installed by PCP QA test $seq on `date`
# The goal here is to have a controlled primary logger that does
# not make requests to pmcd!
\$version=1.1
LOCALHOSTNAME y n PCP_ARCHIVE_DIR/LOCALHOSTNAME -c /dev/null
End-of-File
    $sudo cp $tmp._tmp $PCP_PMLOGGERCONTROL_PATH
}

_restore_loggers()
{
    [ -z "$PCP_PMLOGGERCONTROL_PATH" ] && \
		PCP_PMLOGGERCONTROL_PATH="$PCP_VAR_DIR/config/pmlogger/control"
    if [ -d $PCP_PMLOGGERCONTROL_PATH.$seq.d ]
    then
	$sudo rm -fr $PCP_PMLOGGERCONTROL_PATH.d
	$sudo mv $PCP_PMLOGGERCONTROL_PATH.$seq.d $PCP_PMLOGGERCONTROL_PATH.d
    else
	echo "_restore_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.$seq.d missing"
    fi
    if [ -f $PCP_PMLOGGERCONTROL_PATH.$seq ]
    then
	$sudo mv $PCP_PMLOGGERCONTROL_PATH.$seq $PCP_PMLOGGERCONTROL_PATH
    else
	echo "_restore_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.$seq missing"
    fi
}

# _check_core [dir]
# checks for core files in dir (defaults to .)
#
_check_core()
{
    if [ -z "$1" ]
    then
	dir=""
    else
	if [ -d $1 ]
	then
	    dir=$1/
	else
	    echo "_check_core: aaargh $1 is not a directory!"
	    return
	fi
    fi
    if [ "`echo ${dir}core*`" != "${dir}core*" ]
    then
	[ -z "$here" ] && here=/tmp
	[ -z "$seq" ] && seq=9999
	$PCP_ECHO_PROG $PCP_ECHO_N "Dumped core! (saved in $here as""$PCP_ECHO_C"
	for c in ${dir}core*
	do
	    d=`basename $c`
	    $sudo mv $c $here/$seq.$d
	    $PCP_ECHO_PROG $PCP_ECHO_N " $seq.$d""$PCP_ECHO_C"
	done
	echo ")"
	status=0
    fi
}

# is a pre-existing mmv directory in place?  if so, move it aside
# and prepare a directory with write access by unprivileged users
#
_prepare_pmda_mmv()
{
    if [ -d "$PCP_TMP_DIR/mmv" ]
    then
	_save_config "$PCP_TMP_DIR/mmv"
	$sudo rm -rf $PCP_TMP_DIR/mmv
    fi
    $sudo mkdir -p -m 1777 "$PCP_TMP_DIR/mmv"
    $sudo chown $PCP_USER:$PCP_GROUP "$PCP_TMP_DIR/mmv"
    cullmmv=true
}

_restore_pmda_mmv()
{
    [ -n "$cullmmv" ] && $cullmmv && _restore_config "$PCP_TMP_DIR/mmv"
}

# prepare for a save-able pmcd and pmda configuration.
#
_prepare_pmda_install()
{
    [ -z "$1" ] && echo "Error: bad _prepare_pmda_install call"
    iam=$1

    # copy the pmcd config file to restore state later.
    _save_config $PCP_PMCDCONF_PATH

    cd $PCP_PMDAS_DIR/$iam
    if [ -f Makefile -o -f GNUmakefile ] ; then
	if $sudo ${MAKE:-make} clobber >$tmp._tmp 2>&1 ; then
	    :
	else
	    cat $tmp._tmp
	    echo "Arrgh, ${MAKE:-make} clobber failed"
	    status=1
	    exit $status
	fi
    fi

    # start from a known starting point
    $sudo ./Remove >/dev/null 2>&1
    [ -f $PCP_VAR_DIR/config/$iam/$iam.conf ] && \
	$sudo mv $PCP_VAR_DIR/config/$iam/$iam.conf $tmp.$iam.conf
}

# restore a saved pmcd configuration and ensure pmda back in place.
# $1 is PMDA name
# $2 is optional, and if not empty the (default) Install is not done
# (because the Install may be interactive or may fail without some
# user-provided information)
#
_restore_pmda_install()
{
    [ -z "$1" ] && echo "Error: bad _restore_pmda_install call"
    iam=$1
    signal=$PCP_BINADM_DIR/pmsignal

    [ -f $PCP_VAR_DIR/config/$iam/$iam.conf.$seq ] \
	&& $sudo mv $PCP_VAR_DIR/config/$iam/$iam.conf.$seq $PCP_VAR_DIR/config/$iam/$iam.conf

    if diff $PCP_PMCDCONF_PATH.$seq $PCP_PMCDCONF_PATH > /dev/null 2>&1
    then
	_restore_config $PCP_PMCDCONF_PATH
    else

	if [ -z "$2" ]
	then
	    # do a default install which ensures the pmns and any
	    # views are installed
	    #
	    _here=`pwd`
	    cd $PCP_PMDAS_DIR/$iam
	    $sudo ./Install < /dev/null > /dev/null 2>&1
	    cd $_here
	fi

# PMDA may have been installed differently to default. As everything is
# installed we can use the old pmcd.conf file to restore state.

        if diff $PCP_PMCDCONF_PATH.$seq $PCP_PMCDCONF_PATH > /dev/null 2>&1
        then
	    _restore_config $PCP_PMCDCONF_PATH
        else
	    _restore_config $PCP_PMCDCONF_PATH
            $sudo $signal -a -s HUP pmcd
        fi
    fi
}

# find a local port that is not in use, optionally starting from a suggested port
#
_find_free_port()
{
    base=$1
    [ -z "$base" ] && base=54321

    while $PCP_BINADM_DIR/telnet-probe -c localhost $base
    do
	base=`expr $base + 1`
    done
    echo $base
}

# get pid for the running primary pmlogger
#
_get_primary_logger_pid()
{
    root="$PCP_TMP_DIR/pmlogger/primary"
    if [ ! -L "$root" ]
    then
	echo ""
    elif which realpath >/dev/null 2>&1
    then
	symroot=`realpath "$root"`
	basename "$symroot"
    else
	symroot=`ls -l "$root" | sed -e 's/.*-> //'`
	if [ -z "$symroot" ]
	then
	    echo "Arrgh, cannot get symlink for device for root fs ..."
	    ls -l "$root"
	    status=0
	    exit $status
	fi
	basename "$symroot"
    fi
}

_get_libpcp_config()
{
    pmconfig -L -s > $tmp.config
    . $tmp.config
    rm $tmp.config
}

# core dumping deep inside the libvirt Python wrapper layer ...
# not much we can do here other than skip libvirtpmda stuff`
#
_libvirt_is_ok()
{
    if [ -f /etc/lsb-release ]
    then
	if grep -q 'DISTRIB_ID=Ubuntu' /etc/lsb-release
	then
	    # OK, Ubuntu ... now let's check the libvirt version
	    #
	    cat <<'End-of-File' >$tmp.py
import sys
import libvirt
conn = libvirt.open('qemu:///system')
if conn != None:
    ver = conn.getVersion()
    print(str(ver))
    conn.close()
exit(0)
End-of-File
	    _libvirt_ver=`$PCP_PYTHON_PROG $tmp.py 2>/dev/null`
	    echo "Ubuntu and libvirt version $_libvirt_ver" >>$here/$seq.full
	    case "$_libvirt_ver"
	    in
		2005000)
		    return 1
		    ;;
	    esac
	fi
    fi
    return 0
}

# send pmcd SIGHUP and reliably check that it received (at least) one
#
_sighup_pmcd()
{
    _signal_delay=0.05
    _sighups_before=-1

    eval `pmprobe $@ -v pmcd.sighups 2>/dev/null \
	  | $PCP_AWK_PROG '{ printf "_sighups_before=%d\n", $3 }'`
    if [ $_sighups_before -lt 0 ]
    then
	echo _sighup_pmcd called but pmcd not running
	status=0
	exit $status
    fi

    $sudo $PCP_BINADM_DIR/pmsignal -a -s HUP pmcd >/dev/null 2>&1

    # first make sure pmcd has received SIGHUP
    #
    _sts=1
    for _delay in 0.01 0.05 0.1 0.15 0.25 0.5 1 2
    do
	$PCP_BINADM_DIR/pmsleep $_delay
	_sighups=-1
	eval `pmprobe $@ -v pmcd.sighups 2>/dev/null | \
	    $PCP_AWK_PROG '{ printf "_sighups=%d\n", $3 }'`
	if [ $_sighups -gt $_sighups_before ]
	then
	    _sts=0
	    break
	fi
    done
    if [ $_sts -ne 0 ]
    then
	echo _sighup_pmcd caused no change in pmcd sighup count
	status=0
	exit $status
    fi

    # delay for $_signal_delay while pmcd actually does post-SIGHUP work
    #
    $PCP_BINADM_DIR/pmsleep $_signal_delay
}

# common webapi header filtering
#
_webapi_header_filter()
{
    tee -a $here/$seq.full \
    | col -b \
    | sed \
	-e 's/^\(Content-Length:\) [1-9][0-9]*/\1 SIZE/g' \
	-e 's/^\(User-Agent: curl\).*/\1 VERSION/g' \
	-e 's/^\(Date:\).*/\1 DATE/g' \
	-e 's/\(\"context\":\) [0-9][0-9]*/\1 CTXID/g' \
	-e '/^Connection: Keep-Alive/d' \
    | LC_COLLATE=POSIX sort
}

_webapi_response_filter()
{
    tee -a $here/$seq.full \
    | col -b \
    | sed -e 's,#[0-9]*,####,g' \
        -e 's/ connected$//g' \
        -e '/^\* Adding handle: /d' \
        -e '/^\* Closing connection ####/d' \
        -e '/- Conn .*_pipe.*_pipe/d' \
        -e '/Curl_[\.a-zA-Z0-9]*:/d' \
        -e 's/::1/LOCALADDR/g' \
        -e 's/127\.0\.0\.1/LOCALADDR/g' \
        -e 's/localhost[\.a-zA-Z0-9]*/LOCALHOST/g' \
        -e 's/\(Date:\).*/\1 DATE/' \
        -e 's,\(i>pmproxy\)/.*<.i,\1/VERSION<\\i,g' \
	-e 's/\[[0-9][0-9]* bytes data]/[data not shown]/' \
        -e '/> User-Agent: /d' \
	-e '/Connection: Keep-Alive/d' \
    # end
}
