#!/bin/sh
#
# pcp-qa-summary [vm ...]
#
# Used in concert with pcp-daily -q
#

export LC_COLLATE=POSIX

_usage()
{
    echo "Usage: $0 [options] [vm ...]"
    echo 'Options:'
    echo '  -a      report all tests, not just failing ones (implies -n)'
    echo '  -c      report counts not percentages'
    echo '  -C      include results from CI'
    echo '  -d      debug'
    echo '  -f      sort failure map on %fail [default sort on Test #]'
    echo '  -m N    minimum number of tests run to be reported [default 60]'
    echo '  -n      report skipped and not run tests (reported as "S" and "N")'
    echo '  -r      only most recent results date [default report all dates]'
    echo '  -t      report triaged tests (reported as "T")'
    exit 1
}

_warn_about_BAD()
{
    if [ ! -f $HOME/src/pcp/qa/group ]
    then
	echo "Arrgh, no $HOME/src/pcp/qa/group ... do you know what you're doing?"
	sts=1
	exit
    else
	grep '^[0-9][^ ]* .*BAD' $HOME/src/pcp/qa/group \
	| sed >$tmp.bad \
	    -e '/^[0-9][0-9]*:/d' \
	# end
	if [ -s $tmp.bad ]
	then
	    echo
	    echo "Warning: the following tests have not been run at all because they do not"
	    echo "         satisfy the cross-platform robustness criteria."
	    echo "         (refer to man/html/portability-and-qa.html)"
	    sed -e 's/^/    /' <$tmp.bad
	fi
    fi
}

_map_ci_hostname()
{
    sed \
	-e 's/^test-/ci-/' \
	-e 's/-container$//' \
	-e 's/centos-stream/centos/' \
    # end
}

# must have tried to run at least this number to qualify
# 40 => -g sanity circa PCP 2.7.8 [minus a few not runs]
# 25 => -g remote circa PCP 4.0.0 [minus a few not runs]
#
minrun=25

debug=false
recent=false
aflag=false
cflag=false
nflag=false
tflag=false
failsort=false
CI=false
while getopts 'acCdfm:nrt?' p
do
    case "$p"
    in
	a)	aflag=true
		nflag=true
		;;
	c)	cflag=true
		;;
	C)	CI=true
		;;
	d)	debug=true
		;;
	f)	failsort=true
		;;
	m)	minrun=$OPTARG
		;;
	n)	nflag=true
		;;
	r)	recent=true
		;;
	t)	tflag=true
		;;
	?)	_usage
		# NOTREACHED
    esac
done
shift `expr $OPTIND - 1`

sts=0
if $debug
then
    tmp=`pwd`/tmp-summary
    trap "exit \$sts" 0 1 2 3 15
else
    tmp=/var/tmp/$$
    trap "rm -f $tmp.*; exit \$sts" 0 1 2 3 15
fi
rm -f $tmp.*

if [ $# -eq 0 ]
then
    set -- `[ -d $HOME/Logs/by-vm ] && cd $HOME/Logs/by-vm && echo *`
fi

if [ -f $HOME/whatami.out ]
then
    # Turn
    # Host        PCP      CPU     Operating System
    # ----------- -------- ------  ------------------
    # vm00        3.7.0    x86_64  Ubuntu 12.04 (precise)
    # into
    # vm00|PCP 3.7.0 x86_64 Ubuntu 12.04
    #
    sed <$HOME/whatami.out \
	-e '1,2d' \
	-e '/^[ 	]*$/d' \
	-e 's/  */|PCP#/' \
	-e 's/  */#/' \
	-e 's/  */#/' \
	-e 's/  *(.*)//' \
	-e 's/#/ /g' \
    | sort -t '|' --key=1,1 >$tmp.what
fi

_what()
{
    if [ -f $tmp.what ]
    then
	sort -t '|' --key=1,1 \
	| join \
	    -1 2 -2 1 -t '|' -a 1 -e "$vmdir" -o 1.1,2.1,2.2 \
	    - $tmp.what \
	| awk -F\| '
BEGIN	{ OFS ="|" }
	{ if ($2 ~ /^vm[0-9][0-9]$/)
	    $2 = substr($2,3,2) "|" $2
	  else if ($2 ~ /^bozo-laptop$/)
	    $2 = "bl" "|" $2
	  else if ($2 ~ /^bozo-vm$/)
	    $2 = "bv" "|" $2
	  else if ($2 ~ /^ci-centos[1-9]$/)
	    $2 = "c" substr($2,10,1) "|" $2
	  else if ($2 ~ /^ci-centos-stream[1-9]$/)
	    $2 = "c" substr($2,16,1) "|" $2
	  else if ($2 ~ /^ci-debian[1-9][0-9]$/)
	    $2 = "d" substr($2,10,2) "|" $2
	  else if ($2 ~ /^ci-fedora-rawhide$/)
	    $2 = "frh" "|" $2
	  else if ($2 ~ /^ci-fedora[1-9][0-9]$/)
	    $2 = "f" substr($2,10,2) "|" $2
	  else if ($2 ~ /^ci-ubuntu1804-i386$/)
	    $2 = "u318" "|" $2
	  else if ($2 ~ /^ci-ubuntu2004$/)
	    $2 = "ub20" "|" $2
	  else if ($2 ~ /^ci-ubuntu2204$/)
	    $2 = "ub22" "|" $2
	  else if ($2 ~ /^ci-ubuntu[1-9][0-9]..$/)
	    $2 = "u" substr($2,10,2) "|" $2
	  else
	    $2 = substr($2,1,2) "|" $2
	}
	{ print }' \
	| sed -e 's/|PCP/ PCP/'
    else
	cat
    fi
}

echo "==== QA Summary ===="
echo
echo "Date        Run Pass Fail Nrun Host"
numqahosts=0
for vm
do
    cd $HOME/Logs/by-vm
    if [ -d "$vm" ]
    then
	cd "$vm"
	vmdir="$vm"
    else
	vmdir=`echo ${vm}*`
	if [ -d "$vmdir" ]
	then
	    cd "$vmdir"
	else
	    echo "$vm: arrrgggh no \$HOME/Logs/${vm}* directory"
	    continue
	fi
    fi
    basevmname=`echo $vmdir | sed -e 's/\(vm[0-9][0-9]\)-.*/\1/' -e 's/\.sgi.com//'`
    rm -f $tmp.seen $tmp.found
    for date in `ls 2???-??-?? test.log 2>/dev/null | sort -r`
    do
	[ -f $date ] || continue
	[ "$date" = test.log -a "$CI" = false ] && continue
	touch $tmp.seen
	if [ "$date" = test.log ]
	then
	    # this is from CI aka src/qa-reports ... get date from git
	    # and machine's real name from /bin/pwd, but only of -C in play
	    #
	    _date=`git log --format=%cs`
	    basevmname=`/bin/pwd | sed -e 's@.*/@@' | _map_ci_hostname`
	else
	    _date=$date
	fi
	$debug && echo "Debug: == $vmdir on $_date =="
	# Looking for
	# Not run: 008 009 022 096 114 117 130 148 164 212 223 233 284 290 307 311 329 330 331 332 333 348 361 370 377 390 420 441 442 476 477 478 480 506 512 545 570 584 640 652 654 712 713 714
	# Failures: 062 169 244
	# Failed 3 of 516 tests
	# Triaged: 1662 1701 1722 1723
	# Passed all 67 tests (old format)
	# Passed 67 tests (newer format)
	#
	rm -f $tmp.dbg
	if grep -E '^((Not run:)|Failures:|(Failed [0-9][0-9]* of [0-9][0-9]* tests)|Triaged:|(Passed [0-9][0-9]* tests)|(Passed all [0-9][0-9]* tests)|Info:|===)' <$date >/dev/null
	then
	    awk <$date >$tmp.out '
# states
# 1 - looking for split Not run: list
# 2 - looking for split Failures: list
# 3 - looking for split Triaged: list
# 4 - done (seen final Passed or Failed N of N line)
/^Not run:/	{ notrun = NF - 2
		  nrlist = $3
		  for (i = 4; i <= NF; i++) nrlist = nrlist " " $i
		  print NR,": Not run: start: notrun=" notrun >"'$tmp.dbg'"
		  state = 1
		  next
		}
/^Failures:/	{ nfail = NF - 1
		  flist = $2
		  for (i = 3; i <= NF; i++) flist = flist " " $i
		  print NR,": Failures: start: nfail=" nfail >"'$tmp.dbg'"
		  state = 2
		  next
		}
/^Failed [0-9][0-9]* of [0-9][0-9]* tests/	{
		  failed = $2
		  run = $4
		  state = 4
		}
/^Triaged:/	{ ntriag = NF - 1
		  tlist = $2
		  for (i = 3; i <= NF; i++) tlist = tlist " " $i
		  print NR,": Triaged: start: nfail=" nfail >"'$tmp.dbg'"
		  state = 4
		  next
		}
/^Passed [0-9][0-9]* tests/	{
		  failed = 0
		  run = $2
		  print NR,": Passed: " run >"'$tmp.dbg'"
		  state = 4
		}

/^Passed all [0-9][0-9]* tests/	{
		  failed = 0
		  run = $3
		  print NR,": Passed all: " run >"'$tmp.dbg'"
		  state = 4
		}
state == 4	{ if (run >= '$minrun') printf "%s %4d %4d %4d %4d %4d|%s|%s+%s+%s\n","'$_date'",run,run-failed,failed,notrun,ntriag,"'$basevmname'",flist,nrlist,tlist
		  else print NR,": results skipped: run (" run ") < minrun ('$minrun')" >"'$tmp.dbg'"
		  state = 0
		  notrun = 0
		  nfail = 0
		}
/^Info:/	{
		  state = 0
		  notrun = 0
		  nfail = 0
		  next
		}
/^===/		{
		  state = 0
		  notrun = 0
		  nfail = 0
		  next
		}
state == 1	{ # Not run: list broken over multiple lines
		  notrun += NF
		  for (i = 1; i <= NF; i++) nrlist = nrlist " " $i
		  print NR,": Not run: add",NF," notrun=" notrun >"'$tmp.dbg'"
		  next
		}
state == 2	{ # Failures: list broken over multiple lines
		  nfail += NF
		  for (i = 1; i <= NF; i++) flist = flist " " $i
		  print NR,": Failures: add",NF," nfail=" nfail >"'$tmp.dbg'"
		  next
		}
state == 3	{ # Triaged: list broken over multiple lines
		  ntriag += NF
		  for (i = 1; i <= NF; i++) tlist = tlist " " $i
		  print NR,": Triaged: add",NF," ntriag=" ntriag >"'$tmp.dbg'"
		  next
		}
END		{ print NR,": END state=" state >"'$tmp.dbg'"
		}'
	    # Need to sort multiple runs one this date and pick the one
	    # with the maximum passes and minimum not runs
	    #
	    # 2013-03-13      1      1      0      0|vm19-opensuse-64
	    # 2013-03-13     67     67      0      2|vm19-opensuse-64
	    # 2013-03-13    512    509      3     48|vm19-opensuse-64
	    # 2013-03-13    512    509      3     16|vm19-opensuse-64
	    $debug && sed -e 's/^/Debug: line:/' <$tmp.dbg
	    $debug && cat $tmp.out
	    sort <$tmp.out --key=3,3nr --key=5,5n \
	    | sed -e 1q >$tmp.tmp
	    # $tmp.tmp line is ...
	    # <date> <#run> <#passed> <#failed> <#notrun> <#triaged>|<hostname>|<failed list...>+<notrun list>+<triaged list>
	    if [ -s $tmp.tmp ]
	    then
		if $debug
		then
		    echo "Debug: summary lines ..."
		    sed -e 's/^/Debug: /' <$tmp.out
		fi
		sed -e 's/|[^|]*$//' <$tmp.tmp | _what >>$tmp.found
		# failed ...
		#
		nfail=`awk <$tmp.tmp '{print $4}'`
		if [ $nfail -gt 0 ]
		then
		    awk <$tmp.tmp -F+ '{print $1}' \
		    | sed -e 's/.*|//' \
		    | tr ' ' '\012' \
		    | sed -e "s/\$/ X $basevmname/" >>$tmp.map
		fi

		# not run ...
		#
		nnotrun=0
		if $nflag
		then
		    nnotrun=`sed -e 's/|.*//' <$tmp.tmp | awk '{print $5}'`
		    if [ $nnotrun -gt 0 ]
		    then
			awk <$tmp.tmp -F+ '{print $2}' \
			| tr ' ' '\012' \
			| sed -e "s/\$/ N $basevmname/" >>$tmp.map
		    fi
		fi

		# triaged ...
		#
		ntriag=0
		if $tflag
		then
		    ntriag=`sed -e 's/|.*//' <$tmp.tmp | awk '{print $6}'`
		    if [ $ntriag -gt 0 ]
		    then
			awk <$tmp.tmp -F+ '{print $3}' \
			| tr ' ' '\012' \
			| sed -e "s/\$/ T $basevmname/" >>$tmp.map
		    fi
		fi

		if $aflag || [ $nfail -gt 0 -o $nnotrun -gt 0 ]
		then
		    # pass lines look like:
		    # [97%] 1055 16s ...
		    # or
		    # [97%] 1055
		    # and may contain " [fixup]"
		    #
		    sed <$date -n \
				-e 's/ \[fixup]//' \
		    		-e '/^[[0-9][0-9]*%] [0-9][0-9]* [0-9][0-9]*s \.\.\./{
s/.*] //
s/ .*//
p
}' \
				-e '/^[[0-9][0-9]*%] [0-9][0-9]*$/{
s/.*] //
p
}' \
		    | sed -e "s/\$/ P $basevmname/" >>$tmp.map
		fi
		$recent && break
	    else
		$debug && echo "Debug: tag lines, but no QA results found"
	    fi
	else
	    $debug && echo "Debug: no QA result tag lines found"
	fi
    done
    if [ -f $tmp.found ]
    then
	sort $tmp.found
	cat $tmp.found >>$tmp.counts
	numqahosts=`expr $numqahosts + 1`
    else
	if $recent
	then
	    : be quiet
	else
	    if [ -f $tmp.seen ]
	    then
		printf "%-30s|%s\n" "Daily runs, but no QA" "$basevmname" | _what
	    elif [ ! -f $tmp.seen ]
	    then
		printf "%-30s|%s\n" "No daily runs" "$basevmname" | _what
	    fi
	fi
    fi

done

if [ -f $tmp.counts ]
then
    awk <$tmp.counts '
	{ nrun += $2; nfail += $4; ntriag += $6 }
END	{ printf "Summary: %d run, %d failed (%.2f%%)", nrun, nfail, 100*nfail/nrun
	  if (ntriag > 0)
	    printf ", %d triaged (%.2f%%)", ntriag, 100*ntriag/nrun
	  printf "\n"
	}'
fi

_warn_about_BAD

if [ -f $tmp.map ]
then
    echo
    if $nflag
    then
	echo "==== QA Failure (X), Not Run (N) and Skipped (S) Map ===="
    else
	echo "==== QA Failure (X) and Not Run or Skipped (-) Map ===="
    fi
    echo
    ( ( sed -e 's/.* //' <$tmp.map | sort | uniq ) \
      ; echo "---" \
      ; sort -n $tmp.map \
      ; echo "END" \
    ) \
    | awk '
function heading() {
    # CI abbreviated hostnames are > 2 chars => need 2 heading lines
    hlines = 1
    for (i = 0; i < nh; i++) {
        if (host[i] ~ /^ci-debian/ || host[i] ~ /^ci-fedora/ || host[i] ~ /^ci-ubuntu/) {
	    hlines = 2
	    break
	}
    }
    printf "%4.4s","Host"
    printf " %5.5s",""		# %fail or nfail (-c)
    if (nflag) {
	printf " %5.5s",""	# %pass or npass (-c)
	printf " %5.5s",""	# %nrun or nnrun (-c)
    }
    if (hlines == 2) {
	for (i = 0; i < nh; i++) {
	    if (host[i] ~ /^ci-debian/)
		printf " %2.2s", "d"
	    else if (host[i] ~ /^ci-fedora/)
		printf " %2.2s", "f"
	    else if (host[i] == "ci-ubuntu1804-i386")
		printf " %2.2s", "u3"
	    else if (host[i] == "ci-ubuntu2004")
		printf " %2.2s", "ub"
	    else if (host[i] ~ /^ci-ubuntu/)
		printf " %2.2s", "u"
	    else
		printf " %2.2s", ""
	}
	printf "\n"
	printf "%4.4s",""
	printf " %5.5s",""		# %fail or nfail (-c)
	if (nflag) {
	    printf " %5.5s",""	# %pass or npass (-c)
	    printf " %5.5s",""	# %nrun or nnrun (-c)
	}
    }
    for (i = 0; i < nh; i++) {
	if (host[i] ~ /^vm/)
	    printf " %2.2s",substr(host[i],length(host[i])-1)
	else if (host[i] == "bozo-laptop")
	    printf " %2.2s","bl"
	else if (host[i] == "bozo-vm")
	    printf " %2.2s","bv"
        else if (host[i] ~ /^ci-centos[1-9]/)
	    printf " c%1.1s",substr(host[i],10,1)
	else if ($2 ~ /^ci-centos-stream[1-9]$/)
	    printf " c%1.1s",substr(host[i],16,1)
        else if (host[i] ~ /^ci-debian[1-9][0-9]/)
	    printf " %2.2s",substr(host[i],10,2)
        else if (host[i] ~ /^ci-fedora[1-9][0-9]/)
	    printf " %2.2s",substr(host[i],10,2)
        else if (host[i] == "ci-fedora-rawhide")
	    printf " %2.2s","rh"
        else if (host[i] ~ /^ci-ubuntu[1-9][0-9]/)
	    printf " %2.2s",substr(host[i],10,2)
	else
	    printf " %2.2s",substr(host[i],1,2)
    }
    printf "\n"
    printf "%4.4s","Test"
    if (cflag)
	printf " %5.5s"," fail"
    else
	printf " %5.5s","%fail"
    if (nflag) {
	if (cflag) {
	    printf " %5.5s"," pass"
	    printf " %5.5s"," nrun"
	}
	else {
	    printf " %5.5s","%pass"
	    printf " %5.5s","%nrun"
	}
    }
    for (i = 0; i < nh; i++) printf " %2.2s"," "
    printf " %4.4s QA groups\n","Test"
}
BEGIN	{ nh = 0
	  state = 0
	  aflag = 0
	  if ("'"$aflag"'" == "true") aflag = 1
	  cflag = 0
	  if ("'"$cflag"'" == "true") cflag = 1
	  nflag = 0
	  if ("'"$nflag"'" == "true") nflag = 1
	}
$1 == "---"	{ heading()
		  state = 1
		  last = $1
		  next
		}
state == 0	{ host[nh] = $1; nh++; next }
$1 != last	{ if (last != "---") {
		      skip = nrun = bad = triag = pass = 0
		      for (i = 0; i < nh; i++) {
			  if (fail[i] == "X") bad++
			  if (fail[i] == "T") triag++
			  if (fail[i] == "P") {
			      pass++
			      fail[i] = " "
			  }
			  if (fail[i] == "N") {
			      nrun++
			      if (nflag == 0) fail[i] = "-"
			  }
			  if (fail[i] == "S") {
			      skip++
			      if (nflag == 0) fail[i] = "-"
			  }
		      }
		      if (bad > 0 || triag > 0 || aflag) {
			  printf "%04d",last
			  if (nflag && nrun == nh) {
			    printf "   NOT RUN AT ALL "
			  }
			  else {
			      if (cflag)
				  printf "  %4d",bad
			      else
				  printf "  %3d%%",int(0.5+100*bad/'$numqahosts')
			      if (nflag) {
				  if (cflag) {
				      printf "  %4d",pass
				      printf "  %4d",nrun+skip
				  }
				  else {
				      if (pass > 0)
					  printf "  %3d%%",int(0.5+100*pass/'$numqahosts')
				      else
					  printf "  %3d%%",0
				      if (nrun > 0)
					  printf "  %3d%%",int(0.5+100*(nrun+skip)/'$numqahosts')
				      else
					  printf "  %3d%%",0
				  }
			      }
			  }
			  for (i = 0; i < nh; i++) {
			      printf " %2.2s",fail[i]
			  }
			  printf " %4.4s ",last
			  cmd = "( grep \"^" last " \" $HOME/src/pcp/qa/group || echo \"??? missing in QA group file\" ) | sed -e \"s/^[0-9]* //\""
			  system(cmd)
		      }
		  }
		  for (i = 0; i < nh; i++) fail[i] = "S"
		  last = $1
		 }
$1 == last	{ for (i = 0; i < nh; i++) {
		    if (host[i] == $3) {
			fail[i] = $2
			break
		    }
		  }
		}' \
    | sed \
	-e 's/ remote//' \
	-e 's/ sanity//' \
	-e 's/ ostest//' \
	-e 's/ local//' \
	-e 's/ oss//' \
	-e 's/ #[0-9][0-9]*//g' >$tmp.mapout
    num=0
    if grep '^Host.*  [dfu]' $tmp.mapout >/dev/null
    then
	hlines=3
    else
	hlines=2
    fi
    sed -e ${hlines}q <$tmp.mapout
    if $failsort
    then
	sed -e 1,${hlines}d <$tmp.mapout \
	| sort --key=2,2nr --key=1,1n 
    else
	sed -e 1,${hlines}d <$tmp.mapout
    fi \
    | while read line
    do
	if [ "$num" -eq 22 ]
	then
	    sed -e ${hlines}q <$tmp.mapout
	    num=0
	else
	    num=`expr $num + 1`
	fi
	echo "$line"
    done \
    | sed -e 's/^0/ /'
    # header lines as trailer, with the order slightly reversed
    if [ "$hlines" = 3 ]
    then
	sed -n -e 3p -e 3q <$tmp.mapout
	sed -n -e 1p -e 1q <$tmp.mapout
	sed -n -e 2p -e 2q <$tmp.mapout
    else
	sed -n -e 2p -e 2q <$tmp.mapout
	sed -n -e 1p -e 1q <$tmp.mapout
    fi
fi

