Blame scripts/mash_run.sh

Thomas Oulevey 79d958
usage()
Thomas Oulevey 79d958
{
Thomas Oulevey 79d958
cat << EOF
Thomas Oulevey 79d958
usage: $0 [-t <tag(s)>]
Thomas Oulevey 79d958
Thomas Oulevey 79d958
This script generate mash repo from all SIG tags.
Thomas Oulevey 79d958
Thomas Oulevey 79d958
OPTIONS:
Thomas Oulevey 9f5c13
   -d          mash repo destination : default to $MASH_DEST
Thomas Oulevey 9f5c13
   -f          koji instance fqdn : default to $KOJI_FQDN
Thomas Oulevey 79d958
   -t          tag(s) : storage7-release, storage7-testing etc...
Thomas Oulevey 79d958
EOF
Thomas Oulevey 79d958
}
Thomas Oulevey 79d958
Thomas Oulevey 79d958
# Global config
Thomas Oulevey 39bd31
# koji binary
Thomas Oulevey 39bd31
KOJI="/usr/bin/koji"
Thomas Oulevey 79d958
# mash binary
Thomas Oulevey 79d958
MASH="/usr/bin/mash"
Thomas Oulevey 9f5c13
# default fqdn
Thomas Oulevey 9f5c13
KOJI_FQDN="cbs.centos.org"
Thomas Oulevey 79d958
# mash config containing mash.conf
Thomas Oulevey 79d958
MASH_CONF="/etc/mash"
Thomas Oulevey 79d958
# repos destination
Thomas Oulevey 79d958
MASH_DEST="/mnt/kojishare/repos"
Thomas Oulevey 39bd31
#cache
Thomas Oulevey 39bd31
MASH_CACHE="/var/cache/cbs-mash"
Thomas Oulevey 79d958
# log directory
Thomas Oulevey 79d958
LOG_DIR="/var/log/mash"
Thomas Oulevey 79d958
# End Global
Thomas Oulevey 79d958
Thomas Oulevey 9f5c13
optiond=false
Thomas Oulevey 9f5c13
optionf=false
Thomas Oulevey 79d958
optiont=false
Thomas Oulevey 79d958
optionv=false
Thomas Oulevey 9f5c13
Thomas Oulevey 9f5c13
while getopts ":hvd:f:t:" OPTION
Thomas Oulevey 79d958
do
Thomas Oulevey 79d958
	case ${OPTION} in
Thomas Oulevey 9f5c13
    h)
Thomas Oulevey 79d958
             usage
Thomas Oulevey 79d958
             exit 0
Thomas Oulevey 79d958
             ;;
Thomas Oulevey 79d958
	t)
Thomas Oulevey 79d958
             optiont=true
Thomas Oulevey 79d958
             TAGS="${OPTARG}"
Thomas Oulevey 79d958
             ;;
Thomas Oulevey 9f5c13
	d)
Thomas Oulevey 9f5c13
             optiond=true
Thomas Oulevey 9f5c13
             MASH_DEST="${OPTARG}"
Thomas Oulevey 9f5c13
             ;;
Thomas Oulevey 9f5c13
	f)
Thomas Oulevey 9f5c13
             optionf=true
Thomas Oulevey 9f5c13
             KOJI_FQDN="${OPTARG}"
Thomas Oulevey 9f5c13
             ;;
Thomas Oulevey 79d958
	v)
Thomas Oulevey 9f5c13
	         optionv=true
Thomas Oulevey 9f5c13
	         ;;
Thomas Oulevey 9f5c13
    ?)
Thomas Oulevey 79d958
             exit 0
Thomas Oulevey 79d958
             ;;
Thomas Oulevey 9f5c13
    :)
Thomas Oulevey 79d958
             echo "Option -${OPTARG} requires an argument."
Thomas Oulevey 79d958
             exit 1
Thomas Oulevey 79d958
             ;;
Thomas Oulevey 9f5c13
    esac
Thomas Oulevey 79d958
done
Thomas Oulevey 79d958
Thomas Oulevey 79d958
shift $((${OPTIND} - 1))
Thomas Oulevey 79d958
Thomas Oulevey 9f5c13
# Repoviewurl in mash config
Thomas Oulevey 9f5c13
MASH_VIEWURL="http://$KOJI_FQDN/repos"
Thomas Oulevey 9f5c13
Thomas Oulevey 39bd31
# Check if user is allowed to send command to koji and has 'admin' permission
Thomas Oulevey 9f5c13
PERMS=`${KOJI} list-permissions --mine`
Thomas Oulevey 9f5c13
Thomas Oulevey 9f5c13
for P in $PERMS
Thomas Oulevey 9f5c13
do
Thomas Oulevey d8c6de
    [[ $P == 'admin' ]] && ADMIN=true && break
Thomas Oulevey 9f5c13
done
Thomas Oulevey 9f5c13
Thomas Oulevey 9f5c13
if [ "$ADMIN" != true ]
Thomas Oulevey 39bd31
then
Thomas Oulevey 39bd31
	echo "[ERROR] Koji misconfigure/missing admin privilege for creating SIG tags"
Thomas Oulevey 39bd31
	exit 1
Thomas Oulevey 39bd31
fi
Thomas Oulevey 39bd31
Thomas Oulevey 79d958
if ( ! ${optiont} )
Thomas Oulevey 79d958
then
Thomas Oulevey 39bd31
	CANDIDATE_TAGS=`${KOJI} list-tags | grep "\-candidate"`
Thomas Oulevey 39bd31
	TEST_TAGS=`${KOJI} list-tags | grep "\-testing"`
Thomas Oulevey 39bd31
	RELEASE_TAGS=`${KOJI} list-tags | grep "\-release"`
Thomas Oulevey c46e0e
	PENDING_TAGS=`${KOJI} list-tags | grep "\-pending"`
Thomas Oulevey ca3829
	# openstack: After stein, tags -build contains build dependencies
Thomas Oulevey ca3829
	EXTRA_TAGS=`${KOJI} list-tags | grep "openstack\-[s-z]" | grep build`
Thomas Oulevey ca3829
	TAGS="${CANDIDATE_TAGS} ${TEST_TAGS} ${RELEASE_TAGS} ${PENDING_TAGS} ${EXTRA_TAGS}"
Thomas Oulevey 79d958
fi
Thomas Oulevey 79d958
Thomas Oulevey 79d958
print_mash_template()
Thomas Oulevey 79d958
{
Thomas Oulevey 79d958
	local tag=$1
Thomas Oulevey 38e218
	local arches=$2
Thomas Oulevey 79d958
cat << EOF
Thomas Oulevey 79d958
[${tag}]
Thomas Oulevey 79d958
rpm_path = ${MASH_DEST}/${tag}/%(arch)s/os/Packages
Thomas Oulevey 79d958
repodata_path = ${MASH_DEST}/${tag}/%(arch)s/os/
Thomas Oulevey 79d958
source_path = source/SRPMS
Thomas Oulevey 79d958
debuginfo = True
Thomas Oulevey f15f88
multilib = False
Thomas Oulevey 79d958
multilib_method = devel
Thomas Oulevey 79d958
tag = ${tag}
Thomas Oulevey 79d958
inherit = False
Thomas Oulevey 79d958
strict_keys = False
Thomas Oulevey 79d958
repoviewurl = ${MASH_VIEWURL}/${tag}/%(arch)s/os/
Thomas Oulevey 79d958
repoviewtitle = "${tag^^}"
Thomas Oulevey 38e218
arches = ${arches}
Thomas Oulevey 79d958
delta = True
Thomas Oulevey 79d958
EOF
Thomas Oulevey 79d958
}
Thomas Oulevey 79d958
Thomas Oulevey 79d958
mash_prepare ()
Thomas Oulevey 79d958
{
Thomas Oulevey 79d958
	local tag=$1
Thomas Oulevey 79d958
	local log=$2
Thomas Oulevey 38e218
	local arches=$3
Thomas Oulevey 38e218
	local conf=`mktemp`
Thomas Oulevey 79d958
	( $optionv ) && echo "* Checking ${tag} mash config..."
Thomas Oulevey d85ccd
	# config mash already ok
Thomas Oulevey 79d958
	( $optionv ) && echo " -> [INFO] creating mash config: ${tag}.mash..."
Thomas Oulevey d85ccd
	print_mash_template "${tag}" "${arches}" > $conf
Thomas Oulevey 2a062d
	[ -f ${MASH_CONF}/${tag}.mash ] && diff ${MASH_CONF}/${tag}.mash $conf &>> $log
Thomas Oulevey 38e218
	if [ $? -gt 0 ]
Thomas Oulevey 38e218
	then
Thomas Oulevey 2a062d
		( $optionv ) && echo " -> [INFO] updating mash config ${tag}.mash"
Thomas Oulevey 38e218
		mv $conf ${MASH_CONF}/${tag}.mash
Thomas Oulevey 2a062d
		( $optionv ) && echo " -> [INFO] cleaning mash cache ${tag}.buildlist"
Thomas eb0684
        [ -f $MASH_CACHE/${tag}.buildlist ] && rm $MASH_CACHE/${tag}.buildlist
Thomas Oulevey 6f2d99
	else
Thomas Oulevey 6f2d99
		rm $conf
Thomas Oulevey 38e218
	fi
Thomas Oulevey 79d958
}
Thomas Oulevey 79d958
Thomas Oulevey 79d958
mash_run () {
Thomas Oulevey 79d958
	local tag=$1
Thomas Oulevey 79d958
	local log=$2
Thomas Oulevey 79d958
	${MASH} -p ${MASH_DEST}/ -o ${MASH_DEST}/ ${tag} &> ${log}
Thomas Oulevey 27ae02
	if [ $? -gt 0 ] 
Thomas Oulevey 27ae02
	then
Thomas Oulevey 27ae02
		echo " -> [ERROR] mash run failed ${log}"
Thomas cc5dbf
		[ -f ${MASH_CACHE}/${tag}.buildlist ] && rm -rf $MASH_CACHE/${tag}.buildlist
Thomas Oulevey 27ae02
	fi
Thomas Oulevey 79d958
	( $optionv ) && echo " -> [INFO] mash run succeeded ${log}"
Thomas Oulevey 79d958
}
Thomas Oulevey 79d958
Thomas Oulevey 39bd31
# Ensure the cache directory is available
Thomas Oulevey 39bd31
[ ! -d $MASH_CACHE ] && mkdir -p $MASH_CACHE
Thomas Oulevey 39bd31
Thomas 45bfd3
if ( ! ${optiont} )
Thomas 45bfd3
then
Thomas 45bfd3
	# Ensure we do not have parallel runs
Thomas 45bfd3
	pidfile=/var/tmp/mash-run.pid
Thomas 45bfd3
	if [ -e $pidfile ]; then
Thomas 45bfd3
		pid=`cat $pidfile`
Thomas 45bfd3
		if kill -0 &> /dev/null $pid; then
Thomas 45bfd3
			( $optionv ) && echo "Mash is already running PID:$pid"
Thomas 45bfd3
			exit 1
Thomas 45bfd3
		else
Thomas 45bfd3
			rm $pidfile &>/dev/null
Thomas 45bfd3
		fi
Thomas 45bfd3
	fi
Thomas 45bfd3
	echo $$ > $pidfile
Thomas Oulevey 79d958
fi
Thomas Oulevey 79d958
Thomas 45bfd3
# Cache target list as it cannot include unknown tag at this point
Thomas 45bfd3
TARGETS=`mktemp`
Thomas 45bfd3
${KOJI} list-targets --quiet > $TARGETS
Thomas Oulevey 79d958
Thomas Oulevey 79d958
for TAG in ${TAGS}
Thomas Oulevey 79d958
do
Thomas 45bfd3
	( $optionv ) && echo "Checking $TAG ..."
Thomas Oulevey 650807
	ARCHES=""
Thomas Oulevey 38e218
	BUILDTAG=""
Thomas Oulevey 79d958
	LOG="${LOG_DIR}/mash.${TAG}.log"
Thomas Oulevey d85ccd
	if [[ $TAG = *"infrastructure"* ]]
Thomas Oulevey 75f51b
	then
Thomas Oulevey d85ccd
		FAKETAG=${TAG/release/testing}
Thomas Oulevey d85ccd
	else
Thomas Oulevey d85ccd
		FAKETAG=${TAG/testing/candidate}
Thomas Oulevey 75f51b
		FAKETAG=${FAKETAG/release/candidate}
Thomas Oulevey d85ccd
	fi
Thomas 45bfd3
	BUILDTAG=`grep ${FAKETAG} ${TARGETS} | awk '{print $2}'`
Thomas Oulevey 6f2d99
	[ ! -z "$BUILDTAG" ] && ARCHES=`${KOJI} taginfo ${BUILDTAG} | grep Arches | cut -d ":" -f 2-`
Thomas Oulevey 6f2d99
	[ -z "$ARCHES" ] && ARCHES="x86_64"
Thomas 4b7198
	# Mash not happy with i686 wants i386
Thomas 4b7198
	ARCHES=${ARCHES/i686/i386}
Thomas Oulevey 2a062d
	mash_prepare "${TAG}" "${LOG}" "${ARCHES}"
Thomas Oulevey 39bd31
	if [ ! -f $MASH_CACHE/$TAG.buildlist ]
Thomas Oulevey 39bd31
	then
Thomas Oulevey 39bd31
        	${KOJI} list-tagged $TAG > $MASH_CACHE/$TAG.buildlist
Thomas Oulevey 39bd31
	else
Thomas Oulevey 75f51b
		BUILDLIST=`mktemp`
Thomas Oulevey 75f51b
		${KOJI} list-tagged $TAG > $BUILDLIST
Thomas 45bfd3
		diff $BUILDLIST $MASH_CACHE/$TAG.buildlist &>> $LOG
Thomas 45bfd3
		if [ $? -eq 0 ]
Thomas 45bfd3
		then
Thomas 45bfd3
			echo " -> skipping. No new build in $TAG" &>> $LOG
Thomas eb0684
			[ -f $BUILDLIST ] &&  rm -rf $BUILDLIST
Thomas 45bfd3
			continue
Thomas 45bfd3
		else
Thomas Oulevey 75f51b
			echo " -> updating cache for $TAG" &>> $LOG
Thomas eb0684
			cp $BUILDLIST $MASH_CACHE/$TAG.buildlist
Thomas Oulevey 75f51b
		fi
Thomas eb0684
		[ -f $BUILDLIST ] &&  rm -rf $BUILDLIST
Thomas Oulevey 39bd31
	fi
Thomas Oulevey 79d958
	mash_run "${TAG}" "${LOG}" &
Thomas Oulevey 79d958
done
Thomas Oulevey 79d958
Thomas Oulevey 79d958
wait
Thomas Oulevey c1800f
( ! $optiont ) && rm $pidfile
Thomas Oulevey 79d958
exit 0