arrfab / centos / comps

Forked from centos/comps 5 years ago
Clone

Blame tools/compsgen.sh

c999a3
#!/bin/bash
c999a3
c999a3
#################################################################################
c999a3
# 
c999a3
# Name :  CentOS comps.xml generator 
c999a3
# Function : merge all the upstream comps.xml file into one
c999a3
# How to call it : see usage()
c999a3
# Author : Fabian Arrotin (arrfab@centos.org)
c999a3
# Requirements : sqlite3 / xmlstarlet
c999a3
#
c999a3
#################################################################################
c999a3
c999a3
usage() {
c999a3
c999a3
cat <<  EOF
c999a3
c999a3
You need to call the script like this : $0 -arguments
c999a3
c999a3
  -a : define the arch (required, default:none, values : [i386,x86_64])   
c999a3
  -p : define the path containing all the Upstream comps.xml files for all variants(required, default:none)
c999a3
  -r : release to build ( eg. 6 )
c999a3
  -h : display this help
c999a3
c999a3
EOF
c999a3
c999a3
}
c999a3
c999a3
c999a3
while getopts “ha:p:m:r:” OPTION
c999a3
do
c999a3
     case $OPTION in
c999a3
         h)
c999a3
             usage
c999a3
             exit 1
c999a3
             ;;
c999a3
         a)
c999a3
             arch=$OPTARG
c999a3
             ;;
c999a3
         p)
c999a3
             destdir=$OPTARG
c999a3
             ;;
c999a3
         r)
c999a3
             release=$OPTARG
c999a3
             ;;
c999a3
         ?)
c999a3
             usage
c999a3
             exit
c999a3
             ;;
c999a3
     esac
c999a3
done
c999a3
c999a3
varcheck() {
c999a3
if [ -z "$1" ] ; then
c999a3
        usage
c999a3
        exit 1
c999a3
fi
c999a3
c999a3
}
c999a3
c999a3
compsxmlin() {
c999a3
echo Creating Comps DB tables ...
c999a3
echo "CREATE TABLE categories (id ,name, desc, display_order ); CREATE TABLE languages (lang); CREATE TABLE groups (id, name, desc, def, uservisible, langonly, packagelist); CREATE TABLE grouplist (groupid, category_id);
c999a3
" |sqlite3 $sqlitedb
c999a3
c999a3
# beginning the loop for each xml file
c999a3
c999a3
for xmlfile in $(ls $destdir/*.xml);
c999a3
c999a3
	do
c999a3
c999a3
	# just do that once
c999a3
	if [ "$aretableready" = "1" ] ; then
c999a3
		echo Tables are already there	
c999a3
	else
c999a3
		echo Detecting languages and preparing - altering the tables ... 
c999a3
		for lang in $(grep 'description xml:lang=' $xmlfile|sed s/@/_arobas_/g|cut -f 1 -d '>'|cut -f 2 -d "'"|sort|uniq);do echo "insert into languages values ('$lang') ;"|sqlite3 $sqlitedb;done
c999a3
		for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb);do echo "alter table categories add desc_$lang ; alter table categories add name_$lang ; "|sqlite3 $sqlitedb;done
c999a3
		for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb);do echo "alter table groups add desc_$lang ; alter table groups add name_$lang ; "|sqlite3 $sqlitedb;done
c999a3
		export aretableready='1'
c999a3
	fi	
c999a3
c999a3
c999a3
	echo Processing now $xmlfile
c999a3
c999a3
	echo Detecting categories and filling the tables ...
c999a3
	# categories/id
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile);do isindb=$(echo "select id from categories where id='$id';"|sqlite3 $sqlitedb ) ;test "$isindb" = "$id" || (echo "insert into categories (id) values ('$id');"|sqlite3 $sqlitedb);done
c999a3
	# categories/name
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile); do name=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v name $xmlfile);isindb=$(echo "select name from categories where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$name" || (echo "update categories set name='$name' where id='$id' ;"|sqlite3 $sqlitedb ); done
c999a3
	# categories/description
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile); do desc=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v description $xmlfile);echo "update categories set desc='$desc' where id='$id' ;"|sqlite3 $sqlitedb; done
c999a3
	# categories/display_order
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile); do display_order=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v display_order $xmlfile); isindb=$(echo "select display_order from categories where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$display_order" || (echo "update categories set display_order='$display_order' where id='$id' ;"|sqlite3 $sqlitedb); done
c999a3
	# categories/name_$lang
c999a3
	for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb|sed s/_arobas_/@/g);do for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile);do namelang=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v "name[@xml:lang='$lang']" $xmlfile); langsql=$(echo $lang|sed s/@/_arobas_/g) ; isindb=$(echo "select name_$langsql from categories where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$namelang" || (namelangsql=$(echo $namelang|sed s/"'"/"''"/g) ;echo "update categories set name_$langsql='$namelangsql' where id='$id';"|sqlite3 $sqlitedb) ; done ; done
c999a3
	# categories/description_$lang
c999a3
	for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb|sed s/_arobas_/@/g);do for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile);do desclang=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v "description[@xml:lang='$lang']" $xmlfile); langsql=$(echo $lang|sed s/@/_arobas_/g) ; isindb=$(echo "select desc_$langsql from categories where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$desclang"  || (desclangsql=$(echo $desclang|sed s/"'"/"''"/g) ;echo "update categories set desc_$langsql='$desclangsql' where id='$id';"|sqlite3 $sqlitedb) ; done ; done
c999a3
	# categories/grouplist
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile); do for groupid in $(xmlstarlet sel -t -m "//comps/category[id='$id']" -v grouplist $xmlfile); do isindb=$(echo "select groupid from grouplist where category_id='$id' and groupid='$groupid';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$groupid" || (echo "insert into grouplist (groupid, category_id) values ('$groupid','$id');"|sqlite3 $sqlitedb); done ; done
c999a3
c999a3
	echo Detecting groups and filling the tables ...
c999a3
	# groups/id
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile);do isindb=$(echo "select id from groups where id='$id';"|sqlite3 $sqlitedb ) ;test "$isindb" = "$id"  || (echo "insert into groups (id) values ('$id');"|sqlite3 $sqlitedb);done
c999a3
	# groups/name
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do name=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v name $xmlfile);isindb=$(echo "select name from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$name" || (echo "update groups set name='$name' where id='$id' ;"|sqlite3 $sqlitedb ); done
c999a3
	# groups/default
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do def=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v default $xmlfile);isindb=$(echo "select def from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$def" || (echo "update groups set def='$def' where id='$id' ;"|sqlite3 $sqlitedb ); done
c999a3
	# groups/description
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do desc=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v description $xmlfile);isindb=$(echo "select desc from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$desc" || (descsql=$(echo $desc|sed s/"'"/"''"/g) ; echo "update groups set desc='$descsql' where id='$id' ;"|sqlite3 $sqlitedb ); done
c999a3
	# groups/uservisible
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do uservisible=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v uservisible $xmlfile);isindb=$(echo "select uservisible from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$uservisible" || (echo "update groups set uservisible='$uservisible' where id='$id' ;"|sqlite3 $sqlitedb ); done
c999a3
	# groups/langonly
c999a3
	for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do langonly=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v langonly $xmlfile);isindb=$(echo "select langonly from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$langonly" || (echo "update groups set langonly='$langonly' where id='$id' ;"|sqlite3 $sqlitedb ); done
c999a3
	# groups/name_$lang
c999a3
	for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb|sed s/_arobas_/@/g);do for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile);do namelang=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v "name[@xml:lang='$lang']" $xmlfile); langsql=$(echo $lang|sed s/@/_arobas_/g) ; isindb=$(echo "select name_$langsql from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$namelang" || (namelangsql=$(echo $namelang|sed s/"'"/"''"/g) ; echo "update groups set name_$langsql='$namelangsql' where id='$id';"|sqlite3 $sqlitedb) ; done ; done
c999a3
	# groups/desc_$lang
c999a3
	for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb|sed s/_arobas_/@/g);do for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile);do desclang=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v "description[@xml:lang='$lang']" $xmlfile); langsql=$(echo $lang|sed s/@/_arobas_/g) ; isindb=$(echo "select desc_$langsql from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$desclang" || (desclangsql=$(echo $desclang|sed s/"'"/"''"/g) ; echo "update groups set desc_$langsql='$desclangsql' where id='$id';"|sqlite3 $sqlitedb) ; done ; done
c999a3
	# groups/packagelist
1158e0
	for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); 
1158e0
	  do 
1158e0
	  packagelist=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -c packagelist $xmlfile|sed '/packagelist/d')
1158e0
	  isindb=$(echo "select packagelist from groups where id='$id';"|sqlite3 $sqlitedb ) 
1158e0
	  if [ -z "$isindb" ] ; then
1158e0
	    echo "update groups set packagelist='$packagelist' where id='$id' ;"|sqlite3 $sqlitedb 
1158e0
	  else
1158e0
	    breakline='
1158e0
	    '
1158e0
	    newpkglist=${isindb}${breakline}${packagelist}
1158e0
	    toinsert=$(echo "$newpkglist"|sort|uniq)
1158e0
	    echo "update groups set packagelist='$toinsert' where id='$id' ;"|sqlite3 $sqlitedb 
1158e0
	  fi
1158e0
	done
c999a3
a786fb
	
a786fb
#final done for the xmlfile loop
a786fb
done
a786fb
}
c999a3
a786fb
dbupdate() {
c999a3
	# Fixing some groups that are by default active but have to be deselected on a CentOS install
c999a3
a786fb
    if [ "$release" = "6" ];then
a786fb
	 for groupid in client-mgmt-tools ha ha-management load-balancer resilient-storage scalable-file-systems ;do
a786fb
	    	echo "update groups set def='false' where id='$groupid';"|sqlite3 $sqlitedb
a786fb
	 done
a786fb
	elif [ "$release" = "5" ];then
a786fb
	 for groupid in admin-tools base-x games gnome-desktop graphical-internet graphics java legacy-software-support office printing sound-and-video cluster-storage clustering xen ;do
a786fb
	    	echo "update groups set def='false' where id='$groupid';"|sqlite3 $sqlitedb
a786fb
	 done 
1158e0
	 for groupid in base dialup ;do
1158e0
	    	echo "update groups set uservisible='true' where id='$groupid';"|sqlite3 $sqlitedb
1158e0
	 done 
a786fb
	fi 
c999a3
c999a3
}
c999a3
c999a3
compsxmlout() {
c999a3
echo Generating the $outputxml file ...
c999a3
echo "" >$outputxml
c999a3
echo "" >> $outputxml
c999a3
echo "<comps>" >> $outputxml
c999a3
c999a3
#group
c999a3
c999a3
for id in $(echo "select id from groups;"|sqlite3 $sqlitedb);do
c999a3
	echo "  <group>" >>$outputxml
c999a3
c999a3
	echo "   <id>$id</id>" >> $outputxml
c999a3
c999a3
	name=$(echo "select name from groups where id='$id';"|sqlite3 $sqlitedb)
c999a3
	echo "   <name>$name</name>" >> $outputxml
c999a3
c999a3
	for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb) ;do
c999a3
		namelang=$(echo "select name_$lang from groups where id='$id';"|sqlite3 $sqlitedb )
c999a3
		if [ -n "$namelang" ] ; then
c999a3
			langxml=$(echo $lang|sed s/_arobas_/@/g)
c999a3
			echo "   <name xml:lang='$langxml'>$namelang</name>" >> $outputxml
c999a3
		fi
c999a3
	done
c999a3
c999a3
	desc=$(echo "select desc from groups where id='$id';"|sqlite3 $sqlitedb)
c999a3
	if [ -n "$desc" ] ; then
c999a3
		echo "   <description>$desc</description>" >> $outputxml
c999a3
		for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb) ;do
c999a3
			desclang=$(echo "select desc_$lang from groups where id='$id';"|sqlite3 $sqlitedb )
c999a3
			if [ -n "$desclang" ] ; then
c999a3
				langxml=$(echo $lang|sed s/_arobas_/@/g)
c999a3
				echo "   <description xml:lang='$langxml'>$desclang</description>" >> $outputxml
c999a3
			fi
c999a3
		done
c999a3
	else
c999a3
		echo "   <description/>" >> $outputxml
c999a3
	fi
c999a3
c999a3
	def=$(echo "select def from groups where id='$id';"|sqlite3 $sqlitedb)
c999a3
	echo "   <default>$def</default>" >> $outputxml
c999a3
c999a3
	uservisible=$(echo "select uservisible from groups where id='$id';"|sqlite3 $sqlitedb)
c999a3
	echo "   <uservisible>$uservisible</uservisible>" >> $outputxml
c999a3
c999a3
	langonly=$(echo "select langonly from groups where id='$id';"|sqlite3 $sqlitedb)
c999a3
	if [ -n "$langonly" ] ; then
c999a3
		echo "   <langonly>$langonly</langonly>" >> $outputxml
c999a3
	fi
c999a3
c999a3
	plist=$(echo "select packagelist from groups where id='$id';"|sqlite3 $sqlitedb ) 
1158e0
	echo "   <packagelist>" >>$outputxml
c999a3
	echo "   $plist" >>$outputxml
1158e0
    echo "   </packagelist>" >>$outputxml
c999a3
	echo "  </group>" >>$outputxml
c999a3
done
c999a3
c999a3
#category
c999a3
c999a3
for id in $(echo "select id from categories;"|sqlite3 $sqlitedb);do
c999a3
	echo "  <category>" >>$outputxml
c999a3
c999a3
	echo "   <id>$id</id>" >> $outputxml
c999a3
c999a3
	name=$(echo "select name from categories where id='$id';"|sqlite3 $sqlitedb)
c999a3
	echo "   <name>$name</name>" >> $outputxml
c999a3
c999a3
	for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb) ;do
c999a3
		namelang=$(echo "select name_$lang from categories where id='$id';"|sqlite3 $sqlitedb )
c999a3
		if [ -n "$namelang" ] ; then
c999a3
			langxml=$(echo $lang|sed s/_arobas_/@/g)
c999a3
			echo "   <name xml:lang='$langxml'>$namelang</name>" >> $outputxml
c999a3
		fi
c999a3
	done
c999a3
c999a3
	desc=$(echo "select desc from categories where id='$id';"|sqlite3 $sqlitedb)
c999a3
	echo "   <description>$desc</description>" >> $outputxml
c999a3
c999a3
	for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb) ;do
c999a3
		desclang=$(echo "select desc_$lang from categories where id='$id';"|sqlite3 $sqlitedb )
c999a3
		if [ -n "$desclang" ] ; then
c999a3
			langxml=$(echo $lang|sed s/_arobas_/@/g)
c999a3
			echo "   <description xml:lang='$langxml'>$desclang</description>" >> $outputxml
c999a3
		fi
c999a3
	done
c999a3
c999a3
	echo "   <grouplist>" >>$outputxml
c999a3
	for groupid in $(echo "select groupid from grouplist where category_id='$id';"|sqlite3 $sqlitedb) ; do
c999a3
		echo "    <groupid>$groupid</groupid>" >>$outputxml
c999a3
	done
c999a3
	echo "   </grouplist>" >>$outputxml	
c999a3
c999a3
	echo "  </category>" >>$outputxml	
c999a3
c999a3
done
c999a3
echo "" >> $outputxml
c999a3
echo "</comps>" >> $outputxml
c999a3
c999a3
# Now fixing some well-known branding issues (as the new comps.xml is still generated from upstream xml files)
6c02c7
6c02c7
if [ "$release" = "6" ];then
6c02c7
	 sed -i s/"Red Hat Enterprise Linux"/"CentOS Linux"/g $outputxml
6c02c7
	 sed -i s/"redhat-indexhtml"/"centos-indexhtml"/g $outputxml
6c02c7
     sed -i '/Red_Hat_Enterprise_Linux-Release_Notes/d' $outputxml
6c02c7
     sed -i '/subscription-manager-gnome/d' $outputxml
6c02c7
     sed -i '/subscription-manager/d' $outputxml
6c02c7
     sed -i '/subscription-manager-firstboot/d' $outputxml
6c02c7
elif [ "$release" = "5" ];then
6c02c7
	 sed -i '/redhat-release-notes/d' $outputxml
6c02c7
	 sed -i s/"Red Hat"/"CentOS"/g $outputxml
6c02c7
	 sed -i s/"redhat-release"/"centos-release"/g $outputxml
6c02c7
	 
6c02c7
fi 
6c02c7
c999a3
}
c999a3
c999a3
c999a3
c999a3
sqlitedb="/dev/shm/comps-$release-$arch.db"
c999a3
outputxml="c$release-$arch-comps.xml"
c999a3
c999a3
c999a3
# and now the real processing ...
c999a3
varcheck $arch
c999a3
varcheck $destdir
c999a3
varcheck $release
c999a3
c999a3
for binary in sqlite3 xmlstarlet;do
c999a3
	which $binary > /dev/null 2>&1
c999a3
	if [ "$?" = "1" ] ;then
c999a3
		echo "Binary $binary not found ... exiting"
c999a3
		exit 1
c999a3
	fi
c999a3
	
c999a3
done
c999a3
c999a3
if [ -e $sqlitedb ] ;then
c999a3
	echo "$sqlitedb file already existing - not overwriting by default ..."
c999a3
	exit 1
c999a3
fi
c999a3
c999a3
echo sqlite file = $sqlitedb
c999a3
echo Generated comps xml file = $outputxml
c999a3
c999a3
time compsxmlin
a786fb
dbupdate
c999a3
time compsxmlout
c999a3