From c999a37c7d148aa254e7bd97084b8c7d4fab82b0 Mon Sep 17 00:00:00 2001 From: Fabian Arrotin Date: Aug 29 2011 13:53:18 +0000 Subject: Added first version of the compsgen.sh script (still be to fully tested !) --- diff --git a/tools/compsgen.sh b/tools/compsgen.sh new file mode 100755 index 0000000..989b0d8 --- /dev/null +++ b/tools/compsgen.sh @@ -0,0 +1,267 @@ +#!/bin/bash + +################################################################################# +# +# Name : CentOS comps.xml generator +# Function : merge all the upstream comps.xml file into one +# How to call it : see usage() +# Author : Fabian Arrotin (arrfab@centos.org) +# Requirements : sqlite3 / xmlstarlet +# +################################################################################# + +usage() { + +cat << EOF + +You need to call the script like this : $0 -arguments + + -a : define the arch (required, default:none, values : [i386,x86_64]) + -p : define the path containing all the Upstream comps.xml files for all variants(required, default:none) + -r : release to build ( eg. 6 ) + -h : display this help + +EOF + +} + + +while getopts “ha:p:m:r:” OPTION +do + case $OPTION in + h) + usage + exit 1 + ;; + a) + arch=$OPTARG + ;; + p) + destdir=$OPTARG + ;; + r) + release=$OPTARG + ;; + ?) + usage + exit + ;; + esac +done + +varcheck() { +if [ -z "$1" ] ; then + usage + exit 1 +fi + +} + +compsxmlin() { +echo Creating Comps DB tables ... +echo "CREATE TABLE categories (id ,name, desc, display_order ); CREATE TABLE languages (lang); CREATE TABLE groups (id, name, desc, def, uservisible, langonly, packagelist); CREATE TABLE grouplist (groupid, category_id); +" |sqlite3 $sqlitedb + +# beginning the loop for each xml file + +for xmlfile in $(ls $destdir/*.xml); + + do + + # just do that once + if [ "$aretableready" = "1" ] ; then + echo Tables are already there + else + echo Detecting languages and preparing - altering the tables ... + for lang in $(grep 'description xml:lang=' $xmlfile|sed s/@/_arobas_/g|cut -f 1 -d '>'|cut -f 2 -d "'"|sort|uniq);do echo "insert into languages values ('$lang') ;"|sqlite3 $sqlitedb;done + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb);do echo "alter table categories add desc_$lang ; alter table categories add name_$lang ; "|sqlite3 $sqlitedb;done + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb);do echo "alter table groups add desc_$lang ; alter table groups add name_$lang ; "|sqlite3 $sqlitedb;done + export aretableready='1' + fi + + + echo Processing now $xmlfile + + echo Detecting categories and filling the tables ... + # categories/id + for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile);do isindb=$(echo "select id from categories where id='$id';"|sqlite3 $sqlitedb ) ;test "$isindb" = "$id" || (echo "insert into categories (id) values ('$id');"|sqlite3 $sqlitedb);done + # categories/name + for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile); do name=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v name $xmlfile);isindb=$(echo "select name from categories where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$name" || (echo "update categories set name='$name' where id='$id' ;"|sqlite3 $sqlitedb ); done + # categories/description + for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile); do desc=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v description $xmlfile);echo "update categories set desc='$desc' where id='$id' ;"|sqlite3 $sqlitedb; done + # categories/display_order + for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile); do display_order=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v display_order $xmlfile); isindb=$(echo "select display_order from categories where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$display_order" || (echo "update categories set display_order='$display_order' where id='$id' ;"|sqlite3 $sqlitedb); done + # categories/name_$lang + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb|sed s/_arobas_/@/g);do for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile);do namelang=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v "name[@xml:lang='$lang']" $xmlfile); langsql=$(echo $lang|sed s/@/_arobas_/g) ; isindb=$(echo "select name_$langsql from categories where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$namelang" || (namelangsql=$(echo $namelang|sed s/"'"/"''"/g) ;echo "update categories set name_$langsql='$namelangsql' where id='$id';"|sqlite3 $sqlitedb) ; done ; done + # categories/description_$lang + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb|sed s/_arobas_/@/g);do for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile);do desclang=$(xmlstarlet sel -t -m "//comps/category[id='$id']" -v "description[@xml:lang='$lang']" $xmlfile); langsql=$(echo $lang|sed s/@/_arobas_/g) ; isindb=$(echo "select desc_$langsql from categories where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$desclang" || (desclangsql=$(echo $desclang|sed s/"'"/"''"/g) ;echo "update categories set desc_$langsql='$desclangsql' where id='$id';"|sqlite3 $sqlitedb) ; done ; done + # categories/grouplist + for id in $(xmlstarlet sel -t -m "//comps/category" -v id -n $xmlfile); do for groupid in $(xmlstarlet sel -t -m "//comps/category[id='$id']" -v grouplist $xmlfile); do isindb=$(echo "select groupid from grouplist where category_id='$id' and groupid='$groupid';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$groupid" || (echo "insert into grouplist (groupid, category_id) values ('$groupid','$id');"|sqlite3 $sqlitedb); done ; done + + echo Detecting groups and filling the tables ... + # groups/id + for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile);do isindb=$(echo "select id from groups where id='$id';"|sqlite3 $sqlitedb ) ;test "$isindb" = "$id" || (echo "insert into groups (id) values ('$id');"|sqlite3 $sqlitedb);done + # groups/name + for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do name=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v name $xmlfile);isindb=$(echo "select name from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$name" || (echo "update groups set name='$name' where id='$id' ;"|sqlite3 $sqlitedb ); done + # groups/default + for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do def=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v default $xmlfile);isindb=$(echo "select def from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$def" || (echo "update groups set def='$def' where id='$id' ;"|sqlite3 $sqlitedb ); done + # groups/description + for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do desc=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v description $xmlfile);isindb=$(echo "select desc from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$desc" || (descsql=$(echo $desc|sed s/"'"/"''"/g) ; echo "update groups set desc='$descsql' where id='$id' ;"|sqlite3 $sqlitedb ); done + # groups/uservisible + for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do uservisible=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v uservisible $xmlfile);isindb=$(echo "select uservisible from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$uservisible" || (echo "update groups set uservisible='$uservisible' where id='$id' ;"|sqlite3 $sqlitedb ); done + # groups/langonly + for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do langonly=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v langonly $xmlfile);isindb=$(echo "select langonly from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$langonly" || (echo "update groups set langonly='$langonly' where id='$id' ;"|sqlite3 $sqlitedb ); done + # groups/name_$lang + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb|sed s/_arobas_/@/g);do for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile);do namelang=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v "name[@xml:lang='$lang']" $xmlfile); langsql=$(echo $lang|sed s/@/_arobas_/g) ; isindb=$(echo "select name_$langsql from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$namelang" || (namelangsql=$(echo $namelang|sed s/"'"/"''"/g) ; echo "update groups set name_$langsql='$namelangsql' where id='$id';"|sqlite3 $sqlitedb) ; done ; done + # groups/desc_$lang + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb|sed s/_arobas_/@/g);do for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile);do desclang=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -v "description[@xml:lang='$lang']" $xmlfile); langsql=$(echo $lang|sed s/@/_arobas_/g) ; isindb=$(echo "select desc_$langsql from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$desclang" || (desclangsql=$(echo $desclang|sed s/"'"/"''"/g) ; echo "update groups set desc_$langsql='$desclangsql' where id='$id';"|sqlite3 $sqlitedb) ; done ; done + # groups/packagelist + for id in $(xmlstarlet sel -t -m "//comps/group" -v id -n $xmlfile); do packagelist=$(xmlstarlet sel -t -m "//comps/group[id='$id']" -c packagelist $xmlfile);isindb=$(echo "select packagelist from groups where id='$id';"|sqlite3 $sqlitedb ) ; test "$isindb" = "$packagelist" || (echo "update groups set packagelist='$packagelist' where id='$id' ;"|sqlite3 $sqlitedb ); done + + + # Fixing some groups that are by default active but have to be deselected on a CentOS install + + for groupid in ha ha-management load-balancer resilient-storage scalable-file-systems ;do + echo "update groups set def='false' where id='$groupid';"|sqlite3 $sqlitedb + done + + +#final done for the xmlfile loop +done +} + + +compsxmlout() { +echo Generating the $outputxml file ... +echo "" >$outputxml +echo "" >> $outputxml +echo "" >> $outputxml + +#group + +for id in $(echo "select id from groups;"|sqlite3 $sqlitedb);do + echo " " >>$outputxml + + echo " $id" >> $outputxml + + name=$(echo "select name from groups where id='$id';"|sqlite3 $sqlitedb) + echo " $name" >> $outputxml + + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb) ;do + namelang=$(echo "select name_$lang from groups where id='$id';"|sqlite3 $sqlitedb ) + if [ -n "$namelang" ] ; then + langxml=$(echo $lang|sed s/_arobas_/@/g) + echo " $namelang" >> $outputxml + fi + done + + desc=$(echo "select desc from groups where id='$id';"|sqlite3 $sqlitedb) + if [ -n "$desc" ] ; then + echo " $desc" >> $outputxml + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb) ;do + desclang=$(echo "select desc_$lang from groups where id='$id';"|sqlite3 $sqlitedb ) + if [ -n "$desclang" ] ; then + langxml=$(echo $lang|sed s/_arobas_/@/g) + echo " $desclang" >> $outputxml + fi + done + else + echo " " >> $outputxml + fi + + def=$(echo "select def from groups where id='$id';"|sqlite3 $sqlitedb) + echo " $def" >> $outputxml + + uservisible=$(echo "select uservisible from groups where id='$id';"|sqlite3 $sqlitedb) + echo " $uservisible" >> $outputxml + + langonly=$(echo "select langonly from groups where id='$id';"|sqlite3 $sqlitedb) + if [ -n "$langonly" ] ; then + echo " $langonly" >> $outputxml + fi + + plist=$(echo "select packagelist from groups where id='$id';"|sqlite3 $sqlitedb ) + echo " $plist" >>$outputxml + + echo " " >>$outputxml +done + +#category + +for id in $(echo "select id from categories;"|sqlite3 $sqlitedb);do + echo " " >>$outputxml + + echo " $id" >> $outputxml + + name=$(echo "select name from categories where id='$id';"|sqlite3 $sqlitedb) + echo " $name" >> $outputxml + + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb) ;do + namelang=$(echo "select name_$lang from categories where id='$id';"|sqlite3 $sqlitedb ) + if [ -n "$namelang" ] ; then + langxml=$(echo $lang|sed s/_arobas_/@/g) + echo " $namelang" >> $outputxml + fi + done + + desc=$(echo "select desc from categories where id='$id';"|sqlite3 $sqlitedb) + echo " $desc" >> $outputxml + + for lang in $(echo "select * from languages;" |sqlite3 $sqlitedb) ;do + desclang=$(echo "select desc_$lang from categories where id='$id';"|sqlite3 $sqlitedb ) + if [ -n "$desclang" ] ; then + langxml=$(echo $lang|sed s/_arobas_/@/g) + echo " $desclang" >> $outputxml + fi + done + + echo " " >>$outputxml + for groupid in $(echo "select groupid from grouplist where category_id='$id';"|sqlite3 $sqlitedb) ; do + echo " $groupid" >>$outputxml + done + echo " " >>$outputxml + + echo " " >>$outputxml + +done +echo "" >> $outputxml +echo "" >> $outputxml + +# Now fixing some well-known branding issues (as the new comps.xml is still generated from upstream xml files) +sed -i s/"Red Hat Enterprise Linux"/"CentOS Linux"/g $outputxml +sed -i s/"redhat-indexhtml"/"centos-indexhtml"/g $outputxml +sed -i '/Red_Hat_Enterprise_Linux-Release_Notes/d' $outputxml + +} + + + +sqlitedb="/dev/shm/comps-$release-$arch.db" +outputxml="c$release-$arch-comps.xml" + + +# and now the real processing ... +varcheck $arch +varcheck $destdir +varcheck $release + +for binary in sqlite3 xmlstarlet;do + which $binary > /dev/null 2>&1 + if [ "$?" = "1" ] ;then + echo "Binary $binary not found ... exiting" + exit 1 + fi + +done + +if [ -e $sqlitedb ] ;then + echo "$sqlitedb file already existing - not overwriting by default ..." + exit 1 +fi + +echo sqlite file = $sqlitedb +echo Generated comps xml file = $outputxml + +time compsxmlin +time compsxmlout +