#9 Make the get_sources shell script support flat layout
Merged 8 months ago by bstinson. Opened 8 months ago by pingou.
pingou/centos-git-common new_get_sources  into  master

file modified
+130 -85
@@ -9,6 +9,7 @@ 

  #                  Pat Riehecky <riehecky@fnal.gov>

  #                  Tyler Parsons <tparsons@fnal.gov>

  #                  Tuomo Soini <tis@foobar.fi>

+ set -eux

  

  

  #####################################################################
@@ -114,49 +115,10 @@ 

      exit 1

  fi

  

- # should go into a function section at some point

- weakHashDetection () {

-   strHash=${1};

-   case $((`echo "${strHash}"|wc -m` - 1 )) in

-     128)

-       hashBin='sha512sum'

-       ;;

-     64)

-       hashBin='sha256sum'

-       ;;

-     40)

-       hashBin='sha1sum'

-       ;;

-     32)

-       hashBin='md5sum'

-       ;;

-     *)

-       hashBin='unknown'

-       ;;

-   esac

-   echo ${hashBin};

- }

- 

- # check metadata file and extract package name

- shopt -s nullglob

- set -- .*.metadata

- if (( $# == 0 ))

- then

-     echo 'Missing metadata. Please run from inside a sources git repo' >&2

-     exit 1

- elif (( $# > 1 ))

- then

-     echo "Warning: multiple metadata files found. Using $1"

- fi

- meta=$1

- pn=${meta%.metadata}

- pn=${pn#.}

- 

- if [ ! -d .git ] || [ ! -d SPECS ]; then

-   echo 'You need to run this from inside a sources git repo' >&2

-   exit 1

- fi

- mkdir -p SOURCES

+ if [ ! -d .git ] && ([ ! -d SPECS ] || [[ ! -s sources ]] ); then

+       echo 'You need to run this from inside a sources git repo' >&2

+       exit 1

+     fi

  

  # sort out our branch

  if [ -n "$BRANCH" ]
@@ -181,51 +143,134 @@ 

      else

        branches=("${branches[@]}" "$branch")

      fi

-   done <<< "$(git branch -r --contains HEAD | grep '^\s\+origin/'| sed 's#origin/##g')"

+   done <<< "$(git branch -r --contains HEAD | sed 's#origin/##g')"

  fi

- while read -r fsha fname ; do

-   if [ ".${fsha}" = ".da39a3ee5e6b4b0d3255bfef95601890afd80709" ]; then

-     # zero byte file

-     touch "${fname}"

-   else

-     if [ ${CHECK} -eq 1 ]; then

-       hashType=$(weakHashDetection "${fsha}")

-       if [ "${hashType}" == "unknown" ]; then

-         echo 'Failure: Hash type unknown.' >&2

-         exit 1;

-       else

-         command -v "${hashType}" >/dev/null 2>&1

-         if [[ $? -ne 0 ]]; then

-           echo "Failure: You need ${hashType} in PATH." >&2

-           exit 1;

+ 

+ if [[ -s sources ]]; then

+     # This section is for the "flat" dist-git layout, where the spec file and

+     # patches are all present at the top level directory and the sha of the tarball

+     # present in a 'sources' file.

+     # This code was re-used from the fedpkg-pkg minimal project which is licensed

+     # under GPLv3 or any later version.

+ 

+     pkgname=$(basename "$PWD")

+     # Read first word of first line. For old MD5 format it's the 32 character

+     # hash. Otherwise let's assume the sources have the BSD format where lines

+     # start with hash type.

+     hashtype="$(head -n1 sources | cut -d' ' -f1 | tr '[:upper:]' '[:lower:]')"

+     # The format is

+     #   SHA512 (filename) = ABCDEF

+     # We don't care about the equals sign. We also assume all hashes are

+     # the same type, so we don't need to read it again for each line.

+     while read -r _ filename _ hash || [[ -n "$filename" && -n "$hash" ]]; do

+         if [ -z "$filename" ] || [ -z "$hash" ]; then

+             continue

          fi

-       fi

-     fi

-     if [ -e ${fname} -a ${CHECK} -eq 1 ]; then

- 	# check hash sum and force download if wrong

-         downsum=$(${hashType} "${fname}" | awk '{print $1}')

-         if [ "${fsha}" != "${downsum}" ]; then

-             rm -f "${fname}"

+         # Remove parenthesis around tarball name

+         filename=${filename#(}

+         tarball=${filename%)}

+         if [ ! -e "$tarball" ]; then

+           for br in "${branches[@]}"

+           do

+             br=$(echo ${br}| sed -e s'|remotes/origin/||')

+             url="${SURL}/$pkgname/${br}/$hash"

+             echo "Retrieving ${url}"

+             curl -L ${QUIET} -f "${url}" -o "$tarball" && break

+           done

+         else

+           echo "$filename exists. skipping"

          fi

+     done < sources

+     "${hashtype}sum" -c sources

+ else

+     # This section is for the "non-flat" dist-git layout, where the spec file

+     # is stored in a SPECS folder, the patches in a SOURCES folder and the sha

+     # of the tarball of the project is present in a '.<pkg_name>.metadata' file.

+ 

+     mkdir -p SOURCES

+     # should go into a function section at some point

+     weakHashDetection () {

+       strHash=${1};

+       case $((`echo ${strHash}|wc -m` - 1 )) in

+         128)

+           hashBin='sha512sum'

+           ;;

+         64)

+           hashBin='sha256sum'

+           ;;

+         40)

+           hashBin='sha1sum'

+           ;;

+         32)

+           hashBin='md5sum'

+           ;;

+         *)

+           hashBin='unknown'

+           ;;

+       esac

+       echo ${hashBin};

+     }

+ 

+     # check metadata file and extract package name

+     shopt -s nullglob

+     set -- .*.metadata

+     if (( $# == 0 ))

+     then

+         echo 'Missing metadata. Please run from inside a sources git repo' >&2

+         exit 1

+     elif (( $# > 1 ))

+     then

+         echo "Warning: multiple metadata files found. Using $1"

      fi

-     if [ ! -e "${fname}" ]; then

-       for br in "${branches[@]}"

-       do

-         br=$(echo "${br}"| sed -e s'|remotes/origin/||')

-         url="${SURL}/${pn}/${br}/${fsha}"

-         echo "Retrieving ${url}"

-         curl -L ${QUIET} -f "${url}" -o "${fname}" && break

-       done

-     else

-       echo "${fname} exists. skipping"

-     fi

-     if [ ${CHECK} -eq 1 ]; then

-         downsum=$(${hashType} "${fname}" | awk '{print $1}')

-         if [ "${fsha}" != "${downsum}" ]; then

-             rm -f "${fname}"

-             echo "Failure: ${fname} hash does not match hash from the .metadata file" >&2

+     meta=$1

+     pn=${meta%.metadata}

+     pn=${pn#.}

+ 

+     while read -r fsha fname ; do

+       if [ ".${fsha}" = ".da39a3ee5e6b4b0d3255bfef95601890afd80709" ]; then

+         # zero byte file

+         touch ${fname}

+       else

+         if [ ${CHECK} -eq 1 ]; then

+           hashType=$(weakHashDetection ${fsha})

+           if [ "${hashType}" == "unknown" ]; then

+             echo 'Failure: Hash type unknown.' >&2

              exit 1;

+           else

+             which ${hashType} >/dev/null 2>&1

+             if [[ $? -ne 0 ]]; then

+               echo "Failure: You need ${hashType} in PATH." >&2

+               exit 1;

+             fi

+           fi

          fi

-     fi

-   fi

- done < "${meta}"

+         if [ -e ${fname} -a ${CHECK} -eq 1 ]; then

+ 	    # check hash sum and force download if wrong

+             downsum=$(${hashType} ${fname} | awk '{print $1}')

+             if [ "${fsha}" != "${downsum}" ]; then

+                 rm -f ${fname}

+             fi

+         fi

+         if [ ! -e "${fname}" ]; then

+           for br in "${branches[@]}"

+           do

+             br=$(echo ${br}| sed -e s'|remotes/origin/||')

+             url="${SURL}/${pn}/${br}/${fsha}"

+             echo "Retrieving ${url}"

+             curl -L ${QUIET} -f "${url}" -o "${fname}" && break

+           done

+         else

+           echo "${fname} exists. skipping"

+         fi

+         if [ ${CHECK} -eq 1 ]; then

+             downsum=$(${hashType} ${fname} | awk '{print $1}')

+             if [ "${fsha}" != "${downsum}" ]; then

+                 rm -f ${fname}

+                 echo "Failure: ${fname} hash does not match hash from the .metadata file" >&2

+                 exit 1;

+             fi

+         fi

+       fi

+     done < "${meta}"

+ 

+ fi

The idea of this commit is to make get_sources.sh support both the
'traditional' git layout used on dist-git for CentOS Linux where the
spec files are stored in a SPECS folder and the patches in a SOURCES
folder with the sha of the tarball being in a '.<pkg_name>.metadata'
file as well as the 'flat' git layout that have adopted Fedora as well
as CentOS-Stream (9+) where the spec files and patches are all stored
at the top level directory of the repository and the sha of the tarball
is present in a 'sources' file.

This commit re-uses code from the fedpkg-minimal project which is
license under the GPL (so the License field of the spec file for
centpkg-minimal may need to be adjusted to MIT and GPL).

Signed-off-by: Pierre-Yves Chibon pingou@pingoured.fr

LGTM

I tested the get_sources.sh script using the time package from both https://git.centos.org and https://src.fedoraproject.org and I was able to get the source for that package using for both structures.

Pull-Request has been merged by bstinson

8 months ago
Metadata