Blame get_sources.sh

Karanbir Singh afa935
#!/bin/bash
Karanbir Singh afa935
#
Karanbir Singh afa935
# Might want to drop this in ~/bin/ and chmod u+x it
645621
#
645621
645621
#  Initial Author: Karanbir Singh <kbsingh@centos.org>
645621
#         Updates:
645621
#                  Mike McLean <mikem@redhat.com>
645621
#                  Pat Riehecky <riehecky@fnal.gov>
Tyler Parsons d96c00
#                  Tyler Parsons <tparsons@fnal.gov>
Tuomo Soini bd83a7
#                  Tuomo Soini <tis@foobar.fi>
518994
set -eux
645621
645621
645621
#####################################################################
645621
usage() {
645621
    echo ''                                               >&2
645621
    echo "$0 [-hcq] [-b branch] [--surl url]"             >&2
645621
    echo ''                                               >&2
645621
    echo 'Script to parse the non-text sources metadata file'   >&2
645621
    echo ' and download the required files from the lookaside'  >&2
645621
    echo ' cache.'                                              >&2
645621
    echo ''                                                     >&2
645621
    echo 'PLEASE NOTE: this script is non-destructive, it wont' >&2
645621
    echo ' replace files that already exist, regardless of'     >&2
645621
    echo ' their state, allowing you to have work-in-progress'  >&2
645621
    echo ' content that wont get overwritten.'                  >&2
645621
    echo ''                                                     >&2
645621
    echo 'You need to run this from inside a sources git repo'  >&2
645621
    echo ''                                               >&2
645621
    echo ' -h: This help message'                         >&2
645621
    echo ''                                               >&2
645621
    echo "  $0 -b c7"                                     >&2
645621
    echo "  $0 -q -b c7"                                  >&2
645621
    echo "  $0 -c -b remotes/origin/c7"                   >&2
645621
    echo "  $0 -c -b c7 --surl '$SURL'"                   >&2
645621
    echo "  $0"                                           >&2
645621
    exit 1
645621
}
Karanbir Singh afa935
645621
#####################################################################
Karanbir Singh afa935
696c29
SURL="https://git.centos.org/sources"
645621
645621
QUIET=0
645621
BRANCH=''
645621
CHECK=0
645621
645621
# for setting any overrides, such as SURL, default BRANCH, or force CHECK
1842fe
if [ -f /etc/centos-git-common ]; then
1842fe
  . /etc/centos-git-common
1842fe
fi
1842fe
645621
#####################################################################
645621
# setup args in the right order for making getopt evaluation
645621
# nice and easy.  You'll need to read the manpages for more info
645621
# utilizing 'while' construct rather than 'for arg' to avoid unnecessary
645621
# shifting of program args
645621
args=$(getopt -o hcqb: -l surl: -- "$@")
645621
eval set -- "$args"
645621
645621
while [[ 0 -eq 0 ]]; do
645621
    case $1 in
645621
        -- )
645621
            # end of getopt args, shift off the -- and get out of the loop
645621
            shift
645621
            break
645621
           ;;
645621
         -c )
645621
            # verify the sha1sum of the downloaded file
645621
            CHECK=1
645621
            shift
645621
           ;;
645621
         -q )
645621
            # suppress warnings
645621
            QUIET=1
645621
            shift
645621
           ;;
645621
         -b )
014e38
            # Check this particular branch
645621
            BRANCH=$2
645621
            shift
645621
            shift
645621
           ;;
645621
         --surl )
645621
            # override sources url
645621
            SURL=$2
645621
            shift
645621
            shift
645621
           ;;
645621
         -h )
645621
            # get help
645621
            usage
645621
           ;;
645621
    esac
c61c1c
done
c61c1c
645621
# set curl options this way so defaults can be set in /etc/centos-git-common
645621
# across multiple scripts
645621
if [[ ${QUIET} -eq 1 ]]; then
645621
    QUIET='--silent'
645621
else
645621
    QUIET=''
645621
fi
645621
fd37ab
command -v git >/dev/null 2>&1
645621
if [[ $? -ne 0 ]]; then
645621
    echo 'You need git in PATH' >&2
645621
    exit 1
645621
fi
645621
fd37ab
command -v curl >/dev/null 2>&1
645621
if [[ $? -ne 0 ]]; then
645621
    echo 'You need curl in PATH' >&2
645621
    exit 1
645621
fi
645621
518994
if [ ! -d .git ] && ([ ! -d SPECS ] || [[ ! -s sources ]] ); then
518994
      echo 'You need to run this from inside a sources git repo' >&2
518994
      exit 1
518994
    fi
c61c1c
c61c1c
# sort out our branch
c61c1c
if [ -n "$BRANCH" ]
c61c1c
then
c61c1c
  branches=("$BRANCH")
c61c1c
else
c61c1c
  # generate a list of all branches containing current HEAD
c61c1c
  branches=()
c61c1c
  while IFS='' read -r line
c61c1c
  do
c61c1c
    # input from: git branch --contains HEAD
c61c1c
    branch="${line:2}"
c61c1c
    if [[ "$branch" =~ "detached from" ]]
c61c1c
    then
c61c1c
      # ignore detached heads
c61c1c
      continue
c61c1c
    fi
c61c1c
    if [ ".${line:0:1}" = ".*" ]
c61c1c
    then
c61c1c
      # current branch, put it first
ae9a7e
      branches=("${branch}" ${branches[@]+"${branches[@]}"})
c61c1c
    else
ae9a7e
      branches=(${branches[@]+"${branches[@]}"} "${branch}")
c61c1c
    fi
518994
  done <<< "$(git branch -r --contains HEAD | sed 's#origin/##g')"
c61c1c
fi
518994
518994
if [[ -s sources ]]; then
518994
    # This section is for the "flat" dist-git layout, where the spec file and
518994
    # patches are all present at the top level directory and the sha of the tarball
518994
    # present in a 'sources' file.
518994
    # This code was re-used from the fedpkg-pkg minimal project which is licensed
518994
    # under GPLv3 or any later version.
518994
518994
    pkgname=$(basename "$PWD")
518994
    # Read first word of first line. For old MD5 format it's the 32 character
518994
    # hash. Otherwise let's assume the sources have the BSD format where lines
518994
    # start with hash type.
518994
    hashtype="$(head -n1 sources | cut -d' ' -f1 | tr '[:upper:]' '[:lower:]')"
518994
    # The format is
518994
    #   SHA512 (filename) = ABCDEF
518994
    # We don't care about the equals sign. We also assume all hashes are
518994
    # the same type, so we don't need to read it again for each line.
518994
    while read -r _ filename _ hash || [[ -n "$filename" && -n "$hash" ]]; do
518994
        if [ -z "$filename" ] || [ -z "$hash" ]; then
518994
            continue
Tyler Parsons d96c00
        fi
518994
        # Remove parenthesis around tarball name
518994
        filename=${filename#(}
518994
        tarball=${filename%)}
518994
        if [ ! -e "$tarball" ]; then
518994
          for br in "${branches[@]}"
518994
          do
518994
            br=$(echo ${br}| sed -e s'|remotes/origin/||')
014e38
            # Try the branch-specific lookaside structure
518994
            url="${SURL}/$pkgname/${br}/$hash"
518994
            echo "Retrieving ${url}"
014e38
            HTTP_CODE=$(curl -L ${QUIET} -H Pragma: -o "./$tarball" -R -S --fail --retry 5 "${url}" --write-out "%{http_code}" || true)
014e38
            echo "Returned ${HTTP_CODE}"
014e38
            if [[ ${HTTP_CODE} -gt 199 && ${HTTP_CODE} -lt 300 ]] ; then
014e38
               echo "bailing"
014e38
               break
014e38
            fi
014e38
            # Try the hash-specific lookaside structure
014e38
            url="${SURL}/$pkgname/$tarball/$hashtype/$hash/$tarball"
014e38
            echo "Retrieving ${url}"
014e38
            curl -L ${QUIET} -H Pragma: -o "./$tarball" -R -S --fail --retry 5 "${url}" && break
518994
          done
518994
        else
518994
          echo "$filename exists. skipping"
Tuomo Soini bd83a7
        fi
518994
    done < sources
518994
    "${hashtype}sum" -c sources
518994
else
518994
    # This section is for the "non-flat" dist-git layout, where the spec file
518994
    # is stored in a SPECS folder, the patches in a SOURCES folder and the sha
518994
    # of the tarball of the project is present in a '.<pkg_name>.metadata' file.
518994
518994
    mkdir -p SOURCES
518994
    # should go into a function section at some point
518994
    weakHashDetection () {
518994
      strHash=${1};
518994
      case $((`echo ${strHash}|wc -m` - 1 )) in
518994
        128)
518994
          hashBin='sha512sum'
518994
          ;;
518994
        64)
518994
          hashBin='sha256sum'
518994
          ;;
518994
        40)
518994
          hashBin='sha1sum'
518994
          ;;
518994
        32)
518994
          hashBin='md5sum'
518994
          ;;
518994
        *)
518994
          hashBin='unknown'
518994
          ;;
518994
      esac
518994
      echo ${hashBin};
518994
    }
518994
518994
    # check metadata file and extract package name
518994
    shopt -s nullglob
518994
    set -- .*.metadata
518994
    if (( $# == 0 ))
518994
    then
518994
        echo 'Missing metadata. Please run from inside a sources git repo' >&2
518994
        exit 1
518994
    elif (( $# > 1 ))
518994
    then
518994
        echo "Warning: multiple metadata files found. Using $1"
Tuomo Soini bd83a7
    fi
518994
    meta=$1
518994
    pn=${meta%.metadata}
518994
    pn=${pn#.}
518994
518994
    while read -r fsha fname ; do
518994
      if [ ".${fsha}" = ".da39a3ee5e6b4b0d3255bfef95601890afd80709" ]; then
518994
        # zero byte file
518994
        touch ${fname}
518994
      else
014e38
        hashType=$(weakHashDetection ${fsha})
014e38
        if [ "${hashType}" == "unknown" ]; then
014e38
          echo 'Failure: Hash type unknown.' >&2
014e38
          exit 1;
014e38
        fi
014e38
        hashName=$(echo ${hashType}| sed -e s'|sum||')
014e38
518994
        if [ ${CHECK} -eq 1 ]; then
014e38
          which ${hashType} >/dev/null 2>&1
014e38
          if [[ $? -ne 0 ]]; then
014e38
            echo "Failure: You need ${hashType} in PATH." >&2
Tyler Parsons d96c00
            exit 1;
518994
          fi
Pat Riehecky 061172
        fi
518994
        if [ -e ${fname} -a ${CHECK} -eq 1 ]; then
014e38
            # check hash sum and force download if wrong
518994
            downsum=$(${hashType} ${fname} | awk '{print $1}')
518994
            if [ "${fsha}" != "${downsum}" ]; then
518994
                rm -f ${fname}
518994
            fi
518994
        fi
518994
        if [ ! -e "${fname}" ]; then
518994
          for br in "${branches[@]}"
518994
          do
518994
            br=$(echo ${br}| sed -e s'|remotes/origin/||')
014e38
            # Try the branch-specific lookaside structure
518994
            url="${SURL}/${pn}/${br}/${fsha}"
518994
            echo "Retrieving ${url}"
014e38
            HTTP_CODE=$(curl -L ${QUIET} -H Pragma: -o "${fname}" -R -S --fail --retry 5 "${url}" --write-out "%{http_code}" || true)
014e38
            echo "Returned ${HTTP_CODE}"
014e38
            if [[ ${HTTP_CODE} -gt 199 && ${HTTP_CODE} -lt 300 ]] ; then
014e38
               echo "bailing"
014e38
               break
014e38
            fi
014e38
            # Try the hash-specific lookaside structure
014e38
            url="${SURL}/$pn/$fname/${hashName}/$fsha/$fname"
014e38
            echo "Retrieving ${url}"
014e38
            curl -L ${QUIET} -H Pragma: -o "${fname}" -R -S --fail --retry 5 "${url}" && break
518994
          done
518994
        else
518994
          echo "${fname} exists. skipping"
518994
        fi
518994
        if [ ${CHECK} -eq 1 ]; then
518994
            downsum=$(${hashType} ${fname} | awk '{print $1}')
518994
            if [ "${fsha}" != "${downsum}" ]; then
518994
                rm -f ${fname}
518994
                echo "Failure: ${fname} hash does not match hash from the .metadata file" >&2
518994
                exit 1;
518994
            fi
518994
        fi
518994
      fi
518994
    done < "${meta}"
518994
518994
fi