#11 Import lookaside_upload and get_sources.sh from centos-git-common
Opened 6 months ago by dcavalca. Modified 6 months ago
centos/ dcavalca/centos-packager centos-git-common  into  master

file added
+231

@@ -0,0 +1,231 @@ 

+ #!/bin/bash

+ #

+ # Might want to drop this in ~/bin/ and chmod u+x it

+ #

+ 

+ #  Initial Author: Karanbir Singh <kbsingh@centos.org>

+ #         Updates:

+ #                  Mike McLean <mikem@redhat.com>

+ #                  Pat Riehecky <riehecky@fnal.gov>

+ #                  Tyler Parsons <tparsons@fnal.gov>

+ #                  Tuomo Soini <tis@foobar.fi>

+ 

+ 

+ #####################################################################

+ usage() {

+     echo ''                                               >&2

+     echo "$0 [-hcq] [-b branch] [--surl url]"             >&2

+     echo ''                                               >&2

+     echo 'Script to parse the non-text sources metadata file'   >&2

+     echo ' and download the required files from the lookaside'  >&2

+     echo ' cache.'                                              >&2

+     echo ''                                                     >&2

+     echo 'PLEASE NOTE: this script is non-destructive, it wont' >&2

+     echo ' replace files that already exist, regardless of'     >&2

+     echo ' their state, allowing you to have work-in-progress'  >&2

+     echo ' content that wont get overwritten.'                  >&2

+     echo ''                                                     >&2

+     echo 'You need to run this from inside a sources git repo'  >&2

+     echo ''                                               >&2

+     echo ' -h: This help message'                         >&2

+     echo ''                                               >&2

+     echo "  $0 -b c7"                                     >&2

+     echo "  $0 -q -b c7"                                  >&2

+     echo "  $0 -c -b remotes/origin/c7"                   >&2

+     echo "  $0 -c -b c7 --surl '$SURL'"                   >&2

+     echo "  $0"                                           >&2

+     exit 1

+ }

+ 

+ #####################################################################

+ 

+ SURL="https://git.centos.org/sources"

+ 

+ QUIET=0

+ BRANCH=''

+ CHECK=0

+ 

+ # for setting any overrides, such as SURL, default BRANCH, or force CHECK

+ if [ -f /etc/centos-git-common ]; then

+   . /etc/centos-git-common

+ fi

+ 

+ #####################################################################

+ # setup args in the right order for making getopt evaluation

+ # nice and easy.  You'll need to read the manpages for more info

+ # utilizing 'while' construct rather than 'for arg' to avoid unnecessary

+ # shifting of program args

+ args=$(getopt -o hcqb: -l surl: -- "$@")

+ eval set -- "$args"

+ 

+ while [[ 0 -eq 0 ]]; do

+     case $1 in

+         -- )

+             # end of getopt args, shift off the -- and get out of the loop

+             shift

+             break

+            ;;

+          -c )

+             # verify the sha1sum of the downloaded file

+             CHECK=1

+             shift

+            ;;

+          -q )

+             # suppress warnings

+             QUIET=1

+             shift

+            ;;

+          -b )

+             # Check this particular branch 

+             BRANCH=$2

+             shift

+             shift

+            ;;

+          --surl )

+             # override sources url

+             SURL=$2

+             shift

+             shift

+            ;;

+          -h )

+             # get help

+             usage

+            ;;

+     esac

+ done

+ 

+ # set curl options this way so defaults can be set in /etc/centos-git-common

+ # across multiple scripts

+ if [[ ${QUIET} -eq 1 ]]; then

+     QUIET='--silent'

+ else

+     QUIET=''

+ fi

+ 

+ command -v git >/dev/null 2>&1

+ if [[ $? -ne 0 ]]; then

+     echo 'You need git in PATH' >&2

+     exit 1

+ fi

+ 

+ command -v curl >/dev/null 2>&1

+ if [[ $? -ne 0 ]]; then

+     echo 'You need curl in PATH' >&2

+     exit 1

+ fi

+ 

+ # should go into a function section at some point

+ weakHashDetection () {

+   strHash=${1};

+   case $((`echo "${strHash}"|wc -m` - 1 )) in

+     128)

+       hashBin='sha512sum'

+       ;;

+     64)

+       hashBin='sha256sum'

+       ;;

+     40)

+       hashBin='sha1sum'

+       ;;

+     32)

+       hashBin='md5sum'

+       ;;

+     *)

+       hashBin='unknown'

+       ;;

+   esac

+   echo ${hashBin};

+ }

+ 

+ # check metadata file and extract package name

+ shopt -s nullglob

+ set -- .*.metadata

+ if (( $# == 0 ))

+ then

+     echo 'Missing metadata. Please run from inside a sources git repo' >&2

+     exit 1

+ elif (( $# > 1 ))

+ then

+     echo "Warning: multiple metadata files found. Using $1"

+ fi

+ meta=$1

+ pn=${meta%.metadata}

+ pn=${pn#.}

+ 

+ if [ ! -d .git ] || [ ! -d SPECS ]; then

+   echo 'You need to run this from inside a sources git repo' >&2

+   exit 1

+ fi

+ mkdir -p SOURCES

+ 

+ # sort out our branch

+ if [ -n "$BRANCH" ]

+ then

+   branches=("$BRANCH")

+ else

+   # generate a list of all branches containing current HEAD

+   branches=()

+   while IFS='' read -r line

+   do

+     # input from: git branch --contains HEAD

+     branch="${line:2}"

+     if [[ "$branch" =~ "detached from" ]]

+     then

+       # ignore detached heads

+       continue

+     fi

+     if [ ".${line:0:1}" = ".*" ]

+     then

+       # current branch, put it first

+       branches=("$branch" "${branches[@]}")

+     else

+       branches=("${branches[@]}" "$branch")

+     fi

+   done <<< "$(git branch -r --contains HEAD | grep '^\s\+origin/'| sed 's#origin/##g')"

+ fi

+ while read -r fsha fname ; do

+   if [ ".${fsha}" = ".da39a3ee5e6b4b0d3255bfef95601890afd80709" ]; then

+     # zero byte file

+     touch "${fname}"

+   else

+     if [ ${CHECK} -eq 1 ]; then

+       hashType=$(weakHashDetection "${fsha}")

+       if [ "${hashType}" == "unknown" ]; then

+         echo 'Failure: Hash type unknown.' >&2

+         exit 1;

+       else

+         command -v "${hashType}" >/dev/null 2>&1

+         if [[ $? -ne 0 ]]; then

+           echo "Failure: You need ${hashType} in PATH." >&2

+           exit 1;

+         fi

+       fi

+     fi

+     if [ -e ${fname} -a ${CHECK} -eq 1 ]; then

+ 	# check hash sum and force download if wrong

+         downsum=$(${hashType} "${fname}" | awk '{print $1}')

+         if [ "${fsha}" != "${downsum}" ]; then

+             rm -f "${fname}"

+         fi

+     fi

+     if [ ! -e "${fname}" ]; then

+       for br in "${branches[@]}"

+       do

+         br=$(echo "${br}"| sed -e s'|remotes/origin/||')

+         url="${SURL}/${pn}/${br}/${fsha}"

+         echo "Retrieving ${url}"

+         curl -L ${QUIET} -f "${url}" -o "${fname}" && break

+       done

+     else

+       echo "${fname} exists. skipping"

+     fi

+     if [ ${CHECK} -eq 1 ]; then

+         downsum=$(${hashType} "${fname}" | awk '{print $1}')

+         if [ "${fsha}" != "${downsum}" ]; then

+             rm -f "${fname}"

+             echo "Failure: ${fname} hash does not match hash from the .metadata file" >&2

+             exit 1;

+         fi

+     fi

+   fi

+ done < "${meta}"

@@ -0,0 +1,122 @@ 

+ #!/bin/bash

+ 

+ # This script will let you upload sources/blobs to new CentOS lookaside cache

+ # requirements:

+ #  - curl

+ #  - valid TLS certs from https://accounts.centos.org (or dev instance for testing)

+ #  - valid group membership to let you upload to specific "branch"

+ 

+ # Some variables, switch for new url

+ lookaside_baseurl="https://git.centos.org"

+ 

+ function usage {

+ 

+ cat <<  EOF

+ 

+ You need to call the script like this : $0 -arguments

+ 

+         -f : filename/source to upload (required, default:none)

+         -n : package name for that source (requred, default:none, example "httpd")

+         -b : "branch" where to upload to (required, default:none, example "c7-sig-core")

+         -h : display this help

+ 

+ EOF

+ 

+ }

+ 

+ function varcheck {

+ if [ -z "$1" ] ; then

+         usage

+         exit 1

+ fi

+ 

+ }

+ 

+ function f_log {

+   echo "[+] CentOS Lookaside upload tool -> $*"

+ }

+ 

+ 

+ 

+ while getopts “hf:n:b:” OPTION

+ do

+      case $OPTION in

+          h)

+              usage

+              exit 1

+              ;;

+          f)

+              file=$OPTARG

+              ;;

+          n)

+              pkgname=$OPTARG

+              ;;

+          b)

+              branch=$OPTARG

+              ;;

+          ?)

+              usage

+              exit

+              ;;

+      esac

+ done

+ 

+ varcheck $file

+ varcheck $pkgname

+ varcheck $branch

+ 

+ if [ ! -f ~/.centos.cert ] ;then

+   f_log "No mandatory TLS cert found (~/.centos.cert) .."

+   f_log "please use centos-cert to retrieve your ACO TLS cert"

+   exit 1

+ fi

+ 

+ if [ ! -f "${file}" ] ;then

+   f_log "Source to upload ${file} not found"

+   exit 2

+ fi

+ 

+ checksum=$(sha1sum ${file}|awk '{print $1}')

+ 

+ f_log "Checking if file already uploaded"

+ local_size=$(stat -c %s ${file})

+ remote_size=$(curl --silent -i --head ${lookaside_baseurl}/sources/${pkgname}/${branch}/${checksum}|grep "Content-Length"|cut -f 2 -d ':'|tr -d [:blank:]|tr -d '\r')

+ if [ "$local_size" -eq "$remote_size" ] ; then

+   f_log "File already uploaded"

+   exit 3

+ fi

+ 

+ f_log "Initialing new upload to lookaside"

+ f_log "URL : $lookaside_baseurl"

+ f_log "Source to upload : ${file} "

+ f_log "Package name: $pkgname"

+ f_log "sha1sum: ${checksum}"

+ f_log "Remote branch: ${branch}" 

+ f_log " ====== Trying to upload ======="

+ echo ""

+ curl ${lookaside_baseurl}/sources/upload.cgi \

+ 	--fail \

+ 	--cert ~/.centos.cert \

+ 	--form "name=${pkgname}" \

+ 	--form "branch=${branch}" \

+ 	--form "sha1sum=${checksum}" \

+ 	--form "file=@${file}" \

+ 	--progress-bar | tee /dev/null

+ 

+ upload_result="${PIPESTATUS[0]}"

+ 

+ if [ "$upload_result" -ne "0" ] ;then

+   f_log "[ERROR] Something didn't work to push to ${lookaside_baseurl}/sources/${pkgname}/${branch}/${checksum}"

+   f_log "[ERROR] Verify at the server side"

+   exit 1

+ fi

+ 

+ f_log "Validating that source was correctly uploaded ...."

+ remote_size=$(curl --silent -i --head ${lookaside_baseurl}/sources/${pkgname}/${branch}/${checksum}|grep "Content-Length"|cut -f 2 -d ':'|tr -d [:blank:]|tr -d '\r')

+ if [ "$local_size" -eq "$remote_size" ] ; then

+   f_log "[SUCCESS] Source should be available at ${lookaside_baseurl}/sources/${pkgname}/${branch}/${checksum}"

+ else

+   f_log "[ERROR] it seems there is a mismatch with source size and remote file size"

+ fi

+ 

+ 

file modified
+10 -1

@@ -1,6 +1,6 @@ 

  Name:           centos-packager

  Version:        0.7.0

- Release:        6%{?dist}

+ Release:        7%{?dist}

  Summary:        Tools and files necessary for building CentOS packages

  Group:          Applications/Productivity

  

@@ -9,6 +9,8 @@ 

  Source0:        cbs-koji.conf

  Source1:        COPYING

  Source2:        centos-cert

+ Source3:        lookaside_upload

+ Source4:        get_sources.sh

  

  Requires:       koji

  Requires:       rpm-build rpmdevtools rpmlint

@@ -39,14 +41,21 @@ 

  mkdir -p %{buildroot}/%{_bindir}

  ln -s koji %{buildroot}%{_bindir}/cbs

  install -p -m 0755 %{SOURCE2} %{buildroot}%{_bindir}/centos-cert

+ install -p -m 0755 %{SOURCE3} %{buildroot}%{_bindir}/centos-lookaside-upload

+ install -p -m 0755 %{SOURCE4} %{buildroot}%{_bindir}/centos-get-sources

  

  %files

  %license COPYING

  %config(noreplace) %{_sysconfdir}/koji.conf.d/cbs-koji.conf

  %{_bindir}/cbs

  %{_bindir}/centos-cert

+ %{_bindir}/centos-lookaside-upload

+ %{_bindir}/centos-get-sources

  

  %changelog

+ * Thu May 13 2021 Davide Cavalca <dcavalca@fb.com> - 0.7.0-7

+ - Import lookaside_upload and get_sources.sh from centos-git-common

+ 

  * Tue May  4 2021 Davide Cavalca <dcavalca@fb.com> - 0.7.0-6

  - Use the correct macro for the license file

  - Preserve timestamps on install