From 47f3c3e98fc842aa1e866a8e66385245b46d8be6 Mon Sep 17 00:00:00 2001 From: Davide Cavalca Date: May 13 2021 17:38:01 +0000 Subject: Import lookaside_upload and get_sources.sh from centos-git-common --- diff --git a/SOURCES/get_sources.sh b/SOURCES/get_sources.sh new file mode 100644 index 0000000..80c4b53 --- /dev/null +++ b/SOURCES/get_sources.sh @@ -0,0 +1,231 @@ +#!/bin/bash +# +# Might want to drop this in ~/bin/ and chmod u+x it +# + +# Initial Author: Karanbir Singh +# Updates: +# Mike McLean +# Pat Riehecky +# Tyler Parsons +# Tuomo Soini + + +##################################################################### +usage() { + echo '' >&2 + echo "$0 [-hcq] [-b branch] [--surl url]" >&2 + echo '' >&2 + echo 'Script to parse the non-text sources metadata file' >&2 + echo ' and download the required files from the lookaside' >&2 + echo ' cache.' >&2 + echo '' >&2 + echo 'PLEASE NOTE: this script is non-destructive, it wont' >&2 + echo ' replace files that already exist, regardless of' >&2 + echo ' their state, allowing you to have work-in-progress' >&2 + echo ' content that wont get overwritten.' >&2 + echo '' >&2 + echo 'You need to run this from inside a sources git repo' >&2 + echo '' >&2 + echo ' -h: This help message' >&2 + echo '' >&2 + echo " $0 -b c7" >&2 + echo " $0 -q -b c7" >&2 + echo " $0 -c -b remotes/origin/c7" >&2 + echo " $0 -c -b c7 --surl '$SURL'" >&2 + echo " $0" >&2 + exit 1 +} + +##################################################################### + +SURL="https://git.centos.org/sources" + +QUIET=0 +BRANCH='' +CHECK=0 + +# for setting any overrides, such as SURL, default BRANCH, or force CHECK +if [ -f /etc/centos-git-common ]; then + . /etc/centos-git-common +fi + +##################################################################### +# setup args in the right order for making getopt evaluation +# nice and easy. You'll need to read the manpages for more info +# utilizing 'while' construct rather than 'for arg' to avoid unnecessary +# shifting of program args +args=$(getopt -o hcqb: -l surl: -- "$@") +eval set -- "$args" + +while [[ 0 -eq 0 ]]; do + case $1 in + -- ) + # end of getopt args, shift off the -- and get out of the loop + shift + break + ;; + -c ) + # verify the sha1sum of the downloaded file + CHECK=1 + shift + ;; + -q ) + # suppress warnings + QUIET=1 + shift + ;; + -b ) + # Check this particular branch + BRANCH=$2 + shift + shift + ;; + --surl ) + # override sources url + SURL=$2 + shift + shift + ;; + -h ) + # get help + usage + ;; + esac +done + +# set curl options this way so defaults can be set in /etc/centos-git-common +# across multiple scripts +if [[ ${QUIET} -eq 1 ]]; then + QUIET='--silent' +else + QUIET='' +fi + +command -v git >/dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo 'You need git in PATH' >&2 + exit 1 +fi + +command -v curl >/dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo 'You need curl in PATH' >&2 + exit 1 +fi + +# should go into a function section at some point +weakHashDetection () { + strHash=${1}; + case $((`echo "${strHash}"|wc -m` - 1 )) in + 128) + hashBin='sha512sum' + ;; + 64) + hashBin='sha256sum' + ;; + 40) + hashBin='sha1sum' + ;; + 32) + hashBin='md5sum' + ;; + *) + hashBin='unknown' + ;; + esac + echo ${hashBin}; +} + +# check metadata file and extract package name +shopt -s nullglob +set -- .*.metadata +if (( $# == 0 )) +then + echo 'Missing metadata. Please run from inside a sources git repo' >&2 + exit 1 +elif (( $# > 1 )) +then + echo "Warning: multiple metadata files found. Using $1" +fi +meta=$1 +pn=${meta%.metadata} +pn=${pn#.} + +if [ ! -d .git ] || [ ! -d SPECS ]; then + echo 'You need to run this from inside a sources git repo' >&2 + exit 1 +fi +mkdir -p SOURCES + +# sort out our branch +if [ -n "$BRANCH" ] +then + branches=("$BRANCH") +else + # generate a list of all branches containing current HEAD + branches=() + while IFS='' read -r line + do + # input from: git branch --contains HEAD + branch="${line:2}" + if [[ "$branch" =~ "detached from" ]] + then + # ignore detached heads + continue + fi + if [ ".${line:0:1}" = ".*" ] + then + # current branch, put it first + branches=("$branch" "${branches[@]}") + else + branches=("${branches[@]}" "$branch") + fi + done <<< "$(git branch -r --contains HEAD | grep '^\s\+origin/'| sed 's#origin/##g')" +fi +while read -r fsha fname ; do + if [ ".${fsha}" = ".da39a3ee5e6b4b0d3255bfef95601890afd80709" ]; then + # zero byte file + touch "${fname}" + else + if [ ${CHECK} -eq 1 ]; then + hashType=$(weakHashDetection "${fsha}") + if [ "${hashType}" == "unknown" ]; then + echo 'Failure: Hash type unknown.' >&2 + exit 1; + else + command -v "${hashType}" >/dev/null 2>&1 + if [[ $? -ne 0 ]]; then + echo "Failure: You need ${hashType} in PATH." >&2 + exit 1; + fi + fi + fi + if [ -e ${fname} -a ${CHECK} -eq 1 ]; then + # check hash sum and force download if wrong + downsum=$(${hashType} "${fname}" | awk '{print $1}') + if [ "${fsha}" != "${downsum}" ]; then + rm -f "${fname}" + fi + fi + if [ ! -e "${fname}" ]; then + for br in "${branches[@]}" + do + br=$(echo "${br}"| sed -e s'|remotes/origin/||') + url="${SURL}/${pn}/${br}/${fsha}" + echo "Retrieving ${url}" + curl -L ${QUIET} -f "${url}" -o "${fname}" && break + done + else + echo "${fname} exists. skipping" + fi + if [ ${CHECK} -eq 1 ]; then + downsum=$(${hashType} "${fname}" | awk '{print $1}') + if [ "${fsha}" != "${downsum}" ]; then + rm -f "${fname}" + echo "Failure: ${fname} hash does not match hash from the .metadata file" >&2 + exit 1; + fi + fi + fi +done < "${meta}" diff --git a/SOURCES/lookaside_upload b/SOURCES/lookaside_upload new file mode 100644 index 0000000..4c5612a --- /dev/null +++ b/SOURCES/lookaside_upload @@ -0,0 +1,122 @@ +#!/bin/bash + +# This script will let you upload sources/blobs to new CentOS lookaside cache +# requirements: +# - curl +# - valid TLS certs from https://accounts.centos.org (or dev instance for testing) +# - valid group membership to let you upload to specific "branch" + +# Some variables, switch for new url +lookaside_baseurl="https://git.centos.org" + +function usage { + +cat << EOF + +You need to call the script like this : $0 -arguments + + -f : filename/source to upload (required, default:none) + -n : package name for that source (requred, default:none, example "httpd") + -b : "branch" where to upload to (required, default:none, example "c7-sig-core") + -h : display this help + +EOF + +} + +function varcheck { +if [ -z "$1" ] ; then + usage + exit 1 +fi + +} + +function f_log { + echo "[+] CentOS Lookaside upload tool -> $*" +} + + + +while getopts “hf:n:b:” OPTION +do + case $OPTION in + h) + usage + exit 1 + ;; + f) + file=$OPTARG + ;; + n) + pkgname=$OPTARG + ;; + b) + branch=$OPTARG + ;; + ?) + usage + exit + ;; + esac +done + +varcheck $file +varcheck $pkgname +varcheck $branch + +if [ ! -f ~/.centos.cert ] ;then + f_log "No mandatory TLS cert found (~/.centos.cert) .." + f_log "please use centos-cert to retrieve your ACO TLS cert" + exit 1 +fi + +if [ ! -f "${file}" ] ;then + f_log "Source to upload ${file} not found" + exit 2 +fi + +checksum=$(sha1sum ${file}|awk '{print $1}') + +f_log "Checking if file already uploaded" +local_size=$(stat -c %s ${file}) +remote_size=$(curl --silent -i --head ${lookaside_baseurl}/sources/${pkgname}/${branch}/${checksum}|grep "Content-Length"|cut -f 2 -d ':'|tr -d [:blank:]|tr -d '\r') +if [ "$local_size" -eq "$remote_size" ] ; then + f_log "File already uploaded" + exit 3 +fi + +f_log "Initialing new upload to lookaside" +f_log "URL : $lookaside_baseurl" +f_log "Source to upload : ${file} " +f_log "Package name: $pkgname" +f_log "sha1sum: ${checksum}" +f_log "Remote branch: ${branch}" +f_log " ====== Trying to upload =======" +echo "" +curl ${lookaside_baseurl}/sources/upload.cgi \ + --fail \ + --cert ~/.centos.cert \ + --form "name=${pkgname}" \ + --form "branch=${branch}" \ + --form "sha1sum=${checksum}" \ + --form "file=@${file}" \ + --progress-bar | tee /dev/null + +upload_result="${PIPESTATUS[0]}" + +if [ "$upload_result" -ne "0" ] ;then + f_log "[ERROR] Something didn't work to push to ${lookaside_baseurl}/sources/${pkgname}/${branch}/${checksum}" + f_log "[ERROR] Verify at the server side" + exit 1 +fi + +f_log "Validating that source was correctly uploaded ...." +remote_size=$(curl --silent -i --head ${lookaside_baseurl}/sources/${pkgname}/${branch}/${checksum}|grep "Content-Length"|cut -f 2 -d ':'|tr -d [:blank:]|tr -d '\r') +if [ "$local_size" -eq "$remote_size" ] ; then + f_log "[SUCCESS] Source should be available at ${lookaside_baseurl}/sources/${pkgname}/${branch}/${checksum}" +else + f_log "[ERROR] it seems there is a mismatch with source size and remote file size" +fi + + diff --git a/SPECS/centos-packager.spec b/SPECS/centos-packager.spec index ba8ea3a..27a0504 100644 --- a/SPECS/centos-packager.spec +++ b/SPECS/centos-packager.spec @@ -1,6 +1,6 @@ Name: centos-packager Version: 0.7.0 -Release: 6%{?dist} +Release: 7%{?dist} Summary: Tools and files necessary for building CentOS packages Group: Applications/Productivity @@ -9,6 +9,8 @@ URL: https://git.centos.org/centos/centos-packager Source0: cbs-koji.conf Source1: COPYING Source2: centos-cert +Source3: lookaside_upload +Source4: get_sources.sh Requires: koji Requires: rpm-build rpmdevtools rpmlint @@ -39,14 +41,21 @@ install -p -m 0644 %{SOURCE0} %{buildroot}%{_sysconfdir}/koji.conf.d/cbs-koji.co mkdir -p %{buildroot}/%{_bindir} ln -s koji %{buildroot}%{_bindir}/cbs install -p -m 0755 %{SOURCE2} %{buildroot}%{_bindir}/centos-cert +install -p -m 0755 %{SOURCE3} %{buildroot}%{_bindir}/centos-lookaside-upload +install -p -m 0755 %{SOURCE4} %{buildroot}%{_bindir}/centos-get-sources %files %license COPYING %config(noreplace) %{_sysconfdir}/koji.conf.d/cbs-koji.conf %{_bindir}/cbs %{_bindir}/centos-cert +%{_bindir}/centos-lookaside-upload +%{_bindir}/centos-get-sources %changelog +* Thu May 13 2021 Davide Cavalca - 0.7.0-7 +- Import lookaside_upload and get_sources.sh from centos-git-common + * Tue May 4 2021 Davide Cavalca - 0.7.0-6 - Use the correct macro for the license file - Preserve timestamps on install