From 3edf28ae80fa6c85aca48fff8ec9ec6b7af2e73c Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Feb 10 2021 05:02:00 +0000 Subject: Lots of changes, starting to try out 2 configs for the distro/SIG buildsystems --- diff --git a/src/centpkg-sig.conf b/src/centpkg-sig.conf new file mode 100644 index 0000000..9724c5b --- /dev/null +++ b/src/centpkg-sig.conf @@ -0,0 +1,27 @@ +[centpkg-sig] +lookaside = https://git.centos.org/sources +lookasidehash = sha512 +lookaside_cgi = https://git.centos.org/sources/upload.cgi +distgit_namespaced = True +distgit_namespaces = rpms +gitbaseurl = ssh://git@git.centos.org/%(repo)s.git +anongiturl = https://git.centos.org/%(repo)s +branchre = .+\d$|.+\d-.+|master$ +kojiprofile = cbs +build_client = cbs +lookaside_namespaced = True +git_excludes = + i386/ + i686/ + x86_64/ + ppc/ + ppc64/ + ia64/ + mips/ + arm/ + noarch/ + /*.src.rpm + /build*.log + /.build-*.log + results_*/ + clog diff --git a/src/centpkg.conf b/src/centpkg.conf index 8c24671..afefb24 100644 --- a/src/centpkg.conf +++ b/src/centpkg.conf @@ -1,13 +1,29 @@ [centpkg] -lookaside = https://git.stg.centos.org/sources -lookasidehash = sha1 -lookaside_cgi = https://git.stg.centos.org/sources/upload.cgi -lookaside_request_params = branch +lookaside = https://sources.stream.rdu2.redhat.com/sources +lookasidehash = sha512 +lookaside_cgi = https://sources.stream.rdu2.redhat.com/sources/upload.cgi distgit_namespaced = True distgit_namespaces = rpms -gitbaseurl = https://%(user)s@git.stg.centos.org/%(repo)s.git -anongiturl = https://git.stg.centos.org/%(repo)s +gitbaseurl = ssh://git@gitlab.com/%(repo)s.git +anongiturl = https://gitlab.com/%(repo)s branchre = .+\d$|.+\d-.+|master$ -kojiprofile = cbs -build_client = cbs -clone_config = +kojiprofile = stream +build_client = stream +kerberos_realms = REDHAT.COM,IPA.REDHAT.COM +distgit_namespaced = True +lookaside_namespaced = True +git_excludes = + i386/ + i686/ + x86_64/ + ppc/ + ppc64/ + ia64/ + mips/ + arm/ + noarch/ + /*.src.rpm + /build*.log + /.build-*.log + results_*/ + clog diff --git a/src/centpkg/__init__.py b/src/centpkg/__init__.py index 166ad6b..aa545ba 100644 --- a/src/centpkg/__init__.py +++ b/src/centpkg/__init__.py @@ -19,13 +19,11 @@ import os import re import sys -import urlparse import warnings from pyrpkg import Commands, rpkgError -from pyrpkg.layout import ExplodedSRPMLayout, DistGitLayout -from centos import centos_cert -from .lookaside import CentOSLookasideCache +from pyrpkg.layout import SRPMLayout, DistGitLayout +from .lookaside import StreamLookasideCache,SIGLookasideCache from pyrpkg.utils import cached_property from . import cli @@ -39,7 +37,7 @@ class DistGitDirectory(object): distrobranch = False def __init__(self, branchtext): - sigtobranchre = r'c(?P\d+)-sig-(?P\w+)-?(?P\w+)?-?(?P\w+)?' + sigtobranchre = r'c(?P\d+[s]?)-sig-(?P\w+)-?(?P\w+)?-?(?P\w+)?' distrobranchre = r'c(?P\d+)-?(?P\w+)?' oldbranchre = r'(?P\w+)(?P\d)' sigmatch = re.match(sigtobranchre, branchtext) @@ -77,8 +75,12 @@ class DistGitDirectory(object): releaseorcommon = self.releasename if self.distrobranch: - return '-'.join(filter(None, ['c'+self.centosversion, - projectorcommon])) + if self.centosversion not in ('6','7'): + return 'c{}s-candidate'.format(self.centosversion) + else: + return '-'.join(filter(None, ['c'+self.centosversion, + projectorcommon])) + if not releaseorcommon: if not projectorcommon or projectorcommon == 'common': @@ -100,19 +102,23 @@ class Commands(Commands): ''' super(Commands, self).__init__(*args, **kwargs) - self.source_entry_type = 'old' - @property def distgitdir(self): return DistGitDirectory(self.branch_merge) @cached_property def lookasidecache(self): - return CentOSLookasideCache(self.lookasidehash, - self.lookaside, - self.lookaside_cgi, - self.repo_name, - self.branch_merge) + if self.distgitdir.distrobranch: + return StreamLookasideCache(self.lookasidehash, + self.lookaside, + self.lookaside_cgi, + ) + else: + return SIGLookasideCache(self.lookasidehash, + self.lookaside, + self.lookaside_cgi, + self.repo_name, + self.branch_merge) # redefined loaders def load_rpmdefines(self): @@ -142,16 +148,6 @@ class Commands(Commands): self._target = self.distgitdir.target - def load_user(self): - try: - self._user = centos_cert.CentOSUserCert().CN - except Exception: - print >>sys.stderr, "Could not load user from cert file" - super(Commands, self).load_user() - - # These are the commands defined in the base pyrpkg.Commands class - # and have been implemented here - def upload(self, *args, **kwargs): if not self.distgitdir.distrobranch: self.source_entry_type = 'bsd' diff --git a/src/centpkg/__main__.py b/src/centpkg/__main__.py index db1dcf5..2f28cf0 100644 --- a/src/centpkg/__main__.py +++ b/src/centpkg/__main__.py @@ -16,50 +16,66 @@ import os import sys import logging -import ConfigParser -import argparse -import fedpkg +import six.moves.configparser as ConfigParser + import pyrpkg -import centpkg -from pyrpkg.layout import Layout, ExplodedSRPMLayout, DistGitLayout +import pyrpkg.utils +import centpkg.cli + def main(): - ''' - Where things actually happen - ''' - parser = argparse.ArgumentParser(add_help=False) - parser.add_argument('-C', '--config', help='The rpkg config file to use', - default='/etc/rpkg/centpkg.conf') + # Setup an argparser and parse the known commands to get the config file + program_name = os.path.basename(sys.argv[0]) + + # Modified ArgumentParser provides parameter "allow_abbrev=False" (which affects processing + # of commandline arguments with common prefix). Generaly it is available since python3.6. + # This enables "allow_abbrev" for older python versions. + parser = pyrpkg.cli.ArgumentParser(add_help=False) + parser.add_argument('-S', '--sig', help='Operate as a CentOS SIG user instead of using the Stream distro environment', + default=False, + action='store_true') (args, other) = parser.parse_known_args() + if args.sig: + args.config = '/etc/rpkg/centpkg-sig.conf' + else: + args.config = '/etc/rpkg/centpkg.conf' + # Make sure we have a sane config file - if not os.path.exists(args.config) and not other[-1] in ['--help', '-h']: + if not os.path.exists(args.config) and not other[-1] in ['--help', '-h', 'help']: sys.stderr.write('Invalid config file %s\n' % args.config) sys.exit(1) + # Setup a configuration object and read config file data config = ConfigParser.SafeConfigParser() config.read(args.config) - layout = Layout.load() - if isinstance(layout, DistGitLayout): - config.read('/etc/rpkg/fedpkg.conf') - client = fedpkg.cli.fedpkgClient(config, name='fedpkg') - else: - client = centpkg.cli.centpkgClient(config) - - client.do_imports(site=client.DEFAULT_CLI_NAME) + client = centpkg.cli.centpkgClient(config) + client.do_imports(site='centpkg') client.parse_cmdline() + # This is due to a difference argparse behavior to Python 2 version. + # In Python 3, argparse will proceed to here without reporting + # "too few arguments". Instead, client.args does not have attribute + # command. + if not hasattr(client.args, 'command'): + client.parser.print_help() + sys.exit(1) + if not client.args.path: try: - client.args.path = os.getcwd() - except OSError as err_msg: - print('Could not get current path') - print(err_msg) + client.args.path = pyrpkg.utils.getcwd() + except Exception: + print('Could not get current path, have you deleted it?') sys.exit(1) + # setup the logger -- This logger will take things of INFO or DEBUG and + # log it to stdout. Anything above that (WARN, ERROR, CRITICAL) will go + # to stderr. Normal operation will show anything INFO and above. + # Quiet hides INFO, while Verbose exposes DEBUG. In all cases WARN or + # higher are exposed (via stderr). log = pyrpkg.log client.setupLogging(log) @@ -75,9 +91,12 @@ def main(): sys.exit(client.args.command()) except KeyboardInterrupt: pass - except Exception, e: + except Exception as e: + if getattr(client.args, 'debug', False): + raise log.error('Could not execute %s: %s' % (client.args.command.__name__, e)) sys.exit(1) -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/src/centpkg/cli.py b/src/centpkg/cli.py index b6c573f..3fc5a8f 100755 --- a/src/centpkg/cli.py +++ b/src/centpkg/cli.py @@ -21,3 +21,14 @@ class centpkgClient(cliClient): def __init__(self, config, name=None): self.DEFAULT_CLI_NAME = 'centpkg' super(centpkgClient, self).__init__(config, name) + + self.setup_centos_subparsers() + + def setup_centos_subparsers(self): + self.register_parser() + + def register_parser(self): + self.parser.add_argument('-S', '--sig', help='Operate as a CentOS SIG user instead of using the Stream distro environment', + default=False, + action='store_true') + diff --git a/src/centpkg/lookaside.py b/src/centpkg/lookaside.py index 10fece1..b1a0edf 100644 --- a/src/centpkg/lookaside.py +++ b/src/centpkg/lookaside.py @@ -7,29 +7,35 @@ # the full text of the license. -"""Interact with the CentOS lookaside cache +"""Interact with the Red Hat lookaside cache We need to override the pyrpkg.lookasidecache module to handle our custom download path. """ -import hashlib import io -import logging import os -import sys - import pycurl import six +import sys + -from pyrpkg.errors import DownloadError, InvalidHashType, UploadError +from pyrpkg.errors import InvalidHashType, UploadError from pyrpkg.lookaside import CGILookasideCache -from six.moves import http_client -class CentOSLookasideCache(CGILookasideCache): +class StreamLookasideCache(CGILookasideCache): + def __init__(self, hashtype, download_url, upload_url): + super(StreamLookasideCache, self).__init__( + hashtype, download_url, upload_url) + + self.download_path = ( + '%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s') + + +class SIGLookasideCache(CGILookasideCache): def __init__(self, hashtype, download_url, upload_url, name, branch): - super(CentOSLookasideCache, self).__init__( + super(SIGLookasideCache, self).__init__( hashtype, download_url, upload_url, client_cert="/home/bstinson/.centos.cert") self.branch = branch