From a394b151276f1088a652bd15be26841ad40de5db Mon Sep 17 00:00:00 2001 From: Brian Stinson Date: Nov 15 2018 04:28:34 +0000 Subject: big update for a new rpkg --- diff --git a/centpkg.spec b/centpkg.spec index 9f09c1d..80a886a 100644 --- a/centpkg.spec +++ b/centpkg.spec @@ -31,7 +31,7 @@ BuildRequires: pyrpkg Provides the centpkg command for working with dist-git %prep -%setup -q +%setup -q -c %build diff --git a/setup.py b/setup.py index 3a05f43..d71cd96 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name="centpkg", version=__version__, author="Brian Stinson", - author_email="bstinson@ksu.edu", + author_email="brian@bstinson.com", description="CentOS Plugin to rpkg for managing RPM package sources", url="http://bitbucket.org/bstinsonmhk/centpkg.git", license="GPLv2+", diff --git a/src/centpkg.conf b/src/centpkg.conf index 92fd00d..8c24671 100644 --- a/src/centpkg.conf +++ b/src/centpkg.conf @@ -1,9 +1,13 @@ [centpkg] -lookaside = https://git.centos.org/sources/ +lookaside = https://git.stg.centos.org/sources lookasidehash = sha1 -lookaside_cgi = https://localhost/repo/pkgs/upload.cgi #Not Implemented -gitbaseurl = https://%(user)s@git.centos.org/git/rpms/%(module)s -anongiturl = git://git.centos.org/rpms/%(module)s +lookaside_cgi = https://git.stg.centos.org/sources/upload.cgi +lookaside_request_params = branch +distgit_namespaced = True +distgit_namespaces = rpms +gitbaseurl = https://%(user)s@git.stg.centos.org/%(repo)s.git +anongiturl = https://git.stg.centos.org/%(repo)s branchre = .+\d$|.+\d-.+|master$ -kojiconfig = /etc/koji.conf.d/cbs-koji.conf +kojiprofile = cbs build_client = cbs +clone_config = diff --git a/src/centpkg/__init__.py b/src/centpkg/__init__.py index b02f98b..57ed5fd 100644 --- a/src/centpkg/__init__.py +++ b/src/centpkg/__init__.py @@ -23,7 +23,10 @@ import urlparse import warnings from pyrpkg import Commands, rpkgError +from pyrpkg.layout import ExplodedSRPMLayout, DistGitLayout from centos import centos_cert +from .lookaside import CentOSLookasideCache +from pyrpkg.utils import cached_property from . import cli @@ -36,7 +39,7 @@ class DistGitDirectory(object): distrobranch = False def __init__(self, branchtext): - sigtobranchre = r'sig-(?P\w+)(?P\d)-?(?P\w+)?-?(?P\w+)?' + sigtobranchre = r'c(?P\d+)-sig-(?P\w+)-?(?P\w+)?-?(?P\w+)?' distrobranchre = r'c(?P\d+)-?(?P\w+)?' oldbranchre = r'(?P\w+)(?P\d)' sigmatch = re.match(sigtobranchre, branchtext) @@ -86,23 +89,30 @@ class DistGitDirectory(object): return '-'.join(filter(None, [self.signame+self.centosversion, projectorcommon, releaseorcommon])) + '-el{0}'.format(self.centosversion) + class Commands(Commands): ''' For the pyrpkg commands with centpkg behavior ''' - def __init__(self, path, lookaside, lookasidehash, lookaside_cgi, - gitbaseurl, anongiturl, branchre, kojiconfig, - build_client, user=None, dist=None, target=None, - quiet=False, distgit_namespaced=False): + def __init__(self, *args, **kwargs): ''' Init the object and some configuration details. ''' - super(Commands, self).__init__(path, lookaside, lookasidehash, - lookaside_cgi, gitbaseurl, anongiturl, - branchre, kojiconfig, build_client, - user, dist, target, quiet, distgit_namespaced) + super(Commands, self).__init__(*args, **kwargs) + + self.source_entry_type = 'old' + + @property + def distgitdir(self): + return DistGitDirectory(self.branch_merge) - self.distgitdir = DistGitDirectory(self.branch_merge) + @cached_property + def lookasidecache(self): + return CentOSLookasideCache(self.lookasidehash, + self.lookaside, + self.lookaside_cgi, + self.repo_name, + self.branch_merge) # redefined loaders def load_rpmdefines(self): @@ -113,39 +123,25 @@ class Commands(Commands): if not self.distgitdir.centosversion: raise rpkgError('Could not get the OS version from the branch:{0}'.format(self.branch_merge)) - self._distval = self.distgitdir.centosversion - self._distval = self._distval.replace('.', '_') + self._distvar = self.distgitdir.centosversion + self._distval = self._distvar.replace('.', '_') self._disttag = 'el%s' % self._distval - - self._rpmdefines = ["--define '_topdir {0}'".format(self.path), - "--define '_srcrpmdir {0}'".format(self.path), - "--define '_rpmdir {0}'".format(self.path), - "--define 'dist .{0}'".format(self._disttag), + self._rpmdefines = ["--define '_sourcedir %s'" % self.layout.sourcedir, + "--define '_specdir %s'" % self.layout.specdir, + "--define '_builddir %s'" % self.layout.builddir, + "--define '_srcrpmdir %s'" % self.layout.srcrpmdir, + "--define '_rpmdir %s'" % self.layout.rpmdir, + "--define 'dist .%s'" % self._disttag, # int and float this to remove the decimal - "--define '{0} 1'".format(self._disttag)] - - def load_spec(self): - """This sets the spec attribute""" - - # We are not using the upstream load_spec because the file structure is - # hard-coded - - # Get a list of files in the path we're looking at - files = os.listdir(os.path.join(self.path,'SPECS')) - # Search the files for the first one that ends with ".spec" - for __f in files: - if __f.endswith('.spec') and not __f.startswith('.'): - self._spec = os.path.join('SPECS', __f) - return - - raise rpkgError('No spec file found.') + "--define '%s 1'" % self._disttag] + self.log.debug("RPMDefines: %s" % self._rpmdefines) def load_target(self): """ This sets the target attribute (used for mock and koji) """ # Distribution branches start with c and may or may not end in -plus - # otherwise, it's a sig branch + # otherwise, it's a sig branch if not self.distgitdir.distrobranch: # send distribution packages to build on the the bananas tags for now self._target = 'bananas{0}-{1}'.format(self.distval, self.disttag) @@ -156,157 +152,26 @@ class Commands(Commands): def load_user(self): try: self._user = centos_cert.CentOSUserCert().CN - except: + except Exception: print >>sys.stderr, "Could not load user from cert file" super(Commands, self).load_user() # These are the commands defined in the base pyrpkg.Commands class # and have been implemented here - def sources(self, outdir=None): - """Download source files""" - - # See also: - # https://lists.fedoraproject.org/pipermail/buildsys/2014-July/004313.html - # - # in 'super' the sources function expects a file named 'sources' to be in the base directory. - # A patch has been sent to upstream to allow a more flexible location. - # - # This code doesn't work due to: - # archive.strip().split(' ', 1) # patch provided to upstream to fix - # - # url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module_name, - # file.replace(' ', '%20'), - # csum, file.replace(' ', '%20')) - # - #os.symlink(os.path.join(self.path, '.{0}.metadata'.format(self.module_name)), os.path.join(self.path, 'sources')) - #super(Commands, self).sources(outdir=None) - #os.unlink(os.path.join(self.path, 'sources')) - - # The following is copied from rpkg/__init__.py:sources with minor changes - try: - archives = open(os.path.join(self.path, - '.{0}.metadata'.format(self.module_name)), - 'r').readlines() - except IOError, e: - raise rpkgError('%s is not a valid repo: %s' % (self.path, e)) - # Default to putting the files where the module is - if not outdir: - outdir = self.path - for archive in archives: - try: - # This strip / split is kind a ugly, but checksums shouldn't have - # two spaces in them. sources file might need more structure in the - # future - csum, file = archive.strip().split(None, 1) - except ValueError: - raise rpkgError('Malformed sources file.') - - # The default lookaside hash is stored in centpkg.conf, but there is - # a mix of md5 and sha sums in the CentOS lookaside, here we divine - # which one we are using - - sum_lengths = { 128: 'sha512', - 64: 'sha256', - 40: 'sha1', - 32: 'md5', - } - - self.lookasidehash = sum_lengths[len(csum)] - - # If a directory is specified in the metadata file, append it to - # outdir - if os.path.dirname(file): - outdir = os.path.join(self.path, os.path.dirname(file)) - file = os.path.basename(file) - - # Create the output directory if it's not checked into git - if not os.path.exists(outdir): - self.log.info("Creating OUTDIR: {0}".format(outdir)) - os.makedirs(outdir) - - outfile = os.path.join(outdir, file) - # See if we already have a valid copy downloaded - if os.path.exists(outfile): - if self.lookasidecache.file_is_valid(outfile, csum, self.lookasidehash): - continue - self.log.info("Downloading %s" % (file)) - filepath = '%s/%s/%s' % (self.module_name, - self.branch_merge, - csum, - ) - url = urlparse.urljoin(self.lookaside, filepath) - command = ['curl', '-H', 'Pragma:', '-o', outfile, '-R', '-S', '--fail'] - if self.quiet: - command.append('-s') - command.append(url) - self._run_command(command) - if not self.lookasidecache.file_is_valid(outfile, csum, self.lookasidehash): - raise rpkgError('%s failed checksum' % file) - - return - - - def get_latest_commit(self, *args, **kwargs): - raise NotImplementedError("get_latest_commit is not yet implemented in centpkg") - - def gitbuildhash(self, *args, **kwargs): - raise NotImplementedError("gitbuildhash is not yet implemented in centpkg") + def upload(self, *args, **kwargs): + if not self.distgitdir.distrobranch: + self.source_entry_type = 'bsd' + return super(Commands, self).upload(*args, **kwargs) def import_srpm(self, *args, **kwargs): raise NotImplementedError("import_srpm is not yet implemented in centpkg") - def new(self, *args, **kwargs): - raise NotImplementedError("new is not yet implemented in centpkg") - def patch(self, *args, **kwargs): raise NotImplementedError("patch is not yet implemented in centpkg") - def push(self, *args, **kwargs): - raise NotImplementedError("push is not yet implemented in centpkg") - - def file_exists(self, *args, **kwargs): - raise NotImplementedError("file_exists is not yet implemented in centpkg") - - def upload_file(self, *args, **kwargs): - raise NotImplementedError("upload_file is not yet implemented in centpkg") - - def install(self, *args, **kwargs): raise NotImplementedError("install is not yet implemented in centpkg") def lint(self, *args, **kwargs): raise NotImplementedError("lint is not yet implemented in centpkg") - - - def upload(self, *args, **kwargs): - raise NotImplementedError("upload is not yet implemented in centpkg") - - def prep(self, *args, **kwargs): - raise NotImplementedError("prep is not yet implemented in centpkg") - - def unused_patches(self): - """Discover patches checked into source control that are not used - - Returns a list of unused patches, which may be empty. - """ - - # Create a list for unused patches - unused = [] - # Get the content of spec into memory for fast searching - spec = open(self.spec, 'r').read() - # Replace %{name} with the package name - spec = spec.replace("%{name}", self.module_name) - # Replace %{version} with the package version - spec = spec.replace("%{version}", self.ver) - - # Get a list of files tracked in source control - files = self.repo.git.ls_files('--exclude-standard').split() - for file in map(os.path.basename, files): - # throw out non patches - if not file.endswith(('.patch', '.diff')): - continue - if file not in spec: - unused.append(file) - return unused - diff --git a/src/centpkg/__main__.py b/src/centpkg/__main__.py index 8b1bb76..fd5c7ce 100644 --- a/src/centpkg/__main__.py +++ b/src/centpkg/__main__.py @@ -19,8 +19,10 @@ import logging import ConfigParser import argparse +import fedpkg import pyrpkg import centpkg +from pyrpkg.layout import Layout, ExplodedSRPMLayout, DistGitLayout def main(): ''' @@ -40,8 +42,16 @@ def main(): config = ConfigParser.SafeConfigParser() config.read(args.config) - client = centpkg.cli.centpkgClient(config) - client.do_imports(site='centpkg') + layout = Layout.load() + if isinstance(layout, ExplodedSRPMLayout): + client = centpkg.cli.centpkgClient(config) + elif isinstance(layout, DistGitLayout): + config.read('/etc/rpkg/fedpkg.conf') + client = fedpkg.cli.fedpkgClient(config, name='fedpkg') + else: + raise ValueError("Not a site we know about") + + client.do_imports(site=client.DEFAULT_CLI_NAME) client.parse_cmdline() if not client.args.path: diff --git a/src/centpkg/cli.py b/src/centpkg/cli.py index 8207f39..b6c573f 100755 --- a/src/centpkg/cli.py +++ b/src/centpkg/cli.py @@ -13,46 +13,11 @@ # option) any later version. See http://www.gnu.org/copyleft/gpl.html for # the full text of the license. -import sys -import os -import logging - +from __future__ import print_function from pyrpkg.cli import cliClient + class centpkgClient(cliClient): - ''' - Where we import our custom stuff - ''' - def __init__(self, config, name='centpkg'): - '''init''' + def __init__(self, config, name=None): + self.DEFAULT_CLI_NAME = 'centpkg' super(centpkgClient, self).__init__(config, name) - - -if __name__ == '__main__': - client = centpkgClient() - client.do_imports() - client.parse_cmdline() - - if not client.args.path: - try: - client.args.path = os.getcwd() - except OSError as err_msg: - print('Could not get current path') - print(err_msg) - sys.exit(1) - - log = client.site.log - client.setupLogging(log) - - if client.args.v: - log.setLevel(logging.DEBUG) - elif client.args.q: - log.setLevel(logging.WARNING) - else: - log.setLevel(logging.INFO) - - # Run the necessary command - try: - client.args.command() - except KeyboardInterrupt: - pass diff --git a/src/centpkg/lookaside.py b/src/centpkg/lookaside.py new file mode 100644 index 0000000..10fece1 --- /dev/null +++ b/src/centpkg/lookaside.py @@ -0,0 +1,185 @@ +# Copyright (c) 2018 - Red Hat Inc. +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation; either version 2 of the License, or (at your +# option) any later version. See http://www.gnu.org/copyleft/gpl.html for +# the full text of the license. + + +"""Interact with the CentOS lookaside cache + +We need to override the pyrpkg.lookasidecache module to handle our custom +download path. +""" + +import hashlib +import io +import logging +import os +import sys + +import pycurl +import six + +from pyrpkg.errors import DownloadError, InvalidHashType, UploadError +from pyrpkg.lookaside import CGILookasideCache +from six.moves import http_client + + +class CentOSLookasideCache(CGILookasideCache): + def __init__(self, hashtype, download_url, upload_url, name, branch): + super(CentOSLookasideCache, self).__init__( + hashtype, download_url, upload_url, client_cert="/home/bstinson/.centos.cert") + self.branch = branch + + self.download_path = ( + '%(name)s/%(branch)s/%(hash)s') + + def remote_file_exists(self, name, filename, hash): + """Verify whether a file exists on the lookaside cache + + :param str name: The name of the module. (usually the name of the + SRPM). This can include the namespace as well (depending on what + the server side expects). + :param str filename: The name of the file to check for. + :param str hash: The known good hash of the file. + """ + + # RHEL 7 ships pycurl that does not accept unicode. When given unicode + # type it would explode with "unsupported second type in tuple". Let's + # convert to str just to be sure. + # https://bugzilla.redhat.com/show_bug.cgi?id=1241059 + if six.PY2 and isinstance(filename, six.text_type): + filename = filename.encode('utf-8') + + if six.PY2 and isinstance(self.branch, six.text_type): + self.branch = self.branch.encode('utf-8') + + post_data = [('name', name), + ('%ssum' % self.hashtype, hash), + ('branch', self.branch), + ('filename', filename)] + + with io.BytesIO() as buf: + c = pycurl.Curl() + c.setopt(pycurl.URL, self.upload_url) + c.setopt(pycurl.WRITEFUNCTION, buf.write) + c.setopt(pycurl.HTTPPOST, post_data) + + if self.client_cert is not None: + if os.path.exists(self.client_cert): + c.setopt(pycurl.SSLCERT, self.client_cert) + else: + self.log.warning("Missing certificate: %s" + % self.client_cert) + + if self.ca_cert is not None: + if os.path.exists(self.ca_cert): + c.setopt(pycurl.CAINFO, self.ca_cert) + else: + self.log.warning("Missing certificate: %s", self.ca_cert) + + c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE) + c.setopt(pycurl.USERPWD, ':') + + try: + c.perform() + status = c.getinfo(pycurl.RESPONSE_CODE) + + except Exception as e: + raise UploadError(e) + + finally: + c.close() + + output = buf.getvalue().strip() + + if status != 200: + self.raise_upload_error(status) + + # Lookaside CGI script returns these strings depending on whether + # or not the file exists: + if output == b'Available': + return True + + if output == b'Missing': + return False + + # Something unexpected happened + self.log.debug(output) + raise UploadError('Error checking for %s at %s' + % (filename, self.upload_url)) + + def upload(self, name, filepath, hash): + """Upload a source file + + :param str name: The name of the module. (usually the name of the SRPM) + This can include the namespace as well (depending on what the + server side expects). + :param str filepath: The full path to the file to upload. + :param str hash: The known good hash of the file. + """ + filename = os.path.basename(filepath) + + # As in remote_file_exists, we need to convert unicode strings to str + if six.PY2: + if isinstance(name, six.text_type): + name = name.encode('utf-8') + if isinstance(filepath, six.text_type): + filepath = filepath.encode('utf-8') + + if self.remote_file_exists(name, filename, hash): + self.log.info("File already uploaded: %s", filepath) + return + + self.log.info("Uploading: %s", filepath) + post_data = [('name', name), + ('%ssum' % self.hashtype, hash), + ('branch', self.branch), + ('file', (pycurl.FORM_FILE, filepath))] + + with io.BytesIO() as buf: + c = pycurl.Curl() + c.setopt(pycurl.URL, self.upload_url) + c.setopt(pycurl.NOPROGRESS, False) + c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress) + c.setopt(pycurl.WRITEFUNCTION, buf.write) + c.setopt(pycurl.HTTPPOST, post_data) + + if self.client_cert is not None: + if os.path.exists(self.client_cert): + c.setopt(pycurl.SSLCERT, self.client_cert) + else: + self.log.warning("Missing certificate: %s", self.client_cert) + + if self.ca_cert is not None: + if os.path.exists(self.ca_cert): + c.setopt(pycurl.CAINFO, self.ca_cert) + else: + self.log.warning("Missing certificate: %s", self.ca_cert) + + c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE) + c.setopt(pycurl.USERPWD, ':') + + try: + c.perform() + status = c.getinfo(pycurl.RESPONSE_CODE) + + except Exception as e: + raise UploadError(e) + + finally: + c.close() + + output = buf.getvalue().strip() + + # Get back a new line, after displaying the download progress + sys.stdout.write('\n') + sys.stdout.flush() + + if status != 200: + self.raise_upload_error(status) + + if output: + self.log.debug(output)