diff --git a/centpkg.spec b/centpkg.spec
index 9f09c1d..80a886a 100644
--- a/centpkg.spec
+++ b/centpkg.spec
@@ -31,7 +31,7 @@ BuildRequires: pyrpkg
Provides the centpkg command for working with dist-git
%prep
-%setup -q
+%setup -q -c
%build
diff --git a/setup.py b/setup.py
index 3a05f43..d71cd96 100644
--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@ setup(
name="centpkg",
version=__version__,
author="Brian Stinson",
- author_email="bstinson@ksu.edu",
+ author_email="brian@bstinson.com",
description="CentOS Plugin to rpkg for managing RPM package sources",
url="http://bitbucket.org/bstinsonmhk/centpkg.git",
license="GPLv2+",
diff --git a/src/centpkg.conf b/src/centpkg.conf
index 92fd00d..8c24671 100644
--- a/src/centpkg.conf
+++ b/src/centpkg.conf
@@ -1,9 +1,13 @@
[centpkg]
-lookaside = https://git.centos.org/sources/
+lookaside = https://git.stg.centos.org/sources
lookasidehash = sha1
-lookaside_cgi = https://localhost/repo/pkgs/upload.cgi #Not Implemented
-gitbaseurl = https://%(user)s@git.centos.org/git/rpms/%(module)s
-anongiturl = git://git.centos.org/rpms/%(module)s
+lookaside_cgi = https://git.stg.centos.org/sources/upload.cgi
+lookaside_request_params = branch
+distgit_namespaced = True
+distgit_namespaces = rpms
+gitbaseurl = https://%(user)s@git.stg.centos.org/%(repo)s.git
+anongiturl = https://git.stg.centos.org/%(repo)s
branchre = .+\d$|.+\d-.+|master$
-kojiconfig = /etc/koji.conf.d/cbs-koji.conf
+kojiprofile = cbs
build_client = cbs
+clone_config =
diff --git a/src/centpkg/__init__.py b/src/centpkg/__init__.py
index b02f98b..57ed5fd 100644
--- a/src/centpkg/__init__.py
+++ b/src/centpkg/__init__.py
@@ -23,7 +23,10 @@ import urlparse
import warnings
from pyrpkg import Commands, rpkgError
+from pyrpkg.layout import ExplodedSRPMLayout, DistGitLayout
from centos import centos_cert
+from .lookaside import CentOSLookasideCache
+from pyrpkg.utils import cached_property
from . import cli
@@ -36,7 +39,7 @@ class DistGitDirectory(object):
distrobranch = False
def __init__(self, branchtext):
- sigtobranchre = r'sig-(?P<signame>\w+)(?P<centosversion>\d)-?(?P<projectname>\w+)?-?(?P<releasename>\w+)?'
+ sigtobranchre = r'c(?P<centosversion>\d+)-sig-(?P<signame>\w+)-?(?P<projectname>\w+)?-?(?P<releasename>\w+)?'
distrobranchre = r'c(?P<centosversion>\d+)-?(?P<projectname>\w+)?'
oldbranchre = r'(?P<signame>\w+)(?P<centosversion>\d)'
sigmatch = re.match(sigtobranchre, branchtext)
@@ -86,23 +89,30 @@ class DistGitDirectory(object):
return '-'.join(filter(None, [self.signame+self.centosversion,
projectorcommon, releaseorcommon])) + '-el{0}'.format(self.centosversion)
+
class Commands(Commands):
'''
For the pyrpkg commands with centpkg behavior
'''
- def __init__(self, path, lookaside, lookasidehash, lookaside_cgi,
- gitbaseurl, anongiturl, branchre, kojiconfig,
- build_client, user=None, dist=None, target=None,
- quiet=False, distgit_namespaced=False):
+ def __init__(self, *args, **kwargs):
'''
Init the object and some configuration details.
'''
- super(Commands, self).__init__(path, lookaside, lookasidehash,
- lookaside_cgi, gitbaseurl, anongiturl,
- branchre, kojiconfig, build_client,
- user, dist, target, quiet, distgit_namespaced)
+ super(Commands, self).__init__(*args, **kwargs)
+
+ self.source_entry_type = 'old'
+
+ @property
+ def distgitdir(self):
+ return DistGitDirectory(self.branch_merge)
- self.distgitdir = DistGitDirectory(self.branch_merge)
+ @cached_property
+ def lookasidecache(self):
+ return CentOSLookasideCache(self.lookasidehash,
+ self.lookaside,
+ self.lookaside_cgi,
+ self.repo_name,
+ self.branch_merge)
# redefined loaders
def load_rpmdefines(self):
@@ -113,39 +123,25 @@ class Commands(Commands):
if not self.distgitdir.centosversion:
raise rpkgError('Could not get the OS version from the branch:{0}'.format(self.branch_merge))
- self._distval = self.distgitdir.centosversion
- self._distval = self._distval.replace('.', '_')
+ self._distvar = self.distgitdir.centosversion
+ self._distval = self._distvar.replace('.', '_')
self._disttag = 'el%s' % self._distval
-
- self._rpmdefines = ["--define '_topdir {0}'".format(self.path),
- "--define '_srcrpmdir {0}'".format(self.path),
- "--define '_rpmdir {0}'".format(self.path),
- "--define 'dist .{0}'".format(self._disttag),
+ self._rpmdefines = ["--define '_sourcedir %s'" % self.layout.sourcedir,
+ "--define '_specdir %s'" % self.layout.specdir,
+ "--define '_builddir %s'" % self.layout.builddir,
+ "--define '_srcrpmdir %s'" % self.layout.srcrpmdir,
+ "--define '_rpmdir %s'" % self.layout.rpmdir,
+ "--define 'dist .%s'" % self._disttag,
# int and float this to remove the decimal
- "--define '{0} 1'".format(self._disttag)]
-
- def load_spec(self):
- """This sets the spec attribute"""
-
- # We are not using the upstream load_spec because the file structure is
- # hard-coded
-
- # Get a list of files in the path we're looking at
- files = os.listdir(os.path.join(self.path,'SPECS'))
- # Search the files for the first one that ends with ".spec"
- for __f in files:
- if __f.endswith('.spec') and not __f.startswith('.'):
- self._spec = os.path.join('SPECS', __f)
- return
-
- raise rpkgError('No spec file found.')
+ "--define '%s 1'" % self._disttag]
+ self.log.debug("RPMDefines: %s" % self._rpmdefines)
def load_target(self):
""" This sets the target attribute (used for mock and koji) """
# Distribution branches start with c and may or may not end in -plus
- # otherwise, it's a sig branch
+ # otherwise, it's a sig branch
if not self.distgitdir.distrobranch:
# send distribution packages to build on the the bananas tags for now
self._target = 'bananas{0}-{1}'.format(self.distval, self.disttag)
@@ -156,157 +152,26 @@ class Commands(Commands):
def load_user(self):
try:
self._user = centos_cert.CentOSUserCert().CN
- except:
+ except Exception:
print >>sys.stderr, "Could not load user from cert file"
super(Commands, self).load_user()
# These are the commands defined in the base pyrpkg.Commands class
# and have been implemented here
- def sources(self, outdir=None):
- """Download source files"""
-
- # See also:
- # https://lists.fedoraproject.org/pipermail/buildsys/2014-July/004313.html
- #
- # in 'super' the sources function expects a file named 'sources' to be in the base directory.
- # A patch has been sent to upstream to allow a more flexible location.
- #
- # This code doesn't work due to:
- # archive.strip().split(' ', 1) # patch provided to upstream to fix
- #
- # url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module_name,
- # file.replace(' ', '%20'),
- # csum, file.replace(' ', '%20'))
- #
- #os.symlink(os.path.join(self.path, '.{0}.metadata'.format(self.module_name)), os.path.join(self.path, 'sources'))
- #super(Commands, self).sources(outdir=None)
- #os.unlink(os.path.join(self.path, 'sources'))
-
- # The following is copied from rpkg/__init__.py:sources with minor changes
- try:
- archives = open(os.path.join(self.path,
- '.{0}.metadata'.format(self.module_name)),
- 'r').readlines()
- except IOError, e:
- raise rpkgError('%s is not a valid repo: %s' % (self.path, e))
- # Default to putting the files where the module is
- if not outdir:
- outdir = self.path
- for archive in archives:
- try:
- # This strip / split is kind a ugly, but checksums shouldn't have
- # two spaces in them. sources file might need more structure in the
- # future
- csum, file = archive.strip().split(None, 1)
- except ValueError:
- raise rpkgError('Malformed sources file.')
-
- # The default lookaside hash is stored in centpkg.conf, but there is
- # a mix of md5 and sha sums in the CentOS lookaside, here we divine
- # which one we are using
-
- sum_lengths = { 128: 'sha512',
- 64: 'sha256',
- 40: 'sha1',
- 32: 'md5',
- }
-
- self.lookasidehash = sum_lengths[len(csum)]
-
- # If a directory is specified in the metadata file, append it to
- # outdir
- if os.path.dirname(file):
- outdir = os.path.join(self.path, os.path.dirname(file))
- file = os.path.basename(file)
-
- # Create the output directory if it's not checked into git
- if not os.path.exists(outdir):
- self.log.info("Creating OUTDIR: {0}".format(outdir))
- os.makedirs(outdir)
-
- outfile = os.path.join(outdir, file)
- # See if we already have a valid copy downloaded
- if os.path.exists(outfile):
- if self.lookasidecache.file_is_valid(outfile, csum, self.lookasidehash):
- continue
- self.log.info("Downloading %s" % (file))
- filepath = '%s/%s/%s' % (self.module_name,
- self.branch_merge,
- csum,
- )
- url = urlparse.urljoin(self.lookaside, filepath)
- command = ['curl', '-H', 'Pragma:', '-o', outfile, '-R', '-S', '--fail']
- if self.quiet:
- command.append('-s')
- command.append(url)
- self._run_command(command)
- if not self.lookasidecache.file_is_valid(outfile, csum, self.lookasidehash):
- raise rpkgError('%s failed checksum' % file)
-
- return
-
-
- def get_latest_commit(self, *args, **kwargs):
- raise NotImplementedError("get_latest_commit is not yet implemented in centpkg")
-
- def gitbuildhash(self, *args, **kwargs):
- raise NotImplementedError("gitbuildhash is not yet implemented in centpkg")
+ def upload(self, *args, **kwargs):
+ if not self.distgitdir.distrobranch:
+ self.source_entry_type = 'bsd'
+ return super(Commands, self).upload(*args, **kwargs)
def import_srpm(self, *args, **kwargs):
raise NotImplementedError("import_srpm is not yet implemented in centpkg")
- def new(self, *args, **kwargs):
- raise NotImplementedError("new is not yet implemented in centpkg")
-
def patch(self, *args, **kwargs):
raise NotImplementedError("patch is not yet implemented in centpkg")
- def push(self, *args, **kwargs):
- raise NotImplementedError("push is not yet implemented in centpkg")
-
- def file_exists(self, *args, **kwargs):
- raise NotImplementedError("file_exists is not yet implemented in centpkg")
-
- def upload_file(self, *args, **kwargs):
- raise NotImplementedError("upload_file is not yet implemented in centpkg")
-
-
def install(self, *args, **kwargs):
raise NotImplementedError("install is not yet implemented in centpkg")
def lint(self, *args, **kwargs):
raise NotImplementedError("lint is not yet implemented in centpkg")
-
-
- def upload(self, *args, **kwargs):
- raise NotImplementedError("upload is not yet implemented in centpkg")
-
- def prep(self, *args, **kwargs):
- raise NotImplementedError("prep is not yet implemented in centpkg")
-
- def unused_patches(self):
- """Discover patches checked into source control that are not used
-
- Returns a list of unused patches, which may be empty.
- """
-
- # Create a list for unused patches
- unused = []
- # Get the content of spec into memory for fast searching
- spec = open(self.spec, 'r').read()
- # Replace %{name} with the package name
- spec = spec.replace("%{name}", self.module_name)
- # Replace %{version} with the package version
- spec = spec.replace("%{version}", self.ver)
-
- # Get a list of files tracked in source control
- files = self.repo.git.ls_files('--exclude-standard').split()
- for file in map(os.path.basename, files):
- # throw out non patches
- if not file.endswith(('.patch', '.diff')):
- continue
- if file not in spec:
- unused.append(file)
- return unused
-
diff --git a/src/centpkg/__main__.py b/src/centpkg/__main__.py
index 8b1bb76..fd5c7ce 100644
--- a/src/centpkg/__main__.py
+++ b/src/centpkg/__main__.py
@@ -19,8 +19,10 @@ import logging
import ConfigParser
import argparse
+import fedpkg
import pyrpkg
import centpkg
+from pyrpkg.layout import Layout, ExplodedSRPMLayout, DistGitLayout
def main():
'''
@@ -40,8 +42,16 @@ def main():
config = ConfigParser.SafeConfigParser()
config.read(args.config)
- client = centpkg.cli.centpkgClient(config)
- client.do_imports(site='centpkg')
+ layout = Layout.load()
+ if isinstance(layout, ExplodedSRPMLayout):
+ client = centpkg.cli.centpkgClient(config)
+ elif isinstance(layout, DistGitLayout):
+ config.read('/etc/rpkg/fedpkg.conf')
+ client = fedpkg.cli.fedpkgClient(config, name='fedpkg')
+ else:
+ raise ValueError("Not a site we know about")
+
+ client.do_imports(site=client.DEFAULT_CLI_NAME)
client.parse_cmdline()
if not client.args.path:
diff --git a/src/centpkg/cli.py b/src/centpkg/cli.py
index 8207f39..b6c573f 100755
--- a/src/centpkg/cli.py
+++ b/src/centpkg/cli.py
@@ -13,46 +13,11 @@
# option) any later version. See http://www.gnu.org/copyleft/gpl.html for
# the full text of the license.
-import sys
-import os
-import logging
-
+from __future__ import print_function
from pyrpkg.cli import cliClient
+
class centpkgClient(cliClient):
- '''
- Where we import our custom stuff
- '''
- def __init__(self, config, name='centpkg'):
- '''init'''
+ def __init__(self, config, name=None):
+ self.DEFAULT_CLI_NAME = 'centpkg'
super(centpkgClient, self).__init__(config, name)
-
-
-if __name__ == '__main__':
- client = centpkgClient()
- client.do_imports()
- client.parse_cmdline()
-
- if not client.args.path:
- try:
- client.args.path = os.getcwd()
- except OSError as err_msg:
- print('Could not get current path')
- print(err_msg)
- sys.exit(1)
-
- log = client.site.log
- client.setupLogging(log)
-
- if client.args.v:
- log.setLevel(logging.DEBUG)
- elif client.args.q:
- log.setLevel(logging.WARNING)
- else:
- log.setLevel(logging.INFO)
-
- # Run the necessary command
- try:
- client.args.command()
- except KeyboardInterrupt:
- pass
diff --git a/src/centpkg/lookaside.py b/src/centpkg/lookaside.py
new file mode 100644
index 0000000..10fece1
--- /dev/null
+++ b/src/centpkg/lookaside.py
@@ -0,0 +1,185 @@
+# Copyright (c) 2018 - Red Hat Inc.
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version. See http://www.gnu.org/copyleft/gpl.html for
+# the full text of the license.
+
+
+"""Interact with the CentOS lookaside cache
+
+We need to override the pyrpkg.lookasidecache module to handle our custom
+download path.
+"""
+
+import hashlib
+import io
+import logging
+import os
+import sys
+
+import pycurl
+import six
+
+from pyrpkg.errors import DownloadError, InvalidHashType, UploadError
+from pyrpkg.lookaside import CGILookasideCache
+from six.moves import http_client
+
+
+class CentOSLookasideCache(CGILookasideCache):
+ def __init__(self, hashtype, download_url, upload_url, name, branch):
+ super(CentOSLookasideCache, self).__init__(
+ hashtype, download_url, upload_url, client_cert="/home/bstinson/.centos.cert")
+ self.branch = branch
+
+ self.download_path = (
+ '%(name)s/%(branch)s/%(hash)s')
+
+ def remote_file_exists(self, name, filename, hash):
+ """Verify whether a file exists on the lookaside cache
+
+ :param str name: The name of the module. (usually the name of the
+ SRPM). This can include the namespace as well (depending on what
+ the server side expects).
+ :param str filename: The name of the file to check for.
+ :param str hash: The known good hash of the file.
+ """
+
+ # RHEL 7 ships pycurl that does not accept unicode. When given unicode
+ # type it would explode with "unsupported second type in tuple". Let's
+ # convert to str just to be sure.
+ # https://bugzilla.redhat.com/show_bug.cgi?id=1241059
+ if six.PY2 and isinstance(filename, six.text_type):
+ filename = filename.encode('utf-8')
+
+ if six.PY2 and isinstance(self.branch, six.text_type):
+ self.branch = self.branch.encode('utf-8')
+
+ post_data = [('name', name),
+ ('%ssum' % self.hashtype, hash),
+ ('branch', self.branch),
+ ('filename', filename)]
+
+ with io.BytesIO() as buf:
+ c = pycurl.Curl()
+ c.setopt(pycurl.URL, self.upload_url)
+ c.setopt(pycurl.WRITEFUNCTION, buf.write)
+ c.setopt(pycurl.HTTPPOST, post_data)
+
+ if self.client_cert is not None:
+ if os.path.exists(self.client_cert):
+ c.setopt(pycurl.SSLCERT, self.client_cert)
+ else:
+ self.log.warning("Missing certificate: %s"
+ % self.client_cert)
+
+ if self.ca_cert is not None:
+ if os.path.exists(self.ca_cert):
+ c.setopt(pycurl.CAINFO, self.ca_cert)
+ else:
+ self.log.warning("Missing certificate: %s", self.ca_cert)
+
+ c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
+ c.setopt(pycurl.USERPWD, ':')
+
+ try:
+ c.perform()
+ status = c.getinfo(pycurl.RESPONSE_CODE)
+
+ except Exception as e:
+ raise UploadError(e)
+
+ finally:
+ c.close()
+
+ output = buf.getvalue().strip()
+
+ if status != 200:
+ self.raise_upload_error(status)
+
+ # Lookaside CGI script returns these strings depending on whether
+ # or not the file exists:
+ if output == b'Available':
+ return True
+
+ if output == b'Missing':
+ return False
+
+ # Something unexpected happened
+ self.log.debug(output)
+ raise UploadError('Error checking for %s at %s'
+ % (filename, self.upload_url))
+
+ def upload(self, name, filepath, hash):
+ """Upload a source file
+
+ :param str name: The name of the module. (usually the name of the SRPM)
+ This can include the namespace as well (depending on what the
+ server side expects).
+ :param str filepath: The full path to the file to upload.
+ :param str hash: The known good hash of the file.
+ """
+ filename = os.path.basename(filepath)
+
+ # As in remote_file_exists, we need to convert unicode strings to str
+ if six.PY2:
+ if isinstance(name, six.text_type):
+ name = name.encode('utf-8')
+ if isinstance(filepath, six.text_type):
+ filepath = filepath.encode('utf-8')
+
+ if self.remote_file_exists(name, filename, hash):
+ self.log.info("File already uploaded: %s", filepath)
+ return
+
+ self.log.info("Uploading: %s", filepath)
+ post_data = [('name', name),
+ ('%ssum' % self.hashtype, hash),
+ ('branch', self.branch),
+ ('file', (pycurl.FORM_FILE, filepath))]
+
+ with io.BytesIO() as buf:
+ c = pycurl.Curl()
+ c.setopt(pycurl.URL, self.upload_url)
+ c.setopt(pycurl.NOPROGRESS, False)
+ c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress)
+ c.setopt(pycurl.WRITEFUNCTION, buf.write)
+ c.setopt(pycurl.HTTPPOST, post_data)
+
+ if self.client_cert is not None:
+ if os.path.exists(self.client_cert):
+ c.setopt(pycurl.SSLCERT, self.client_cert)
+ else:
+ self.log.warning("Missing certificate: %s", self.client_cert)
+
+ if self.ca_cert is not None:
+ if os.path.exists(self.ca_cert):
+ c.setopt(pycurl.CAINFO, self.ca_cert)
+ else:
+ self.log.warning("Missing certificate: %s", self.ca_cert)
+
+ c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
+ c.setopt(pycurl.USERPWD, ':')
+
+ try:
+ c.perform()
+ status = c.getinfo(pycurl.RESPONSE_CODE)
+
+ except Exception as e:
+ raise UploadError(e)
+
+ finally:
+ c.close()
+
+ output = buf.getvalue().strip()
+
+ # Get back a new line, after displaying the download progress
+ sys.stdout.write('\n')
+ sys.stdout.flush()
+
+ if status != 200:
+ self.raise_upload_error(status)
+
+ if output:
+ self.log.debug(output)