lrossett / centos / centpkg

Forked from centos/centpkg 3 years ago
Clone
85a850
#pylint: disable=line-too-long,abstract-class-not-used
85a850
'''
85a850
    Top level function library for centpkg
85a850
'''
85a850
#
85a850
# Author(s):
85a850
#            Jesse Keating <jkeating@redhat.com>
85a850
#            Pat Riehecky <riehecky@fnal.gov>
85a850
#            Brian Stinson <bstinson@ksu.edu>
85a850
#
85a850
# This program is free software; you can redistribute it and/or modify it
85a850
# under the terms of the GNU General Public License as published by the
85a850
# Free Software Foundation; either version 2 of the License, or (at your
85a850
# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
85a850
# the full text of the license.
85a850
85a850
85a850
Brian Stinson ca61eb
import os
Brian Stinson ca61eb
import re
e7fd56
import warnings
Brian Stinson 2fb8a5
ebeaf1
from pyrpkg import Commands, rpkgError
Brian Stinson 6fc4bb
from . import centos_cert
Brian Stinson 2fb8a5
from . import cli
Brian Stinson 2fb8a5
7ae64c
class DistGitDirectory(object):
7ae64c
86b0d3
    signame = None
86b0d3
    centosversion = None
86b0d3
    projectname = None
86b0d3
    releasename = None
639034
    distrobranch = False
86b0d3
7ae64c
    def __init__(self, branchtext):
7ae64c
        sigtobranchre = r'sig-(?P<signame>\w+)(?P<centosversion>\d)-?(?P<projectname>\w+)?-?(?P<releasename>\w+)?'
639034
        distrobranchre = r'c(?P<centosversion>\d+)-?(?P<projectname>\w+)?'
e7fd56
        oldbranchre = r'(?P<signame>\w+)(?P<centosversion>\d)'
639034
        sigmatch = re.match(sigtobranchre, branchtext)
639034
        distromatch = re.match(distrobranchre, branchtext)
e7fd56
        oldbranchmatch = re.match(oldbranchre, branchtext)
639034
        if sigmatch:
639034
            gd = sigmatch.groupdict()
9af8c4
            self.signame = gd['signame']
9af8c4
            self.centosversion = gd['centosversion']
7ae64c
35f2c1
            # Users have the option to specify (or not specify) common in their
35f2c1
            # git repos. Ww need to handle these cases because common is not a
35f2c1
            # project nor is it a release.
9af8c4
            if gd['projectname'] != 'common':
9af8c4
                self.projectname = gd['projectname']
9af8c4
            if gd['releasename'] != 'common':
9af8c4
                self.releasename = gd['releasename']
639034
        elif distromatch:
639034
            gd = distromatch.groupdict()
639034
            self.distrobranch = True
639034
            self.signame = 'centos'
639034
            self.centosversion = gd['centosversion']
639034
639034
            if gd['projectname'] != 'common':
639034
                self.projectname = gd['projectname']
e7fd56
        elif oldbranchmatch:
e7fd56
            warnings.warn("This branch is deprecated and will be removed soon",
e7fd56
                          DeprecationWarning)
639034
        else:
639034
            raise ValueError("Branchname: {0} is not valid".format(branchtext))
9af8c4
9af8c4
    @property
9af8c4
    def target(self):
9af8c4
        projectorcommon = self.projectname
9af8c4
        releaseorcommon = self.releasename
9af8c4
639034
        if self.distrobranch:
639034
            return '-'.join(filter(None, ['c'+self.centosversion,
639034
                                          projectorcommon]))
639034
9af8c4
        if not releaseorcommon:
9af8c4
            if not projectorcommon or projectorcommon == 'common':
9af8c4
                projectorcommon = 'common'
9af8c4
            else:
9af8c4
                releaseorcommon = 'common'
9af8c4
9af8c4
        return '-'.join(filter(None, [self.signame+self.centosversion,
9af8c4
                                      projectorcommon, releaseorcommon])) + '-el{0}'.format(self.centosversion)
9af8c4
9af8c4
class Commands(Commands):
85a850
    '''
85a850
        For the pyrpkg commands with centpkg behavior
85a850
    '''
Brian Stinson 2fb8a5
    def __init__(self, path, lookaside, lookasidehash, lookaside_cgi,
Brian Stinson 2fb8a5
                 gitbaseurl, anongiturl, branchre, kojiconfig,
Brian Stinson 2fb8a5
                 build_client, user=None, dist=None, target=None,
Brian Stinson 2fb8a5
                 quiet=False):
85a850
        '''
85a850
            Init the object and some configuration details.
85a850
        '''
85a850
        super(Commands, self).__init__(path, lookaside, lookasidehash,
Brian Stinson 2fb8a5
                                      lookaside_cgi, gitbaseurl, anongiturl,
Brian Stinson 2fb8a5
                                      branchre, kojiconfig, build_client,
85a850
                                      user, dist, target, quiet)
Brian Stinson 2fb8a5
639034
        self.distgitdir = DistGitDirectory(self.branch_merge)
639034
Brian Stinson ca61eb
    # redefined loaders
Brian Stinson ca61eb
    def load_rpmdefines(self):
85a850
        '''
85a850
            Populate rpmdefines based on branch data
85a850
        '''
a1a2e2
a1a2e2
        if not self.distgitdir.centosversion:
a1a2e2
            raise rpkgError('Could not get the OS version from the branch:{0}'.format(self.branch_merge))
a1a2e2
a1a2e2
        self._distval = self.distgitdir.centosversion
Brian Stinson ca61eb
        self._distval = self._distval.replace('.', '_')
Brian Stinson f12d46
Brian Stinson d6d6d7
        self._disttag = 'el%s' % self._distval
Brian Stinson f12d46
Brian Stinson 2eaea9
        self._rpmdefines = ["--define '_topdir {0}'".format(self.path),
Brian Stinson d653ab
                            "--define '_srcrpmdir {0}'".format(self.path),
Brian Stinson d653ab
                            "--define '_rpmdir {0}'".format(self.path),
Brian Stinson 2eaea9
                            "--define 'dist .{0}'".format(self._disttag),
Brian Stinson ca61eb
                            # int and float this to remove the decimal
Brian Stinson 2eaea9
                            "--define '{0} 1'".format(self._disttag)]
Brian Stinson ca61eb
Brian Stinson ca61eb
    def load_spec(self):
Brian Stinson ca61eb
        """This sets the spec attribute"""
Brian Stinson ca61eb
Brian Stinson ca61eb
        # We are not using the upstream load_spec because the file structure is
Brian Stinson ca61eb
        # hard-coded
Brian Stinson ca61eb
Brian Stinson ca61eb
        # Get a list of files in the path we're looking at
Brian Stinson ca61eb
        files = os.listdir(os.path.join(self.path,'SPECS'))
Brian Stinson ca61eb
        # Search the files for the first one that ends with ".spec"
85a850
        for __f in files:
85a850
            if __f.endswith('.spec') and not __f.startswith('.'):
85a850
                self._spec = os.path.join('SPECS', __f)
Brian Stinson ca61eb
                return
Brian Stinson 8f2a1a
ebeaf1
        raise rpkgError('No spec file found.')
Brian Stinson ca61eb
Brian Stinson 6ba768
    def load_target(self):
Brian Stinson 6ba768
        """ This sets the target attribute (used for mock and koji) """
Brian Stinson 3f76b3
Brian Stinson 3f76b3
        # Distribution branches start with c and may or may not end in -plus
Brian Stinson 3f76b3
        # otherwise, it's a sig branch 
3ab1c8
        if not self.distgitdir.distrobranch:
Brian Stinson 3f76b3
            # send distribution packages to build on the the bananas tags for now
Brian Stinson 3f76b3
            self._target = 'bananas{0}-{1}'.format(self.distval, self.disttag)
Brian Stinson 3f76b3
        else:
Brian Stinson 3f76b3
            # sig packages are built in the tag that matches the git branch
3ab1c8
            self._target = self.distgitdir.target
Brian Stinson 6ba768
Brian Stinson 046d91
    def load_user(self):
Brian Stinson 046d91
        try:
Brian Stinson 046d91
            self._user = centos_cert.read_user_cert()
Brian Stinson 046d91
        except Exception, e:
Brian Stinson 046d91
            super(Commands, self).load_user()
Brian Stinson 046d91
85a850
    # These are the commands defined in the base pyrpkg.Commands class
85a850
    # and have been implemented here
85a850
85a850
    def sources(self, outdir=None):
85a850
        """Download source files"""
85a850
9f3a0f
        #  See also:
9f3a0f
        # https://lists.fedoraproject.org/pipermail/buildsys/2014-July/004313.html
9f3a0f
        #
85a850
        # in 'super' the sources function expects a file named 'sources' to be in the base directory.
85a850
        # A patch has been sent to upstream to allow a more flexible location.
85a850
        #
85a850
        # This code doesn't work due to:
85a850
        #              archive.strip().split('  ', 1) # patch provided to upstream to fix
85a850
        #
85a850
        #              url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module_name,
85a850
        #                                        file.replace(' ', '%20'),
85a850
        #                                        csum, file.replace(' ', '%20'))
85a850
        #
85a850
        #os.symlink(os.path.join(self.path, '.{0}.metadata'.format(self.module_name)), os.path.join(self.path, 'sources'))
85a850
        #super(Commands, self).sources(outdir=None)
85a850
        #os.unlink(os.path.join(self.path, 'sources'))
85a850
85a850
        # The following is copied from rpkg/__init__.py:sources with minor changes
85a850
        try:
85a850
            archives = open(os.path.join(self.path,
85a850
                                         '.{0}.metadata'.format(self.module_name)),
85a850
                            'r').readlines()
85a850
        except IOError, e:
ebeaf1
            raise rpkgError('%s is not a valid repo: %s' % (self.path, e))
85a850
        # Default to putting the files where the module is
85a850
        if not outdir:
85a850
            outdir = self.path
85a850
        for archive in archives:
85a850
            try:
85a850
                # This strip / split is kind a ugly, but checksums shouldn't have
85a850
                # two spaces in them.  sources file might need more structure in the
85a850
                # future
85a850
                csum, file = archive.strip().split(None, 1)
85a850
            except ValueError:
ebeaf1
                raise rpkgError('Malformed sources file.')
Brian Stinson 5a3f50
Brian Stinson 3d4268
            # The default lookaside hash is stored in centpkg.conf, but there is
Brian Stinson 3d4268
            # a mix of md5 and sha sums in the CentOS lookaside, here we divine
Brian Stinson 3d4268
            # which one we are using
Brian Stinson 3d4268
Brian Stinson 3d4268
            sum_lengths = { 128: 'sha512',
Brian Stinson 3d4268
                            64: 'sha256',
Brian Stinson 3d4268
                            40: 'sha1',
Brian Stinson 3d4268
                            32: 'md5',
Brian Stinson 3d4268
                          }
Brian Stinson 3d4268
Brian Stinson 3d4268
            self.lookasidehash = sum_lengths[len(csum)]
Brian Stinson 3d4268
Brian Stinson 5a3f50
            # If a directory is specified in the metadata file, append it to
Brian Stinson 5a3f50
            # outdir
Brian Stinson 5a3f50
            if os.path.dirname(file):
Brian Stinson 5a3f50
                outdir = os.path.join(self.path, os.path.dirname(file))
Brian Stinson 5a3f50
                file = os.path.basename(file)
Brian Stinson 5a3f50
Brian Stinson 5a3f50
            # Create the output directory if it's not checked into git
Brian Stinson 5a3f50
            if not os.path.exists(outdir):
Brian Stinson 5a3f50
                self.log.info("Creating OUTDIR: {0}".format(outdir))
Brian Stinson 5a3f50
                os.makedirs(outdir)
Brian Stinson 5a3f50
85a850
            outfile = os.path.join(outdir, file)
Brian Stinson 5a3f50
            # See if we already have a valid copy downloaded
85a850
            if os.path.exists(outfile):
85a850
                if self._verify_file(outfile, csum, self.lookasidehash):
85a850
                    continue
85a850
            self.log.info("Downloading %s" % (file))
85a850
            url = '%s/%s/%s/%s' % (self.lookaside, self.module_name,
85a850
                                      self.branch_merge,
85a850
                                      csum,
85a850
                                      )
85a850
            command = ['curl', '-H', 'Pragma:', '-o', outfile, '-R', '-S', '--fail']
85a850
            if self.quiet:
85a850
                command.append('-s')
85a850
            command.append(url)
85a850
            self._run_command(command)
85a850
            if not self._verify_file(outfile, csum, self.lookasidehash):
ebeaf1
                raise rpkgError('%s failed checksum' % file)
85a850
85a850
        return
Brian Stinson ca61eb
85a850
Brian Stinson 2fb8a5
    def get_latest_commit(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("get_latest_commit is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def gitbuildhash(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("gitbuildhash is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def import_srpm(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("import_srpm is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def new(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("new is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def patch(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("patch is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def push(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("push is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def file_exists(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("file_exists is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def upload_file(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("upload_file is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def install(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("install is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def lint(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("lint is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def upload(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("upload is not yet implemented in centpkg")
Brian Stinson 2fb8a5
Brian Stinson 2fb8a5
    def prep(self, *args, **kwargs):
Brian Stinson ae56d4
        raise NotImplementedError("prep is not yet implemented in centpkg")
Brian Stinson 2fb8a5
1fe6e4
    def unused_patches(self):
1fe6e4
        """Discover patches checked into source control that are not used
1fe6e4
1fe6e4
        Returns a list of unused patches, which may be empty.
1fe6e4
        """
1fe6e4
1fe6e4
        # Create a list for unused patches
1fe6e4
        unused = []
1fe6e4
        # Get the content of spec into memory for fast searching
1fe6e4
        spec = open(self.spec, 'r').read()
1fe6e4
        # Replace %{name} with the package name
1fe6e4
        spec = spec.replace("%{name}", self.module_name)
1fe6e4
        # Replace %{version} with the package version
1fe6e4
        spec = spec.replace("%{version}", self.ver)
1fe6e4
1fe6e4
        # Get a list of files tracked in source control
1fe6e4
        files = self.repo.git.ls_files('--exclude-standard').split()
1fe6e4
        for file in map(os.path.basename, files):
1fe6e4
            # throw out non patches
1fe6e4
            if not file.endswith(('.patch', '.diff')):
1fe6e4
                continue
1fe6e4
            if file not in spec:
1fe6e4
                unused.append(file)
1fe6e4
        return unused
Brian Stinson 2fb8a5