lrossett / centos / centpkg

Forked from centos/centpkg 3 years ago
Clone
a394b1
# Copyright (c) 2018 - Red Hat Inc.
a394b1
#
a394b1
# This program is free software; you can redistribute it and/or modify it
a394b1
# under the terms of the GNU General Public License as published by the
a394b1
# Free Software Foundation; either version 2 of the License, or (at your
a394b1
# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
a394b1
# the full text of the license.
a394b1
a394b1
Brian Stinson 3edf28
"""Interact with the Red Hat lookaside cache
a394b1
a394b1
We need to override the pyrpkg.lookasidecache module to handle our custom
a394b1
download path.
a394b1
"""
a394b1
a394b1
import io
a394b1
import os
a394b1
import pycurl
a394b1
import six
Brian Stinson 3edf28
import sys
Brian Stinson 3edf28
29dd93
from pyrpkg.errors import InvalidHashType, UploadError, LayoutError
a394b1
from pyrpkg.lookaside import CGILookasideCache
29dd93
from pyrpkg.layout.layouts import DistGitLayout
29dd93
29dd93
from . import utils
29dd93
29dd93
29dd93
def is_dist_git(folder):
29dd93
    """
29dd93
    Indicates if a folder is using a dist-git layout.
29dd93
29dd93
    Parameters
29dd93
    ----------
29dd93
    folder: str
29dd93
        The directory to inspect.
29dd93
29dd93
    Returns
29dd93
    -------
29dd93
    bool
29dd93
        A bool flag indicating if `folder` is using
29dd93
        a dist-git layout format.
29dd93
    """
29dd93
    result = False
29dd93
29dd93
    try:
29dd93
        DistGitLayout.from_path(folder)
29dd93
        result = True
29dd93
    except LayoutError:
29dd93
        result = False
29dd93
    finally:
29dd93
        return result
a394b1
a394b1
Brian Stinson 3edf28
class StreamLookasideCache(CGILookasideCache):
29dd93
    """
29dd93
    CentosStream lookaside specialized class.
29dd93
29dd93
    It inherits most of its behavior from `pyrpkg.lookasideCGILookasideCache`.
29dd93
    """
29dd93
29dd93
    def __init__(self, hashtype, download_url, upload_url, client_cert=None, ca_cert=None):
Brian Stinson 3edf28
        super(StreamLookasideCache, self).__init__(
29dd93
            hashtype, download_url, upload_url,
29dd93
            client_cert=client_cert, ca_cert=ca_cert)
Brian Stinson 3edf28
29dd93
    def remote_file_exists(self, name, filename, hashstr):
29dd93
        """
29dd93
        Check if a remote file exists.
29dd93
29dd93
        This method inherits the behavior of its parent class from pyrpkg.
29dd93
29dd93
        It uses the internal `utils.get_repo_name` mehtod to parse the name in case
29dd93
        it is a scm url. 
29dd93
29dd93
        Parameters
29dd93
        ----------
29dd93
        name: str
29dd93
            The repository name and org.
29dd93
29dd93
        filename: str
29dd93
            The filename (something.tar.gz).
29dd93
29dd93
        hash:
29dd93
            The hash string for the file.
29dd93
29dd93
        Returns
29dd93
        -------
29dd93
        bool
29dd93
            A boolean value to inditicate if the file exists.
29dd93
        """
29dd93
        _name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
29dd93
29dd93
        return super(StreamLookasideCache, self).remote_file_exists(
29dd93
                _name, filename, hashstr)
29dd93
29dd93
    def upload(self, name, filename, hashstr, offline=False):
29dd93
        """
29dd93
        Uploads a file to lookaside cache.
29dd93
29dd93
        This method inherits the behavior of its parent class from pyrpkg.
29dd93
29dd93
        It uses the internal `utils.get_repo_name` mehtod to parse the name in case
29dd93
        it is a scm url. 
29dd93
29dd93
        Parameters
29dd93
        ----------
29dd93
        name: str
29dd93
            The repository name and org.
29dd93
29dd93
        filename: str
29dd93
            The filename (something.tar.gz).
29dd93
29dd93
        hash:
29dd93
            The hash string for the file.
29dd93
29dd93
        Raises
29dd93
        ------
29dd93
        pyrpkg.errors.rpkgError
29dd93
            Raises specialized classes that inherits from pyrokg base errors.
29dd93
29dd93
        Returns
29dd93
        -------
29dd93
        None
29dd93
            Does not return anything
29dd93
        """
29dd93
        _name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
29dd93
29dd93
        return super(StreamLookasideCache, self).upload(
29dd93
            _name, filename, hashstr)
29dd93
29dd93
    def download(self, name, filename, hashstr, outfile, hashtype=None, **kwargs):
29dd93
        """
29dd93
        Downloads a file from lookaside cache to the local filesystem.
29dd93
29dd93
        This method inherits the behavior of its parent class from pyrpkg.
29dd93
29dd93
        It uses the internal `utils.get_repo_name` mehtod to parse the name in case
29dd93
        it is a scm url. 
29dd93
29dd93
        Parameters
29dd93
        ----------
29dd93
        name: str
29dd93
            The repository name and org.
29dd93
29dd93
        filename: str
29dd93
            The filename (something.tar.gz).
29dd93
29dd93
        hash: str
29dd93
            The hash string for the file.
29dd93
29dd93
        outfile: str
29dd93
29dd93
29dd93
        Raises
29dd93
        ------
29dd93
        pyrpkg.errors.rpkgError
29dd93
            Raises specialized implementations of  `yrpkg.errors.rpkgError`.
29dd93
29dd93
        Returns
29dd93
        -------
29dd93
        None
29dd93
            Does not return anything
29dd93
        """
29dd93
        _name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
29dd93
29dd93
        return super(StreamLookasideCache, self).download(
29dd93
           _name, filename, hashstr, outfile, hashtype=hashtype, **kwargs)
Brian Stinson 3edf28
Brian Stinson 3edf28
Brian Stinson 3edf28
class SIGLookasideCache(CGILookasideCache):
a394b1
    def __init__(self, hashtype, download_url, upload_url, name, branch):
Brian Stinson 3edf28
        super(SIGLookasideCache, self).__init__(
a394b1
            hashtype, download_url, upload_url, client_cert="/home/bstinson/.centos.cert")
a394b1
        self.branch = branch
a394b1
a394b1
        self.download_path = (
a394b1
            '%(name)s/%(branch)s/%(hash)s')
a394b1
a394b1
    def remote_file_exists(self, name, filename, hash):
a394b1
        """Verify whether a file exists on the lookaside cache
a394b1
a394b1
        :param str name: The name of the module. (usually the name of the
a394b1
            SRPM). This can include the namespace as well (depending on what
a394b1
            the server side expects).
a394b1
        :param str filename: The name of the file to check for.
a394b1
        :param str hash: The known good hash of the file.
a394b1
        """
a394b1
a394b1
        # RHEL 7 ships pycurl that does not accept unicode. When given unicode
a394b1
        # type it would explode with "unsupported second type in tuple". Let's
a394b1
        # convert to str just to be sure.
a394b1
        # https://bugzilla.redhat.com/show_bug.cgi?id=1241059
29dd93
        _name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
29dd93
a394b1
        if six.PY2 and isinstance(filename, six.text_type):
a394b1
            filename = filename.encode('utf-8')
a394b1
a394b1
        if six.PY2 and isinstance(self.branch, six.text_type):
a394b1
            self.branch = self.branch.encode('utf-8')
a394b1
29dd93
        post_data = [('name', _name),
a394b1
                     ('%ssum' % self.hashtype, hash),
a394b1
                     ('branch', self.branch),
a394b1
                     ('filename', filename)]
a394b1
a394b1
        with io.BytesIO() as buf:
a394b1
            c = pycurl.Curl()
a394b1
            c.setopt(pycurl.URL, self.upload_url)
a394b1
            c.setopt(pycurl.WRITEFUNCTION, buf.write)
a394b1
            c.setopt(pycurl.HTTPPOST, post_data)
a394b1
a394b1
            if self.client_cert is not None:
a394b1
                if os.path.exists(self.client_cert):
a394b1
                    c.setopt(pycurl.SSLCERT, self.client_cert)
a394b1
                else:
a394b1
                    self.log.warning("Missing certificate: %s"
a394b1
                                     % self.client_cert)
a394b1
a394b1
            if self.ca_cert is not None:
a394b1
                if os.path.exists(self.ca_cert):
a394b1
                    c.setopt(pycurl.CAINFO, self.ca_cert)
a394b1
                else:
a394b1
                    self.log.warning("Missing certificate: %s", self.ca_cert)
a394b1
a394b1
            c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
a394b1
            c.setopt(pycurl.USERPWD, ':')
a394b1
a394b1
            try:
a394b1
                c.perform()
a394b1
                status = c.getinfo(pycurl.RESPONSE_CODE)
a394b1
a394b1
            except Exception as e:
a394b1
                raise UploadError(e)
a394b1
a394b1
            finally:
a394b1
                c.close()
a394b1
a394b1
            output = buf.getvalue().strip()
a394b1
a394b1
        if status != 200:
a394b1
            self.raise_upload_error(status)
a394b1
a394b1
        # Lookaside CGI script returns these strings depending on whether
a394b1
        # or not the file exists:
a394b1
        if output == b'Available':
a394b1
            return True
a394b1
a394b1
        if output == b'Missing':
a394b1
            return False
a394b1
a394b1
        # Something unexpected happened
a394b1
        self.log.debug(output)
a394b1
        raise UploadError('Error checking for %s at %s'
a394b1
                          % (filename, self.upload_url))
a394b1
a394b1
    def upload(self, name, filepath, hash):
a394b1
        """Upload a source file
a394b1
a394b1
        :param str name: The name of the module. (usually the name of the SRPM)
a394b1
            This can include the namespace as well (depending on what the
a394b1
            server side expects).
a394b1
        :param str filepath: The full path to the file to upload.
a394b1
        :param str hash: The known good hash of the file.
a394b1
        """
a394b1
        filename = os.path.basename(filepath)
a394b1
a394b1
        # As in remote_file_exists, we need to convert unicode strings to str
a394b1
        if six.PY2:
a394b1
            if isinstance(name, six.text_type):
a394b1
                name = name.encode('utf-8')
a394b1
            if isinstance(filepath, six.text_type):
a394b1
                filepath = filepath.encode('utf-8')
a394b1
a394b1
        if self.remote_file_exists(name, filename, hash):
a394b1
            self.log.info("File already uploaded: %s", filepath)
a394b1
            return
a394b1
a394b1
        self.log.info("Uploading: %s", filepath)
a394b1
        post_data = [('name', name),
a394b1
                     ('%ssum' % self.hashtype, hash),
a394b1
                     ('branch', self.branch),
a394b1
                     ('file', (pycurl.FORM_FILE, filepath))]
a394b1
a394b1
        with io.BytesIO() as buf:
a394b1
            c = pycurl.Curl()
a394b1
            c.setopt(pycurl.URL, self.upload_url)
a394b1
            c.setopt(pycurl.NOPROGRESS, False)
a394b1
            c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress)
a394b1
            c.setopt(pycurl.WRITEFUNCTION, buf.write)
a394b1
            c.setopt(pycurl.HTTPPOST, post_data)
a394b1
a394b1
            if self.client_cert is not None:
a394b1
                if os.path.exists(self.client_cert):
a394b1
                    c.setopt(pycurl.SSLCERT, self.client_cert)
a394b1
                else:
a394b1
                    self.log.warning("Missing certificate: %s", self.client_cert)
a394b1
a394b1
            if self.ca_cert is not None:
a394b1
                if os.path.exists(self.ca_cert):
a394b1
                    c.setopt(pycurl.CAINFO, self.ca_cert)
a394b1
                else:
a394b1
                    self.log.warning("Missing certificate: %s", self.ca_cert)
a394b1
a394b1
            c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
a394b1
            c.setopt(pycurl.USERPWD, ':')
a394b1
a394b1
            try:
a394b1
                c.perform()
a394b1
                status = c.getinfo(pycurl.RESPONSE_CODE)
a394b1
a394b1
            except Exception as e:
a394b1
                raise UploadError(e)
a394b1
a394b1
            finally:
a394b1
                c.close()
a394b1
a394b1
            output = buf.getvalue().strip()
a394b1
a394b1
        # Get back a new line, after displaying the download progress
a394b1
        sys.stdout.write('\n')
a394b1
        sys.stdout.flush()
a394b1
a394b1
        if status != 200:
a394b1
            self.raise_upload_error(status)
a394b1
a394b1
        if output:
a394b1
            self.log.debug(output)