Blame src/centpkg/lookaside.py

a394b1
# Copyright (c) 2018 - Red Hat Inc.
a394b1
#
a394b1
# This program is free software; you can redistribute it and/or modify it
a394b1
# under the terms of the GNU General Public License as published by the
a394b1
# Free Software Foundation; either version 2 of the License, or (at your
a394b1
# option) any later version.  See http://www.gnu.org/copyleft/gpl.html for
a394b1
# the full text of the license.
a394b1
a394b1
Brian Stinson 3edf28
"""Interact with the Red Hat lookaside cache
a394b1
a394b1
We need to override the pyrpkg.lookasidecache module to handle our custom
a394b1
download path.
a394b1
"""
a394b1
a394b1
import io
a394b1
import os
a394b1
import pycurl
a394b1
import six
Brian Stinson 3edf28
import sys
Brian Stinson 3edf28
a394b1
Brian Stinson 3edf28
from pyrpkg.errors import InvalidHashType, UploadError
a394b1
from pyrpkg.lookaside import CGILookasideCache
a394b1
a394b1
Brian Stinson 3edf28
class StreamLookasideCache(CGILookasideCache):
Brian Stinson 3edf28
    def __init__(self, hashtype, download_url, upload_url):
Brian Stinson 3edf28
        super(StreamLookasideCache, self).__init__(
Brian Stinson 3edf28
            hashtype, download_url, upload_url)
Brian Stinson 3edf28
Brian Stinson 3edf28
        self.download_path = (
Brian Stinson 3edf28
            '%(name)s/%(filename)s/%(hashtype)s/%(hash)s/%(filename)s')
Brian Stinson 3edf28
Brian Stinson 3edf28
Brian Stinson 3edf28
class SIGLookasideCache(CGILookasideCache):
a394b1
    def __init__(self, hashtype, download_url, upload_url, name, branch):
Brian Stinson 3edf28
        super(SIGLookasideCache, self).__init__(
a394b1
            hashtype, download_url, upload_url, client_cert="/home/bstinson/.centos.cert")
a394b1
        self.branch = branch
a394b1
a394b1
        self.download_path = (
a394b1
            '%(name)s/%(branch)s/%(hash)s')
a394b1
a394b1
    def remote_file_exists(self, name, filename, hash):
a394b1
        """Verify whether a file exists on the lookaside cache
a394b1
a394b1
        :param str name: The name of the module. (usually the name of the
a394b1
            SRPM). This can include the namespace as well (depending on what
a394b1
            the server side expects).
a394b1
        :param str filename: The name of the file to check for.
a394b1
        :param str hash: The known good hash of the file.
a394b1
        """
a394b1
a394b1
        # RHEL 7 ships pycurl that does not accept unicode. When given unicode
a394b1
        # type it would explode with "unsupported second type in tuple". Let's
a394b1
        # convert to str just to be sure.
a394b1
        # https://bugzilla.redhat.com/show_bug.cgi?id=1241059
a394b1
        if six.PY2 and isinstance(filename, six.text_type):
a394b1
            filename = filename.encode('utf-8')
a394b1
a394b1
        if six.PY2 and isinstance(self.branch, six.text_type):
a394b1
            self.branch = self.branch.encode('utf-8')
a394b1
a394b1
        post_data = [('name', name),
a394b1
                     ('%ssum' % self.hashtype, hash),
a394b1
                     ('branch', self.branch),
a394b1
                     ('filename', filename)]
a394b1
a394b1
        with io.BytesIO() as buf:
a394b1
            c = pycurl.Curl()
a394b1
            c.setopt(pycurl.URL, self.upload_url)
a394b1
            c.setopt(pycurl.WRITEFUNCTION, buf.write)
a394b1
            c.setopt(pycurl.HTTPPOST, post_data)
a394b1
a394b1
            if self.client_cert is not None:
a394b1
                if os.path.exists(self.client_cert):
a394b1
                    c.setopt(pycurl.SSLCERT, self.client_cert)
a394b1
                else:
a394b1
                    self.log.warning("Missing certificate: %s"
a394b1
                                     % self.client_cert)
a394b1
a394b1
            if self.ca_cert is not None:
a394b1
                if os.path.exists(self.ca_cert):
a394b1
                    c.setopt(pycurl.CAINFO, self.ca_cert)
a394b1
                else:
a394b1
                    self.log.warning("Missing certificate: %s", self.ca_cert)
a394b1
a394b1
            c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
a394b1
            c.setopt(pycurl.USERPWD, ':')
a394b1
a394b1
            try:
a394b1
                c.perform()
a394b1
                status = c.getinfo(pycurl.RESPONSE_CODE)
a394b1
a394b1
            except Exception as e:
a394b1
                raise UploadError(e)
a394b1
a394b1
            finally:
a394b1
                c.close()
a394b1
a394b1
            output = buf.getvalue().strip()
a394b1
a394b1
        if status != 200:
a394b1
            self.raise_upload_error(status)
a394b1
a394b1
        # Lookaside CGI script returns these strings depending on whether
a394b1
        # or not the file exists:
a394b1
        if output == b'Available':
a394b1
            return True
a394b1
a394b1
        if output == b'Missing':
a394b1
            return False
a394b1
a394b1
        # Something unexpected happened
a394b1
        self.log.debug(output)
a394b1
        raise UploadError('Error checking for %s at %s'
a394b1
                          % (filename, self.upload_url))
a394b1
a394b1
    def upload(self, name, filepath, hash):
a394b1
        """Upload a source file
a394b1
a394b1
        :param str name: The name of the module. (usually the name of the SRPM)
a394b1
            This can include the namespace as well (depending on what the
a394b1
            server side expects).
a394b1
        :param str filepath: The full path to the file to upload.
a394b1
        :param str hash: The known good hash of the file.
a394b1
        """
a394b1
        filename = os.path.basename(filepath)
a394b1
a394b1
        # As in remote_file_exists, we need to convert unicode strings to str
a394b1
        if six.PY2:
a394b1
            if isinstance(name, six.text_type):
a394b1
                name = name.encode('utf-8')
a394b1
            if isinstance(filepath, six.text_type):
a394b1
                filepath = filepath.encode('utf-8')
a394b1
a394b1
        if self.remote_file_exists(name, filename, hash):
a394b1
            self.log.info("File already uploaded: %s", filepath)
a394b1
            return
a394b1
a394b1
        self.log.info("Uploading: %s", filepath)
a394b1
        post_data = [('name', name),
a394b1
                     ('%ssum' % self.hashtype, hash),
a394b1
                     ('branch', self.branch),
a394b1
                     ('file', (pycurl.FORM_FILE, filepath))]
a394b1
a394b1
        with io.BytesIO() as buf:
a394b1
            c = pycurl.Curl()
a394b1
            c.setopt(pycurl.URL, self.upload_url)
a394b1
            c.setopt(pycurl.NOPROGRESS, False)
a394b1
            c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress)
a394b1
            c.setopt(pycurl.WRITEFUNCTION, buf.write)
a394b1
            c.setopt(pycurl.HTTPPOST, post_data)
a394b1
a394b1
            if self.client_cert is not None:
a394b1
                if os.path.exists(self.client_cert):
a394b1
                    c.setopt(pycurl.SSLCERT, self.client_cert)
a394b1
                else:
a394b1
                    self.log.warning("Missing certificate: %s", self.client_cert)
a394b1
a394b1
            if self.ca_cert is not None:
a394b1
                if os.path.exists(self.ca_cert):
a394b1
                    c.setopt(pycurl.CAINFO, self.ca_cert)
a394b1
                else:
a394b1
                    self.log.warning("Missing certificate: %s", self.ca_cert)
a394b1
a394b1
            c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
a394b1
            c.setopt(pycurl.USERPWD, ':')
a394b1
a394b1
            try:
a394b1
                c.perform()
a394b1
                status = c.getinfo(pycurl.RESPONSE_CODE)
a394b1
a394b1
            except Exception as e:
a394b1
                raise UploadError(e)
a394b1
a394b1
            finally:
a394b1
                c.close()
a394b1
a394b1
            output = buf.getvalue().strip()
a394b1
a394b1
        # Get back a new line, after displaying the download progress
a394b1
        sys.stdout.write('\n')
a394b1
        sys.stdout.flush()
a394b1
a394b1
        if status != 200:
a394b1
            self.raise_upload_error(status)
a394b1
a394b1
        if output:
a394b1
            self.log.debug(output)