|
|
a394b1 |
# Copyright (c) 2018 - Red Hat Inc.
|
|
|
a394b1 |
#
|
|
|
a394b1 |
# This program is free software; you can redistribute it and/or modify it
|
|
|
a394b1 |
# under the terms of the GNU General Public License as published by the
|
|
|
a394b1 |
# Free Software Foundation; either version 2 of the License, or (at your
|
|
|
a394b1 |
# option) any later version. See http://www.gnu.org/copyleft/gpl.html for
|
|
|
a394b1 |
# the full text of the license.
|
|
|
a394b1 |
|
|
|
a394b1 |
|
|
Brian Stinson |
3edf28 |
"""Interact with the Red Hat lookaside cache
|
|
|
a394b1 |
|
|
|
a394b1 |
We need to override the pyrpkg.lookasidecache module to handle our custom
|
|
|
a394b1 |
download path.
|
|
|
a394b1 |
"""
|
|
|
a394b1 |
|
|
|
a394b1 |
import io
|
|
|
a394b1 |
import os
|
|
|
a394b1 |
import pycurl
|
|
|
a394b1 |
import six
|
|
Brian Stinson |
3edf28 |
import sys
|
|
Brian Stinson |
3edf28 |
|
|
|
29dd93 |
from pyrpkg.errors import InvalidHashType, UploadError, LayoutError
|
|
|
a394b1 |
from pyrpkg.lookaside import CGILookasideCache
|
|
|
29dd93 |
from pyrpkg.layout.layouts import DistGitLayout
|
|
|
29dd93 |
|
|
|
29dd93 |
from . import utils
|
|
|
29dd93 |
|
|
|
29dd93 |
|
|
|
29dd93 |
def is_dist_git(folder):
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
Indicates if a folder is using a dist-git layout.
|
|
|
29dd93 |
|
|
|
29dd93 |
Parameters
|
|
|
29dd93 |
----------
|
|
|
29dd93 |
folder: str
|
|
|
29dd93 |
The directory to inspect.
|
|
|
29dd93 |
|
|
|
29dd93 |
Returns
|
|
|
29dd93 |
-------
|
|
|
29dd93 |
bool
|
|
|
29dd93 |
A bool flag indicating if `folder` is using
|
|
|
29dd93 |
a dist-git layout format.
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
result = False
|
|
|
29dd93 |
|
|
|
29dd93 |
try:
|
|
|
29dd93 |
DistGitLayout.from_path(folder)
|
|
|
29dd93 |
result = True
|
|
|
29dd93 |
except LayoutError:
|
|
|
29dd93 |
result = False
|
|
|
29dd93 |
finally:
|
|
|
29dd93 |
return result
|
|
|
a394b1 |
|
|
|
a394b1 |
|
|
Brian Stinson |
3edf28 |
class StreamLookasideCache(CGILookasideCache):
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
CentosStream lookaside specialized class.
|
|
|
29dd93 |
|
|
|
29dd93 |
It inherits most of its behavior from `pyrpkg.lookasideCGILookasideCache`.
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
|
|
|
29dd93 |
def __init__(self, hashtype, download_url, upload_url, client_cert=None, ca_cert=None):
|
|
Brian Stinson |
3edf28 |
super(StreamLookasideCache, self).__init__(
|
|
|
29dd93 |
hashtype, download_url, upload_url,
|
|
|
29dd93 |
client_cert=client_cert, ca_cert=ca_cert)
|
|
Brian Stinson |
3edf28 |
|
|
|
29dd93 |
def remote_file_exists(self, name, filename, hashstr):
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
Check if a remote file exists.
|
|
|
29dd93 |
|
|
|
29dd93 |
This method inherits the behavior of its parent class from pyrpkg.
|
|
|
29dd93 |
|
|
|
122263 |
It uses the internal `utils.get_repo_name` method to parse the name in case
|
|
|
29dd93 |
it is a scm url.
|
|
|
29dd93 |
|
|
|
29dd93 |
Parameters
|
|
|
29dd93 |
----------
|
|
|
29dd93 |
name: str
|
|
|
29dd93 |
The repository name and org.
|
|
|
29dd93 |
|
|
|
29dd93 |
filename: str
|
|
|
29dd93 |
The filename (something.tar.gz).
|
|
|
29dd93 |
|
|
|
29dd93 |
hash:
|
|
|
29dd93 |
The hash string for the file.
|
|
|
29dd93 |
|
|
|
29dd93 |
Returns
|
|
|
29dd93 |
-------
|
|
|
29dd93 |
bool
|
|
|
29dd93 |
A boolean value to inditicate if the file exists.
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
|
|
|
29dd93 |
|
|
|
29dd93 |
return super(StreamLookasideCache, self).remote_file_exists(
|
|
|
29dd93 |
_name, filename, hashstr)
|
|
|
29dd93 |
|
|
|
29dd93 |
def upload(self, name, filename, hashstr, offline=False):
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
Uploads a file to lookaside cache.
|
|
|
29dd93 |
|
|
|
29dd93 |
This method inherits the behavior of its parent class from pyrpkg.
|
|
|
29dd93 |
|
|
|
122263 |
It uses the internal `utils.get_repo_name` method to parse the name in case
|
|
|
29dd93 |
it is a scm url.
|
|
|
29dd93 |
|
|
|
29dd93 |
Parameters
|
|
|
29dd93 |
----------
|
|
|
29dd93 |
name: str
|
|
|
29dd93 |
The repository name and org.
|
|
|
29dd93 |
|
|
|
29dd93 |
filename: str
|
|
|
29dd93 |
The filename (something.tar.gz).
|
|
|
29dd93 |
|
|
|
29dd93 |
hash:
|
|
|
29dd93 |
The hash string for the file.
|
|
|
29dd93 |
|
|
|
29dd93 |
Raises
|
|
|
29dd93 |
------
|
|
|
29dd93 |
pyrpkg.errors.rpkgError
|
|
|
122263 |
Raises specialized classes that inherits from pyrpkg base errors.
|
|
|
29dd93 |
|
|
|
29dd93 |
Returns
|
|
|
29dd93 |
-------
|
|
|
29dd93 |
None
|
|
|
29dd93 |
Does not return anything
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
|
|
|
29dd93 |
|
|
|
29dd93 |
return super(StreamLookasideCache, self).upload(
|
|
|
29dd93 |
_name, filename, hashstr)
|
|
|
29dd93 |
|
|
|
29dd93 |
def download(self, name, filename, hashstr, outfile, hashtype=None, **kwargs):
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
Downloads a file from lookaside cache to the local filesystem.
|
|
|
29dd93 |
|
|
|
29dd93 |
This method inherits the behavior of its parent class from pyrpkg.
|
|
|
29dd93 |
|
|
|
122263 |
It uses the internal `utils.get_repo_name` method to parse the name in case
|
|
|
29dd93 |
it is a scm url.
|
|
|
29dd93 |
|
|
|
29dd93 |
Parameters
|
|
|
29dd93 |
----------
|
|
|
29dd93 |
name: str
|
|
|
29dd93 |
The repository name and org.
|
|
|
29dd93 |
|
|
|
29dd93 |
filename: str
|
|
|
29dd93 |
The filename (something.tar.gz).
|
|
|
29dd93 |
|
|
|
29dd93 |
hash: str
|
|
|
29dd93 |
The hash string for the file.
|
|
|
29dd93 |
|
|
|
29dd93 |
outfile: str
|
|
|
29dd93 |
|
|
|
29dd93 |
|
|
|
29dd93 |
Raises
|
|
|
29dd93 |
------
|
|
|
29dd93 |
pyrpkg.errors.rpkgError
|
|
|
29dd93 |
Raises specialized implementations of `yrpkg.errors.rpkgError`.
|
|
|
29dd93 |
|
|
|
29dd93 |
Returns
|
|
|
29dd93 |
-------
|
|
|
29dd93 |
None
|
|
|
29dd93 |
Does not return anything
|
|
|
29dd93 |
"""
|
|
|
29dd93 |
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
|
|
|
29dd93 |
|
|
|
29dd93 |
return super(StreamLookasideCache, self).download(
|
|
|
29dd93 |
_name, filename, hashstr, outfile, hashtype=hashtype, **kwargs)
|
|
Brian Stinson |
3edf28 |
|
|
Brian Stinson |
3edf28 |
|
|
Brian Stinson |
3edf28 |
class SIGLookasideCache(CGILookasideCache):
|
|
|
a394b1 |
def __init__(self, hashtype, download_url, upload_url, name, branch):
|
|
Brian Stinson |
3edf28 |
super(SIGLookasideCache, self).__init__(
|
|
|
a394b1 |
hashtype, download_url, upload_url, client_cert="/home/bstinson/.centos.cert")
|
|
|
a394b1 |
self.branch = branch
|
|
|
a394b1 |
|
|
|
a394b1 |
self.download_path = (
|
|
|
a394b1 |
'%(name)s/%(branch)s/%(hash)s')
|
|
|
a394b1 |
|
|
|
a394b1 |
def remote_file_exists(self, name, filename, hash):
|
|
|
a394b1 |
"""Verify whether a file exists on the lookaside cache
|
|
|
a394b1 |
|
|
|
a394b1 |
:param str name: The name of the module. (usually the name of the
|
|
|
a394b1 |
SRPM). This can include the namespace as well (depending on what
|
|
|
a394b1 |
the server side expects).
|
|
|
a394b1 |
:param str filename: The name of the file to check for.
|
|
|
a394b1 |
:param str hash: The known good hash of the file.
|
|
|
a394b1 |
"""
|
|
|
a394b1 |
|
|
|
a394b1 |
# RHEL 7 ships pycurl that does not accept unicode. When given unicode
|
|
|
a394b1 |
# type it would explode with "unsupported second type in tuple". Let's
|
|
|
a394b1 |
# convert to str just to be sure.
|
|
|
a394b1 |
# https://bugzilla.redhat.com/show_bug.cgi?id=1241059
|
|
|
29dd93 |
_name = utils.get_repo_name(name) if is_dist_git(os.getcwd()) else name
|
|
|
29dd93 |
|
|
|
a394b1 |
if six.PY2 and isinstance(filename, six.text_type):
|
|
|
a394b1 |
filename = filename.encode('utf-8')
|
|
|
a394b1 |
|
|
|
a394b1 |
if six.PY2 and isinstance(self.branch, six.text_type):
|
|
|
a394b1 |
self.branch = self.branch.encode('utf-8')
|
|
|
a394b1 |
|
|
|
29dd93 |
post_data = [('name', _name),
|
|
|
a394b1 |
('%ssum' % self.hashtype, hash),
|
|
|
a394b1 |
('branch', self.branch),
|
|
|
a394b1 |
('filename', filename)]
|
|
|
a394b1 |
|
|
|
a394b1 |
with io.BytesIO() as buf:
|
|
|
a394b1 |
c = pycurl.Curl()
|
|
|
a394b1 |
c.setopt(pycurl.URL, self.upload_url)
|
|
|
a394b1 |
c.setopt(pycurl.WRITEFUNCTION, buf.write)
|
|
|
a394b1 |
c.setopt(pycurl.HTTPPOST, post_data)
|
|
|
a394b1 |
|
|
|
a394b1 |
if self.client_cert is not None:
|
|
|
a394b1 |
if os.path.exists(self.client_cert):
|
|
|
a394b1 |
c.setopt(pycurl.SSLCERT, self.client_cert)
|
|
|
a394b1 |
else:
|
|
|
a394b1 |
self.log.warning("Missing certificate: %s"
|
|
|
a394b1 |
% self.client_cert)
|
|
|
a394b1 |
|
|
|
a394b1 |
if self.ca_cert is not None:
|
|
|
a394b1 |
if os.path.exists(self.ca_cert):
|
|
|
a394b1 |
c.setopt(pycurl.CAINFO, self.ca_cert)
|
|
|
a394b1 |
else:
|
|
|
a394b1 |
self.log.warning("Missing certificate: %s", self.ca_cert)
|
|
|
a394b1 |
|
|
|
a394b1 |
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
|
|
|
a394b1 |
c.setopt(pycurl.USERPWD, ':')
|
|
|
a394b1 |
|
|
|
a394b1 |
try:
|
|
|
a394b1 |
c.perform()
|
|
|
a394b1 |
status = c.getinfo(pycurl.RESPONSE_CODE)
|
|
|
a394b1 |
|
|
|
a394b1 |
except Exception as e:
|
|
|
a394b1 |
raise UploadError(e)
|
|
|
a394b1 |
|
|
|
a394b1 |
finally:
|
|
|
a394b1 |
c.close()
|
|
|
a394b1 |
|
|
|
a394b1 |
output = buf.getvalue().strip()
|
|
|
a394b1 |
|
|
|
a394b1 |
if status != 200:
|
|
|
a394b1 |
self.raise_upload_error(status)
|
|
|
a394b1 |
|
|
|
a394b1 |
# Lookaside CGI script returns these strings depending on whether
|
|
|
a394b1 |
# or not the file exists:
|
|
|
a394b1 |
if output == b'Available':
|
|
|
a394b1 |
return True
|
|
|
a394b1 |
|
|
|
a394b1 |
if output == b'Missing':
|
|
|
a394b1 |
return False
|
|
|
a394b1 |
|
|
|
a394b1 |
# Something unexpected happened
|
|
|
a394b1 |
self.log.debug(output)
|
|
|
a394b1 |
raise UploadError('Error checking for %s at %s'
|
|
|
a394b1 |
% (filename, self.upload_url))
|
|
|
a394b1 |
|
|
|
a394b1 |
def upload(self, name, filepath, hash):
|
|
|
a394b1 |
"""Upload a source file
|
|
|
a394b1 |
|
|
|
a394b1 |
:param str name: The name of the module. (usually the name of the SRPM)
|
|
|
a394b1 |
This can include the namespace as well (depending on what the
|
|
|
a394b1 |
server side expects).
|
|
|
a394b1 |
:param str filepath: The full path to the file to upload.
|
|
|
a394b1 |
:param str hash: The known good hash of the file.
|
|
|
a394b1 |
"""
|
|
|
a394b1 |
filename = os.path.basename(filepath)
|
|
|
a394b1 |
|
|
|
a394b1 |
# As in remote_file_exists, we need to convert unicode strings to str
|
|
|
a394b1 |
if six.PY2:
|
|
|
a394b1 |
if isinstance(name, six.text_type):
|
|
|
a394b1 |
name = name.encode('utf-8')
|
|
|
a394b1 |
if isinstance(filepath, six.text_type):
|
|
|
a394b1 |
filepath = filepath.encode('utf-8')
|
|
|
a394b1 |
|
|
|
a394b1 |
if self.remote_file_exists(name, filename, hash):
|
|
|
a394b1 |
self.log.info("File already uploaded: %s", filepath)
|
|
|
a394b1 |
return
|
|
|
a394b1 |
|
|
|
a394b1 |
self.log.info("Uploading: %s", filepath)
|
|
|
a394b1 |
post_data = [('name', name),
|
|
|
a394b1 |
('%ssum' % self.hashtype, hash),
|
|
|
a394b1 |
('branch', self.branch),
|
|
|
a394b1 |
('file', (pycurl.FORM_FILE, filepath))]
|
|
|
a394b1 |
|
|
|
a394b1 |
with io.BytesIO() as buf:
|
|
|
a394b1 |
c = pycurl.Curl()
|
|
|
a394b1 |
c.setopt(pycurl.URL, self.upload_url)
|
|
|
a394b1 |
c.setopt(pycurl.NOPROGRESS, False)
|
|
|
a394b1 |
c.setopt(pycurl.PROGRESSFUNCTION, self.print_progress)
|
|
|
a394b1 |
c.setopt(pycurl.WRITEFUNCTION, buf.write)
|
|
|
a394b1 |
c.setopt(pycurl.HTTPPOST, post_data)
|
|
|
a394b1 |
|
|
|
a394b1 |
if self.client_cert is not None:
|
|
|
a394b1 |
if os.path.exists(self.client_cert):
|
|
|
a394b1 |
c.setopt(pycurl.SSLCERT, self.client_cert)
|
|
|
a394b1 |
else:
|
|
|
a394b1 |
self.log.warning("Missing certificate: %s", self.client_cert)
|
|
|
a394b1 |
|
|
|
a394b1 |
if self.ca_cert is not None:
|
|
|
a394b1 |
if os.path.exists(self.ca_cert):
|
|
|
a394b1 |
c.setopt(pycurl.CAINFO, self.ca_cert)
|
|
|
a394b1 |
else:
|
|
|
a394b1 |
self.log.warning("Missing certificate: %s", self.ca_cert)
|
|
|
a394b1 |
|
|
|
a394b1 |
c.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_GSSNEGOTIATE)
|
|
|
a394b1 |
c.setopt(pycurl.USERPWD, ':')
|
|
|
a394b1 |
|
|
|
a394b1 |
try:
|
|
|
a394b1 |
c.perform()
|
|
|
a394b1 |
status = c.getinfo(pycurl.RESPONSE_CODE)
|
|
|
a394b1 |
|
|
|
a394b1 |
except Exception as e:
|
|
|
a394b1 |
raise UploadError(e)
|
|
|
a394b1 |
|
|
|
a394b1 |
finally:
|
|
|
a394b1 |
c.close()
|
|
|
a394b1 |
|
|
|
a394b1 |
output = buf.getvalue().strip()
|
|
|
a394b1 |
|
|
|
a394b1 |
# Get back a new line, after displaying the download progress
|
|
|
a394b1 |
sys.stdout.write('\n')
|
|
|
a394b1 |
sys.stdout.flush()
|
|
|
a394b1 |
|
|
|
a394b1 |
if status != 200:
|
|
|
a394b1 |
self.raise_upload_error(status)
|
|
|
a394b1 |
|
|
|
a394b1 |
if output:
|
|
|
a394b1 |
self.log.debug(output)
|