From f5ba2652d93a19fe798057befa1dafd07d85cf12 Mon Sep 17 00:00:00 2001 From: Leonardo Rossetti Date: Apr 14 2022 11:35:39 +0000 Subject: centos-sig lookaside cache support --- diff --git a/src/bin/centpkg-sig b/src/bin/centpkg-sig old mode 100644 new mode 100755 diff --git a/src/centpkg-sig.conf b/src/centpkg-sig.conf index 0e9ac51..e75eb01 100644 --- a/src/centpkg-sig.conf +++ b/src/centpkg-sig.conf @@ -1,7 +1,10 @@ [centpkg-sig] lookaside = https://git.centos.org/sources lookasidehash = sha512 -lookaside_cgi = https://git.centos.org/sources/upload.cgi +lookaside_cgi = https://git.centos.org/sources/upload_sig.cgi +# lookaside_cgi = https://git.centos.org/sources/upload.cgi +lookaside_structure = hash +# lookaside_structure = branch distgit_namespaced = True distgit_namespaces = rpms gitbaseurl = ssh://git@git.centos.org/%(repo)s.git diff --git a/src/centpkg/__init__.py b/src/centpkg/__init__.py index 8653930..8e7ba4e 100644 --- a/src/centpkg/__init__.py +++ b/src/centpkg/__init__.py @@ -151,6 +151,11 @@ class Commands(Commands): # should only be used when configured for SHA512 self.source_entry_type = 'bsd' if self.lookasidehash != 'md5' else 'old' self.branchre = 'c\d{1,}(s)?(tream)?|master' + self.lookaside_structure = None + + def update(self, config): + if self.lookaside_structure is None: + self.lookaside_structure = config.get('lookaside_structure', 'hash') @property def distgitdir(self): @@ -168,7 +173,8 @@ class Commands(Commands): self.lookaside, self.lookaside_cgi, self.repo_name, - self.branch_merge) + self.branch_merge, + structure=self.lookaside_structure) # redefined loaders def load_rpmdefines(self): diff --git a/src/centpkg/cli.py b/src/centpkg/cli.py index 61202fb..faebb0d 100755 --- a/src/centpkg/cli.py +++ b/src/centpkg/cli.py @@ -35,6 +35,11 @@ class centpkgClient(cliClient): self.setup_centos_subparsers() + def load_cmd(self): + super(centpkgClient, self).load_cmd() + cfg = self.config[self.get_name()] + self._cmd.update(cfg) + def setup_centos_subparsers(self): self.register_do_fork() self.register_request_gated_side_tag() diff --git a/src/centpkg/lookaside.py b/src/centpkg/lookaside.py index 04735e5..9565ddb 100644 --- a/src/centpkg/lookaside.py +++ b/src/centpkg/lookaside.py @@ -94,7 +94,7 @@ class StreamLookasideCache(CGILookasideCache): return super(StreamLookasideCache, self).remote_file_exists( _name, filename, hashstr) - def upload(self, name, filename, hashstr, offline=False): + def upload(self, name, filename, hashstr, offline=False, **kwargs): """ Uploads a file to lookaside cache. @@ -169,13 +169,32 @@ class StreamLookasideCache(CGILookasideCache): class SIGLookasideCache(CGILookasideCache): - def __init__(self, hashtype, download_url, upload_url, name, branch): + def __init__(self, hashtype, download_url, upload_url, name, branch, structure='hash'): super(SIGLookasideCache, self).__init__( hashtype, download_url, upload_url, client_cert="/home/bstinson/.centos.cert") + + self.name = name self.branch = branch - - self.download_path = ( - '%(name)s/%(branch)s/%(hash)s') + self.structure = structure + + @property + def download_path(self): + if self.structure == 'hash': + return '%(name)s/%(filename)s/%(hashtype)s/%(hash)s' + return '%(name)s/%(branch)s/%(hash)s' + + def get_download_url(self, name, filename, hash, hashtype=None, **kwargs): + if self.structure == 'hash': + path_dict = { + 'name': name, + 'filename': filename, + 'hash': hash, + 'hashtype': hashtype + } + path = self.download_path % path_dict + return os.path.join(self.download_url, path) + + return super(SIGLookasideCache, self).get_download_url(name, filename, hash, hashtype, **kwargs) def remote_file_exists(self, name, filename, hash): """Verify whether a file exists on the lookaside cache @@ -201,8 +220,9 @@ class SIGLookasideCache(CGILookasideCache): post_data = [('name', _name), ('%ssum' % self.hashtype, hash), - ('branch', self.branch), ('filename', filename)] + if self.structure == 'branch': + post_data.append(('branch', self.branch)) with io.BytesIO() as buf: c = pycurl.Curl() @@ -279,8 +299,9 @@ class SIGLookasideCache(CGILookasideCache): self.log.info("Uploading: %s", filepath) post_data = [('name', name), ('%ssum' % self.hashtype, hash), - ('branch', self.branch), ('file', (pycurl.FORM_FILE, filepath))] + if self.structure == 'branch': + post_data.append(('branch', self.branch)) with io.BytesIO() as buf: c = pycurl.Curl()