From 37d60b626fcb3e3f68b02c2c24e4ae5149cf223f Mon Sep 17 00:00:00 2001 From: Marek Blaha Date: Thu, 23 Jul 2020 16:27:22 +0200 Subject: [PATCH] [reposync] Add latest NEVRAs per stream to download (RhBug: 1833074) This covers situation when package with the newest NEVRA is part of an older version of a stream and reposync was used with --newest-only switch. With this patch these package versions are going to be downloaded: - the latest NEVRAs from non-modular packages - all packages from stream version with the latest package NEVRA (in case the latest NEVRA is part of multiple stream versions only the highest is downloaded) - all packages from the latest stream version https://bugzilla.redhat.com/show_bug.cgi?id=1833074 --- plugins/reposync.py | 53 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/plugins/reposync.py b/plugins/reposync.py index 548a05b4..7556e7eb 100644 --- a/plugins/reposync.py +++ b/plugins/reposync.py @@ -207,27 +207,60 @@ def download_metadata(self, repo): def _get_latest(self, query): """ - return query with latest nonmodular package and all packages from latest version per stream + return union of these queries: + - the latest NEVRAs from non-modular packages + - all packages from stream version with the latest package NEVRA + (this should not be needed but the latest package NEVRAs might be + part of an older module version) + - all packages from the latest stream version """ if not dnf.base.WITH_MODULES: return query.latest() + query.apply() module_packages = self.base._moduleContainer.getModulePackages() all_artifacts = set() module_dict = {} # {NameStream: {Version: [modules]}} + artifact_version = {} # {artifact: {NameStream: [Version]}} for module_package in module_packages: - all_artifacts.update(module_package.getArtifacts()) + artifacts = module_package.getArtifacts() + all_artifacts.update(artifacts) module_dict.setdefault(module_package.getNameStream(), {}).setdefault( module_package.getVersionNum(), []).append(module_package) - non_modular_latest = query.filter( + for artifact in artifacts: + artifact_version.setdefault(artifact, {}).setdefault( + module_package.getNameStream(), []).append(module_package.getVersionNum()) + + # the latest NEVRAs from non-modular packages + latest_query = query.filter( pkg__neq=query.filter(nevra_strict=all_artifacts)).latest() - latest_artifacts = set() - for version_dict in module_dict.values(): - keys = sorted(version_dict.keys(), reverse=True) - for module in version_dict[keys[0]]: - latest_artifacts.update(module.getArtifacts()) - latest_modular_query = query.filter(nevra_strict=latest_artifacts) - return latest_modular_query.union(non_modular_latest) + + # artifacts from the newest version and those versions that contain an artifact + # with the highest NEVRA + latest_stream_artifacts = set() + for namestream, version_dict in module_dict.items(): + # versions that will be synchronized + versions = set() + # add the newest stream version + versions.add(sorted(version_dict.keys(), reverse=True)[0]) + # collect all artifacts in all stream versions + stream_artifacts = set() + for modules in version_dict.values(): + for module in modules: + stream_artifacts.update(module.getArtifacts()) + # find versions to which the packages with the highest NEVRAs belong + for latest_pkg in query.filter(nevra_strict=stream_artifacts).latest(): + # here we depend on modules.yaml allways containing full NEVRA (including epoch) + nevra = "{0.name}-{0.epoch}:{0.version}-{0.release}.{0.arch}".format(latest_pkg) + # download only highest version containing the latest artifact + versions.add(max(artifact_version[nevra][namestream])) + # add all artifacts from selected versions for synchronization + for version in versions: + for module in version_dict[version]: + latest_stream_artifacts.update(module.getArtifacts()) + latest_query = latest_query.union(query.filter(nevra_strict=latest_stream_artifacts)) + + return latest_query def get_pkglist(self, repo): query = self.base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES).available().filterm(