From 37d60b626fcb3e3f68b02c2c24e4ae5149cf223f Mon Sep 17 00:00:00 2001

From: Marek Blaha <mblaha@redhat.com>

Date: Thu, 23 Jul 2020 16:27:22 +0200

Subject: [PATCH] [reposync] Add latest NEVRAs per stream to download (RhBug:

 1833074)

This covers situation when package with the newest NEVRA is part of

an older version of a stream and reposync was used with --newest-only

switch.

With this patch these package versions are going to be downloaded:

- the latest NEVRAs from non-modular packages

- all packages from stream version with the latest package NEVRA (in

  case the latest NEVRA is part of multiple stream versions only the

  highest is downloaded)

- all packages from the latest stream version

https://bugzilla.redhat.com/show_bug.cgi?id=1833074

---

 plugins/reposync.py | 53 ++++++++++++++++++++++++++++++++++++---------

 1 file changed, 43 insertions(+), 10 deletions(-)

diff --git a/plugins/reposync.py b/plugins/reposync.py

index 548a05b4..7556e7eb 100644

--- a/plugins/reposync.py

+++ b/plugins/reposync.py

@@ -207,27 +207,60 @@ def download_metadata(self, repo):

     def _get_latest(self, query):

         """

-        return query with latest nonmodular package and all packages from latest version per stream

+        return union of these queries:

+        - the latest NEVRAs from non-modular packages

+        - all packages from stream version with the latest package NEVRA

+          (this should not be needed but the latest package NEVRAs might be

+          part of an older module version)

+        - all packages from the latest stream version

         """

         if not dnf.base.WITH_MODULES:

             return query.latest()

+

         query.apply()

         module_packages = self.base._moduleContainer.getModulePackages()

         all_artifacts = set()

         module_dict = {}  # {NameStream: {Version: [modules]}}

+        artifact_version = {} # {artifact: {NameStream: [Version]}}

         for module_package in module_packages:

-            all_artifacts.update(module_package.getArtifacts())

+            artifacts = module_package.getArtifacts()

+            all_artifacts.update(artifacts)

             module_dict.setdefault(module_package.getNameStream(), {}).setdefault(

                 module_package.getVersionNum(), []).append(module_package)

-        non_modular_latest = query.filter(

+            for artifact in artifacts:

+                artifact_version.setdefault(artifact, {}).setdefault(

+                    module_package.getNameStream(), []).append(module_package.getVersionNum())

+

+        # the latest NEVRAs from non-modular packages

+        latest_query = query.filter(

             pkg__neq=query.filter(nevra_strict=all_artifacts)).latest()

-        latest_artifacts = set()

-        for version_dict in module_dict.values():

-            keys = sorted(version_dict.keys(), reverse=True)

-            for module in version_dict[keys[0]]:

-                latest_artifacts.update(module.getArtifacts())

-        latest_modular_query = query.filter(nevra_strict=latest_artifacts)

-        return latest_modular_query.union(non_modular_latest)

+

+        # artifacts from the newest version and those versions that contain an artifact

+        # with the highest NEVRA

+        latest_stream_artifacts = set()

+        for namestream, version_dict in module_dict.items():

+            # versions that will be synchronized

+            versions = set()

+            # add the newest stream version

+            versions.add(sorted(version_dict.keys(), reverse=True)[0])

+            # collect all artifacts in all stream versions

+            stream_artifacts = set()

+            for modules in version_dict.values():

+                for module in modules:

+                    stream_artifacts.update(module.getArtifacts())

+            # find versions to which the packages with the highest NEVRAs belong

+            for latest_pkg in query.filter(nevra_strict=stream_artifacts).latest():

+                # here we depend on modules.yaml allways containing full NEVRA (including epoch)

+                nevra = "{0.name}-{0.epoch}:{0.version}-{0.release}.{0.arch}".format(latest_pkg)

+                # download only highest version containing the latest artifact

+                versions.add(max(artifact_version[nevra][namestream]))

+            # add all artifacts from selected versions for synchronization

+            for version in versions:

+                for module in version_dict[version]:

+                    latest_stream_artifacts.update(module.getArtifacts())

+        latest_query = latest_query.union(query.filter(nevra_strict=latest_stream_artifacts))

+

+        return latest_query

     def get_pkglist(self, repo):

         query = self.base.sack.query(flags=hawkey.IGNORE_MODULAR_EXCLUDES).available().filterm(