From 6ea94d9c768eb45975f314e11ab9dd88284fa380 Mon Sep 17 00:00:00 2001

From: Jaroslav Mracek <jmracek@redhat.com>

Date: Mon, 27 Sep 2021 11:29:01 +0200

Subject: [PATCH] Add new command modulesync (RhBug:1868047)

It will download module metadata from all enabled repositories,

module artifacts and profiles of matching modules. Then it creates

a repository.

= changelog =

msg:           Add a new subpackage with modulesync command. The command

downloads packages from modules and/or creates a repository with modular

data.

type:          enhancement

resolves:      https://bugzilla.redhat.com/show_bug.cgi?id=1868047

---

 dnf-plugins-core.spec  |  20 ++++++++++++++++++++

 doc/CMakeLists.txt     |   1 +

 doc/conf.py            |   1 +

 doc/index.rst          |   1 +

 doc/modulesync.rst     | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

 plugins/CMakeLists.txt |   1 +

 plugins/modulesync.py  | 208 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++

 7 files changed, 335 insertions(+)

 create mode 100644 doc/modulesync.rst

 create mode 100644 plugins/modulesync.py

diff --git a/dnf-plugins-core.spec b/dnf-plugins-core.spec

index cef836f..afdbcbb 100644

--- a/dnf-plugins-core.spec

+++ b/dnf-plugins-core.spec

@@ -402,6 +402,19 @@ versions of those packages. This allows you to e.g. protect packages from being

 updated by newer versions.

 %endif

+%if %{with python3}

+%package -n python3-dnf-plugin-modulesync

+Summary:        Download module metadata and packages and create repository

+Requires:       python3-%{name} = %{version}-%{release}

+Requires:       createrepo_c >= 0.17.4

+Provides:       dnf-plugin-modulesync =  %{version}-%{release}

+Provides:       dnf-command(modulesync)

+

+%description -n python3-dnf-plugin-modulesync

+Download module metadata from all enabled repositories, module artifacts and profiles of matching modules and create

+repository.

+%endif

+

 %prep

 %autosetup

 %if %{with python2}

@@ -762,6 +775,13 @@ ln -sf %{_mandir}/man1/%{yum_utils_subpackage_name}.1.gz %{buildroot}%{_mandir}/

 %endif

 %endif

+%if %{with python3}

+%files -n python3-dnf-plugin-modulesync

+%{python3_sitelib}/dnf-plugins/modulesync.*

+%{python3_sitelib}/dnf-plugins/__pycache__/modulesync.*

+%{_mandir}/man8/dnf-modulesync.*

+%endif

+

 %changelog

 * Mon Apr 12 2021 Nicola Sella <nsella@redhat.com> - 4.0.21-1

 - Add missing command line option to documentation

diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt

index 3fb665d..ff84cf8 100644

--- a/doc/CMakeLists.txt

+++ b/doc/CMakeLists.txt

@@ -28,6 +28,7 @@ INSTALL(FILES ${CMAKE_CURRENT_BINARY_DIR}/dnf-builddep.8

     ${CMAKE_CURRENT_BINARY_DIR}/dnf-generate_completion_cache.8

     ${CMAKE_CURRENT_BINARY_DIR}/dnf-groups-manager.8

     ${CMAKE_CURRENT_BINARY_DIR}/dnf-leaves.8

+    ${CMAKE_CURRENT_BINARY_DIR}/dnf-modulesync.8

     ${CMAKE_CURRENT_BINARY_DIR}/dnf-needs-restarting.8

     ${CMAKE_CURRENT_BINARY_DIR}/dnf-repoclosure.8

     ${CMAKE_CURRENT_BINARY_DIR}/dnf-repodiff.8

diff --git a/doc/conf.py b/doc/conf.py

index 645185a..41d6936 100644

--- a/doc/conf.py

+++ b/doc/conf.py

@@ -254,6 +254,7 @@ man_pages = [

     ('groups-manager', 'dnf-groups-manager', u'DNF groups-manager Plugin', AUTHORS, 8),

     ('leaves', 'dnf-leaves', u'DNF leaves Plugin', AUTHORS, 8),

     ('local', 'dnf-local', u'DNF local Plugin', AUTHORS, 8),

+    ('modulesync', 'dnf-modulesync', u'DNF modulesync Plugin', AUTHORS, 8),

     ('needs_restarting', 'dnf-needs-restarting', u'DNF needs_restarting Plugin', AUTHORS, 8),

     ('repoclosure', 'dnf-repoclosure', u'DNF repoclosure Plugin', AUTHORS, 8),

     ('repodiff', 'dnf-repodiff', u'DNF repodiff Plugin', AUTHORS, 8),

diff --git a/doc/index.rst b/doc/index.rst

index 7213253..07f6052 100644

--- a/doc/index.rst

+++ b/doc/index.rst

@@ -37,6 +37,7 @@ This documents core plugins of DNF:

    leaves

    local

    migrate

+   modulesync

    needs_restarting

    post-transaction-actions

    repoclosure

diff --git a/doc/modulesync.rst b/doc/modulesync.rst

new file mode 100644

index 0000000..2837287

--- /dev/null

+++ b/doc/modulesync.rst

@@ -0,0 +1,103 @@

+..

+  Copyright (C) 2015  Red Hat, Inc.

+

+  This copyrighted material is made available to anyone wishing to use,

+  modify, copy, or redistribute it subject to the terms and conditions of

+  the GNU General Public License v.2, or (at your option) any later version.

+  This program is distributed in the hope that it will be useful, but WITHOUT

+  ANY WARRANTY expressed or implied, including the implied warranties of

+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General

+  Public License for more details.  You should have received a copy of the

+  GNU General Public License along with this program; if not, write to the

+  Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

+  02110-1301, USA.  Any Red Hat trademarks that are incorporated in the

+  source code or documentation are not subject to the GNU General Public

+  License and may only be used or replicated with the express permission of

+  Red Hat, Inc.

+

+====================

+DNF modulesync Plugin

+====================

+

+Download packages from modules and/or create a repository with modular data.

+

+--------

+Synopsis

+--------

+

+``dnf modulesync [options] [<module-spec>...]``

+

+-----------

+Description

+-----------

+

+`modulesync` downloads packages from modules according to provided arguments and creates a repository with modular data

+in working directory. In environment with modules it is recommend to use the command for redistribution of packages,

+because DNF does not allow installation of modular packages without modular metadata on the system (Fail-safe

+mechanism). The command without an argument creates a repository like `createrepo_c` but with modular metadata collected

+from all available repositories.

+

+See examples.

+

+---------

+Arguments

+---------

+

+``<module-spec>``

+    Module specification for the package to download. The argument is an optional.

+

+-------

+Options

+-------

+

+All general DNF options are accepted. Namely, the ``--destdir`` option can be used to specify directory where packages

+will be downloaded and the new repository created. See `Options` in :manpage:`dnf(8)` for details.

+

+

+``-n, --newest-only``

+    Download only packages from the newest modules.

+

+``--enable_source_repos``

+    Enable repositories with source packages

+

+``--enable_debug_repos``

+    Enable repositories with debug-info and debug-source packages

+

+``--resolve``

+    Resolve and download needed dependencies

+

+--------

+Examples

+--------

+

+``dnf modulesync nodejs``

+    Download packages from `nodejs` module and crete a repository with modular metadata in working directory

+

+``dnf download nodejs``

+

+``dnf modulesync``

+    The first `download` command downloads nodejs package into working directory. In environment with modules `nodejs`

+    package can be a modular package therefore when I create a repository I have to insert also modular metadata

+    from available repositories to ensure 100% functionality. Instead of `createrepo_c` use `dnf modulesync`

+    to create a repository in working directory with nodejs package and modular metadata.

+

+``dnf --destdir=/tmp/my-temp modulesync nodejs:14/minimal --resolve``

+    Download package required for installation of `minimal` profile from module `nodejs` and stream `14` into directory

+    `/tmp/my-temp` and all required dependencies. Then it will create a repository in `/tmp/my-temp` directory with

+    previously downloaded packages and modular metadata from all available repositories.

+

+``dnf module install nodejs:14/minimal --downloadonly --destdir=/tmp/my-temp``

+

+``dnf modulesync --destdir=/tmp/my-temp``

+    The first `dnf module install` command downloads package from required for installation of `minimal` profile from module

+    `nodejs` and stream `14` into directory `/tmp/my-temp`. The second command `dnf modulesync` will create

+    a repository in `/tmp/my-temp` directory with previously downloaded packages and modular metadata from all

+    available repositories. In comparison to `dnf --destdir=/tmp/my-temp modulesync nodejs:14/minimal --resolve` it will

+    only download packages required for installation on current system.

+

+

+--------

+See Also

+--------

+

+* :manpage:`dnf(8)`, DNF Command Reference

diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt

index f66d3df..59f148f 100644

--- a/plugins/CMakeLists.txt

+++ b/plugins/CMakeLists.txt

@@ -22,6 +22,7 @@ INSTALL (FILES repograph.py DESTINATION ${PYTHON_INSTALL_DIR}/dnf-plugins)

 INSTALL (FILES repomanage.py DESTINATION ${PYTHON_INSTALL_DIR}/dnf-plugins)

 INSTALL (FILES reposync.py DESTINATION ${PYTHON_INSTALL_DIR}/dnf-plugins)

 INSTALL (FILES show_leaves.py DESTINATION ${PYTHON_INSTALL_DIR}/dnf-plugins)

+INSTALL (FILES modulesync.py DESTINATION ${PYTHON_INSTALL_DIR}/dnf-plugins)

 INSTALL (FILES versionlock.py DESTINATION ${PYTHON_INSTALL_DIR}/dnf-plugins)

 ADD_SUBDIRECTORY (dnfpluginscore)

diff --git a/plugins/modulesync.py b/plugins/modulesync.py

new file mode 100644

index 0000000..c1c33e4

--- /dev/null

+++ b/plugins/modulesync.py

@@ -0,0 +1,208 @@

+# Copyright (C) 2021  Red Hat, Inc.

+#

+# This copyrighted material is made available to anyone wishing to use,

+# modify, copy, or redistribute it subject to the terms and conditions of

+# the GNU General Public License v.2, or (at your option) any later version.

+# This program is distributed in the hope that it will be useful, but WITHOUT

+# ANY WARRANTY expressed or implied, including the implied warranties of

+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General

+# Public License for more details.  You should have received a copy of the

+# GNU General Public License along with this program; if not, write to the

+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA

+# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the

+# source code or documentation are not subject to the GNU General Public

+# License and may only be used or replicated with the express permission of

+# Red Hat, Inc.

+#

+

+from __future__ import absolute_import

+from __future__ import unicode_literals

+from dnfpluginscore import _, P_, logger

+from dnf.cli.option_parser import OptionParser

+

+import os

+import shutil

+import subprocess

+

+import dnf

+import dnf.cli

+import dnf.i18n

+import hawkey

+

+

+@dnf.plugin.register_command

+class SyncToolCommand(dnf.cli.Command):

+

+    aliases = ['modulesync']

+    summary = _('Download packages from modules and/or create a repository with modular data')

+

+    def __init__(self, cli):

+        super(SyncToolCommand, self).__init__(cli)

+

+    @staticmethod

+    def set_argparser(parser):

+        parser.add_argument('module', nargs='*', metavar=_('MODULE'),

+                            help=_('modules to download'))

+        parser.add_argument("--enable_source_repos", action='store_true',

+                            help=_('enable repositories with source packages'))

+        parser.add_argument("--enable_debug_repos", action='store_true',

+                            help=_('enable repositories with debug-info and debug-source packages'))

+        parser.add_argument('--resolve', action='store_true',

+                            help=_('resolve and download needed dependencies'))

+        parser.add_argument('-n', '--newest-only', default=False, action='store_true',

+                            help=_('download only packages from newest modules'))

+

+    def configure(self):

+        # setup sack and populate it with enabled repos

+        demands = self.cli.demands

+        demands.sack_activation = True

+        demands.available_repos = True

+

+        demands.load_system_repo = False

+

+        if self.opts.enable_source_repos:

+            self.base.repos.enable_source_repos()

+

+        if self.opts.enable_debug_repos:

+            self.base.repos.enable_debug_repos()

+

+        if self.opts.destdir:

+            self.base.conf.destdir = self.opts.destdir

+        else:

+            self.base.conf.destdir = dnf.i18n.ucd(os.getcwd())

+

+    def run(self):

+        """Execute the util action here."""

+

+        pkgs = self.base.sack.query().filterm(empty=True)

+        no_matched_spec = []

+        for module_spec in self.opts.module:

+            try:

+                pkgs = pkgs.union(self._get_packages_from_modules(module_spec))

+            except dnf.exceptions.Error:

+                no_matched_spec.append(module_spec)

+        if no_matched_spec:

+            msg = P_("Unable to find a match for argument: '{}'", "Unable to find a match for arguments: '{}'",

+                     len(no_matched_spec)).format("' '".join(no_matched_spec))

+            raise dnf.exceptions.Error(msg)

+

+        if self.opts.resolve:

+            pkgs = pkgs.union(self._get_providers_of_requires(pkgs))

+

+        # download rpms

+        self._do_downloads(pkgs)

+

+        # Create a repository at destdir with modular data

+        remove_tmp_moduleyamls_files = []

+        for repo in self.base.repos.iter_enabled():

+            module_md_path = repo.get_metadata_path('modules')

+            if module_md_path:

+                filename = "".join([repo.id, "-", os.path.basename(module_md_path)])

+                dest_path = os.path.join(self.base.conf.destdir, filename)

+                shutil.copy(module_md_path, dest_path)

+                remove_tmp_moduleyamls_files.append(dest_path)

+        args = ["createrepo_c", "--update", "--unique-md-filenames", self.base.conf.destdir]

+        p = subprocess.run(args)

+        if p.returncode:

+            msg = _("Creation of repository failed with return code {}. All downloaded content was kept on the system")

+            msg = msg.format(p.returncode)

+            raise dnf.exceptions.Error(msg)

+        for file_path in remove_tmp_moduleyamls_files:

+            os.remove(file_path)

+

+    def _do_downloads(self, pkgs):

+        """

+        Perform the download for a list of packages

+        """

+        pkg_dict = {}

+        for pkg in pkgs:

+            pkg_dict.setdefault(str(pkg), []).append(pkg)

+

+        to_download = []

+

+        for pkg_list in pkg_dict.values():

+            pkg_list.sort(key=lambda x: (x.repo.priority, x.repo.cost))

+            to_download.append(pkg_list[0])

+        if to_download:

+            self.base.download_packages(to_download, self.base.output.progress)

+

+    def _get_packages_from_modules(self, module_spec):

+        """Gets packages from modules matching module spec

+        1. From module artifacts

+        2. From module profiles"""

+        result_query = self.base.sack.query().filterm(empty=True)

+        module_base = dnf.module.module_base.ModuleBase(self.base)

+        module_list, nsvcap = module_base.get_modules(module_spec)

+        if self.opts.newest_only:

+            module_list = self.base._moduleContainer.getLatestModules(module_list, False)

+        for module in module_list:

+            for artifact in module.getArtifacts():

+                query = self.base.sack.query(flags=hawkey.IGNORE_EXCLUDES).filterm(nevra_strict=artifact)

+                if query:

+                    result_query = result_query.union(query)

+                else:

+                    msg = _("No match for artifact '{0}' from module '{1}'").format(

+                        artifact, module.getFullIdentifier())

+                    logger.warning(msg)

+            if nsvcap.profile:

+                profiles_set = module.getProfiles(nsvcap.profile)

+            else:

+                profiles_set = module.getProfiles()

+            if profiles_set:

+                for profile in profiles_set:

+                    for pkg_name in profile.getContent():

+                        query = self.base.sack.query(flags=hawkey.IGNORE_EXCLUDES).filterm(name=pkg_name)

+                        # Prefer to add modular providers selected by argument

+                        if result_query.intersection(query):

+                            continue

+                        # Add all packages with the same name as profile described

+                        elif query:

+                            result_query = result_query.union(query)

+                        else:

+                            msg = _("No match for package name '{0}' in profile {1} from module {2}")\

+                                .format(pkg_name, profile.getName(), module.getFullIdentifier())

+                            logger.warning(msg)

+        if not module_list:

+            msg = _("No mach for argument '{}'").format(module_spec)

+            raise dnf.exceptions.Error(msg)

+

+        return result_query

+

+    def _get_providers_of_requires(self, to_test, done=None, req_dict=None):

+        done = done if done else to_test

+        # req_dict = {}  {req : set(pkgs)}

+        if req_dict is None:

+            req_dict = {}

+        test_requires = []

+        for pkg in to_test:

+            for require in pkg.requires:

+                if require not in req_dict:

+                    test_requires.append(require)

+                req_dict.setdefault(require, set()).add(pkg)

+

+        if self.opts.newest_only:

+            #  Prepare cache with all packages related affected by modular filtering

+            names = set()

+            for module in self.base._moduleContainer.getModulePackages():

+                for artifact in module.getArtifacts():

+                    name, __, __ = artifact.rsplit("-", 2)

+                    names.add(name)

+            modular_related = self.base.sack.query(flags=hawkey.IGNORE_EXCLUDES).filterm(provides=names)

+

+        requires = self.base.sack.query().filterm(empty=True)

+        for require in test_requires:

+            q = self.base.sack.query(flags=hawkey.IGNORE_EXCLUDES).filterm(provides=require)

+

+            if not q:

+                #  TODO(jmracek) Shell we end with an error or with RC 1?

+                logger.warning((_("Unable to satisfy require {}").format(require)))

+            else:

+                if self.opts.newest_only:

+                    if not modular_related.intersection(q):

+                        q.filterm(latest_per_arch_by_priority=1)

+                requires = requires.union(q.difference(done))

+        done = done.union(requires)

+        if requires:

+            done = self._get_providers_of_requires(requires, done=done, req_dict=req_dict)

+

+        return done

--

libgit2 1.1.0