richardphibel / rpms / dnf

Forked from rpms/dnf 2 years ago
Clone
Blob Blame History Raw
From 6ed0458744090ab307da9d9118690372b2e66ca8 Mon Sep 17 00:00:00 2001
From: Jaroslav Mracek <jmracek@redhat.com>
Date: Wed, 11 Nov 2020 12:47:21 +0100
Subject: [PATCH 1/5] Make module_base better industrialized for method reuse

It will allow to use internal for module switch command.
---
 dnf/module/module_base.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/dnf/module/module_base.py b/dnf/module/module_base.py
index 49c871c4..0da4fab1 100644
--- a/dnf/module/module_base.py
+++ b/dnf/module/module_base.py
@@ -323,7 +323,7 @@ class ModuleBase(object):
             assert len(streamDict) == 1
         return moduleDict
 
-    def _resolve_specs_enable_update_sack(self, module_specs):
+    def _resolve_specs_enable(self, module_specs):
         no_match_specs = []
         error_spec = []
         module_dicts = {}
@@ -339,6 +339,9 @@ class ModuleBase(object):
                 error_spec.append(spec)
                 logger.error(ucd(e))
                 logger.error(_("Unable to resolve argument {}").format(spec))
+        return no_match_specs, error_spec, module_dicts
+
+    def _update_sack(self):
         hot_fix_repos = [i.id for i in self.base.repos.iter_enabled() if i.module_hotfixes]
         try:
             solver_errors = self.base.sack.filter_modules(
@@ -347,6 +350,10 @@ class ModuleBase(object):
                 debugsolver=self.base.conf.debug_solver)
         except hawkey.Exception as e:
             raise dnf.exceptions.Error(ucd(e))
+        return solver_errors
+
+    def _enable_dependencies(self, module_dicts):
+        error_spec = []
         for spec, (nsvcap, moduleDict) in module_dicts.items():
             for streamDict in moduleDict.values():
                 for modules in streamDict.values():
@@ -357,6 +364,17 @@ class ModuleBase(object):
                         error_spec.append(spec)
                         logger.error(ucd(e))
                         logger.error(_("Unable to resolve argument {}").format(spec))
+        return error_spec
+
+    def _resolve_specs_enable_update_sack(self, module_specs):
+        no_match_specs, error_spec, module_dicts = self._resolve_specs_enable(module_specs)
+
+        solver_errors = self._update_sack()
+
+        dependency_error_spec = self._enable_dependencies(module_dicts)
+        if dependency_error_spec:
+            error_spec.extend(dependency_error_spec)
+
         return no_match_specs, error_spec, solver_errors, module_dicts
 
     def _modules_reset_or_disable(self, module_specs, to_state):
@@ -379,14 +397,7 @@ class ModuleBase(object):
                 if to_state == STATE_DISABLED:
                     self.base._moduleContainer.disable(name)
 
-        hot_fix_repos = [i.id for i in self.base.repos.iter_enabled() if i.module_hotfixes]
-        try:
-            solver_errors = self.base.sack.filter_modules(
-                self.base._moduleContainer, hot_fix_repos, self.base.conf.installroot,
-                self.base.conf.module_platform_id, update_only=True,
-                debugsolver=self.base.conf.debug_solver)
-        except hawkey.Exception as e:
-            raise dnf.exceptions.Error(ucd(e))
+        solver_errors = self._update_sack()
         return no_match_specs, solver_errors
 
     def _get_package_name_set_and_remove_profiles(self, module_list, nsvcap, remove=False):
-- 
2.26.2


From e6473f4e6f17bb635e023b8905f29b318b8795bf Mon Sep 17 00:00:00 2001
From: Jaroslav Mracek <jmracek@redhat.com>
Date: Wed, 11 Nov 2020 17:09:16 +0100
Subject: [PATCH 2/5] Add module switch-to support (RhBug:1792020)

It is a combination of module rpm distrosync, module profile switch and
module stream switch.

= changelog =
msg:           Add new `module switch-to` command for switching content
of module streams
type:          enhancement
resolves:      https://bugzilla.redhat.com/show_bug.cgi?id=1792020
---
 VERSION.cmake              |   2 +-
 dnf.spec                   |   2 +-
 dnf/cli/commands/module.py |  24 ++++-
 dnf/module/module_base.py  | 182 ++++++++++++++++++++++++++++++++-----
 4 files changed, 185 insertions(+), 25 deletions(-)

diff --git a/dnf/cli/commands/module.py b/dnf/cli/commands/module.py
index 5a6c0069..4cdc915e 100644
--- a/dnf/cli/commands/module.py
+++ b/dnf/cli/commands/module.py
@@ -271,6 +271,28 @@ class ModuleCommand(commands.Command):
 
             logger.error(dnf.exceptions.MarkingErrors(no_match_group_specs=skipped_groups))
 
+    class SwitchToSubCommand(SubCommand):
+
+        aliases = ('switch-to',)
+        summary = _('switch a module to a stream and distrosync rpm packages')
+
+        def configure(self):
+            demands = self.cli.demands
+            demands.available_repos = True
+            demands.sack_activation = True
+            demands.resolving = True
+            demands.root_user = True
+            self.base.conf.module_stream_switch = True
+
+        def run_on_module(self):
+            try:
+                self.module_base.switch_to(self.opts.module_spec, strict=self.base.conf.strict)
+            except dnf.exceptions.MarkingErrors as e:
+                if self.base.conf.strict:
+                    if e.no_match_group_specs or e.error_group_specs:
+                        raise e
+                logger.error(str(e))
+
     class ProvidesSubCommand(SubCommand):
 
         aliases = ("provides", )
@@ -319,7 +341,7 @@ class ModuleCommand(commands.Command):
 
     SUBCMDS = {ListSubCommand, InfoSubCommand, EnableSubCommand,
                DisableSubCommand, ResetSubCommand, InstallSubCommand, UpdateSubCommand,
-               RemoveSubCommand, ProvidesSubCommand, RepoquerySubCommand}
+               RemoveSubCommand, SwitchToSubCommand, ProvidesSubCommand, RepoquerySubCommand}
 
     SUBCMDS_NOT_REQUIRED_ARG = {ListSubCommand}
 
diff --git a/dnf/module/module_base.py b/dnf/module/module_base.py
index 0da4fab1..03d54f72 100644
--- a/dnf/module/module_base.py
+++ b/dnf/module/module_base.py
@@ -140,31 +140,140 @@ class ModuleBase(object):
         if fail_safe_repo_used:
             raise dnf.exceptions.Error(_(
                 "Installing module from Fail-Safe repository is not allowed"))
-        #  Remove source packages they cannot be installed or upgraded
-        base_no_source_query = self.base.sack.query().filterm(arch__neq=['src', 'nosrc']).apply()
-        install_base_query = base_no_source_query.filter(nevra_strict=install_set_artefacts)
+        __, profiles_errors = self._install_profiles_internal(
+            install_set_artefacts, install_dict, strict)
+        if profiles_errors:
+            error_specs.extend(profiles_errors)
 
-        # add hot-fix packages
-        hot_fix_repos = [i.id for i in self.base.repos.iter_enabled() if i.module_hotfixes]
-        hotfix_packages = base_no_source_query.filter(
-            reponame=hot_fix_repos, name=install_dict.keys())
-        install_base_query = install_base_query.union(hotfix_packages)
+        if no_match_specs or error_specs or solver_errors:
+            raise dnf.exceptions.MarkingErrors(no_match_group_specs=no_match_specs,
+                                               error_group_specs=error_specs,
+                                               module_depsolv_errors=solver_errors)
 
-        for pkg_name, set_specs in install_dict.items():
-            query = install_base_query.filter(name=pkg_name)
-            if not query:
-                # package can also be non-modular or part of another stream
-                query = base_no_source_query.filter(name=pkg_name)
-                if not query:
-                    for spec in set_specs:
-                        logger.error(_("Unable to resolve argument {}").format(spec))
-                    logger.error(_("No match for package {}").format(pkg_name))
-                    error_specs.extend(set_specs)
-                    continue
-            self.base._goal.group_members.add(pkg_name)
+    def switch_to(self, module_specs, strict=True):
+        # :api
+        no_match_specs, error_specs, module_dicts = self._resolve_specs_enable(module_specs)
+        # collect name of artifacts from new modules for distrosync
+        new_artifacts_names = set()
+        # collect name of artifacts from active modules for distrosync before sack update
+        active_artifacts_names = set()
+        src_arches = {"nosrc", "src"}
+        for spec, (nsvcap, moduledict) in module_dicts.items():
+            for name in moduledict.keys():
+                for module in self.base._moduleContainer.query(name, "", "", "", ""):
+                    if self.base._moduleContainer.isModuleActive(module):
+                        for artifact in module.getArtifacts():
+                            arch = artifact.rsplit(".", 1)[1]
+                            if arch in src_arches:
+                                continue
+                            pkg_name = artifact.rsplit("-", 2)[0]
+                            active_artifacts_names.add(pkg_name)
+
+        solver_errors = self._update_sack()
+
+        dependency_error_spec = self._enable_dependencies(module_dicts)
+        if dependency_error_spec:
+            error_specs.extend(dependency_error_spec)
+
+        # <package_name, set_of_spec>
+        fail_safe_repo = hawkey.MODULE_FAIL_SAFE_REPO_NAME
+        install_dict = {}
+        install_set_artifacts = set()
+        fail_safe_repo_used = False
+
+        # list of name: [profiles] for module profiles being removed
+        removed_profiles = self.base._moduleContainer.getRemovedProfiles()
+
+        for spec, (nsvcap, moduledict) in module_dicts.items():
+            for name, streamdict in moduledict.items():
+                for stream, module_list in streamdict.items():
+                    install_module_list = [x for x in module_list
+                                           if self.base._moduleContainer.isModuleActive(x.getId())]
+                    if not install_module_list:
+                        "No active matches for argument '{0}' in module '{1}:{2}'"
+                        logger.error(_("No active matches for argument '{0}' in module "
+                                       "'{1}:{2}'").format(spec, name, stream))
+                        error_specs.append(spec)
+                        continue
+                    profiles = []
+                    latest_module = self._get_latest(install_module_list)
+                    if latest_module.getRepoID() == fail_safe_repo:
+                        msg = _(
+                            "Installing module '{0}' from Fail-Safe repository {1} is not allowed")
+                        logger.critical(msg.format(latest_module.getNameStream(), fail_safe_repo))
+                        fail_safe_repo_used = True
+                    if nsvcap.profile:
+                        profiles.extend(latest_module.getProfiles(nsvcap.profile))
+                        if not profiles:
+                            available_profiles = latest_module.getProfiles()
+                            if available_profiles:
+                                profile_names = ", ".join(sorted(
+                                    [profile.getName() for profile in available_profiles]))
+                                msg = _("Unable to match profile for argument {}. Available "
+                                        "profiles for '{}:{}': {}").format(
+                                    spec, name, stream, profile_names)
+                            else:
+                                msg = _("Unable to match profile for argument {}").format(spec)
+                            logger.error(msg)
+                            no_match_specs.append(spec)
+                            continue
+                    elif name in removed_profiles:
+
+                        for profile in removed_profiles[name]:
+                            module_profiles = latest_module.getProfiles(profile)
+                            if not module_profiles:
+                                logger.warning(
+                                    _("Installed profile '{0}' is not available in module "
+                                      "'{1}' stream '{2}'").format(profile, name, stream))
+                                continue
+                            profiles.extend(module_profiles)
+                    for profile in profiles:
+                        self.base._moduleContainer.install(latest_module, profile.getName())
+                        for pkg_name in profile.getContent():
+                            install_dict.setdefault(pkg_name, set()).add(spec)
+                    for module in install_module_list:
+                        artifacts = module.getArtifacts()
+                        install_set_artifacts.update(artifacts)
+                        for artifact in artifacts:
+                            arch = artifact.rsplit(".", 1)[1]
+                            if arch in src_arches:
+                                continue
+                            pkg_name = artifact.rsplit("-", 2)[0]
+                            new_artifacts_names.add(pkg_name)
+        if fail_safe_repo_used:
+            raise dnf.exceptions.Error(_(
+                "Installing module from Fail-Safe repository is not allowed"))
+        install_base_query, profiles_errors = self._install_profiles_internal(
+            install_set_artifacts, install_dict, strict)
+        if profiles_errors:
+            error_specs.extend(profiles_errors)
+
+        # distrosync module name
+        all_names = set()
+        all_names.update(new_artifacts_names)
+        all_names.update(active_artifacts_names)
+        remove_query = self.base.sack.query().filterm(empty=True)
+        for pkg_name in all_names:
+            query = self.base.sack.query().filterm(name=pkg_name)
+            installed = query.installed()
+            if not installed:
+                continue
+            available = query.available()
+            if not available:
+                logger.warning(_("No packages available to distrosync for package name "
+                                 "'{}'").format(pkg_name))
+                if pkg_name not in new_artifacts_names:
+                    remove_query = remove_query.union(query)
+                continue
+
+            only_new_module = query.intersection(install_base_query)
+            if only_new_module:
+                query = only_new_module
             sltr = dnf.selector.Selector(self.base.sack)
             sltr.set(pkg=query)
-            self.base._goal.install(select=sltr, optional=(not strict))
+            self.base._goal.distupgrade(select=sltr)
+        self.base._remove_if_unneeded(remove_query)
+
         if no_match_specs or error_specs or solver_errors:
             raise dnf.exceptions.MarkingErrors(no_match_group_specs=no_match_specs,
                                                error_group_specs=error_specs,
@@ -183,7 +292,7 @@ class ModuleBase(object):
         fail_safe_repo = hawkey.MODULE_FAIL_SAFE_REPO_NAME
         fail_safe_repo_used = False
 
-        #  Remove source packages they cannot be installed or upgraded
+        #  Remove source packages because they cannot be installed or upgraded
         base_no_source_query = self.base.sack.query().filterm(arch__neq=['src', 'nosrc']).apply()
 
         for spec in module_specs:
@@ -694,6 +803,35 @@ class ModuleBase(object):
     def _format_repoid(self, repo_name):
         return "{}\n".format(self.base.output.term.bold(repo_name))
 
+    def _install_profiles_internal(self, install_set_artifacts, install_dict, strict):
+        #  Remove source packages because they cannot be installed or upgraded
+        base_no_source_query = self.base.sack.query().filterm(arch__neq=['src', 'nosrc']).apply()
+        install_base_query = base_no_source_query.filter(nevra_strict=install_set_artifacts)
+        error_specs = []
+
+        # add hot-fix packages
+        hot_fix_repos = [i.id for i in self.base.repos.iter_enabled() if i.module_hotfixes]
+        hotfix_packages = base_no_source_query.filter(
+            reponame=hot_fix_repos, name=install_dict.keys())
+        install_base_query = install_base_query.union(hotfix_packages)
+
+        for pkg_name, set_specs in install_dict.items():
+            query = install_base_query.filter(name=pkg_name)
+            if not query:
+                # package can also be non-modular or part of another stream
+                query = base_no_source_query.filter(name=pkg_name)
+                if not query:
+                    for spec in set_specs:
+                        logger.error(_("Unable to resolve argument {}").format(spec))
+                    logger.error(_("No match for package {}").format(pkg_name))
+                    error_specs.extend(set_specs)
+                    continue
+            self.base._goal.group_members.add(pkg_name)
+            sltr = dnf.selector.Selector(self.base.sack)
+            sltr.set(pkg=query)
+            self.base._goal.install(select=sltr, optional=(not strict))
+        return install_base_query, error_specs
+
 
 def format_modular_solver_errors(errors):
     msg = dnf.util._format_resolve_problems(errors)
-- 
2.26.2


From df8c74679193bf27db584b3ad225997b2f5f4b87 Mon Sep 17 00:00:00 2001
From: Jaroslav Mracek <jmracek@redhat.com>
Date: Thu, 12 Nov 2020 13:51:02 +0100
Subject: [PATCH 3/5] [minor] Rename all variables with artefact to artifact

---
 dnf/module/module_base.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/dnf/module/module_base.py b/dnf/module/module_base.py
index 03d54f72..7298c9a3 100644
--- a/dnf/module/module_base.py
+++ b/dnf/module/module_base.py
@@ -73,7 +73,7 @@ class ModuleBase(object):
         # <package_name, set_of_spec>
         fail_safe_repo = hawkey.MODULE_FAIL_SAFE_REPO_NAME
         install_dict = {}
-        install_set_artefacts = set()
+        install_set_artifacts = set()
         fail_safe_repo_used = False
         for spec, (nsvcap, moduledict) in module_dicts.items():
             for name, streamdict in moduledict.items():
@@ -136,12 +136,12 @@ class ModuleBase(object):
                         for pkg_name in profile.getContent():
                             install_dict.setdefault(pkg_name, set()).add(spec)
                     for module in install_module_list:
-                        install_set_artefacts.update(module.getArtifacts())
+                        install_set_artifacts.update(module.getArtifacts())
         if fail_safe_repo_used:
             raise dnf.exceptions.Error(_(
                 "Installing module from Fail-Safe repository is not allowed"))
         __, profiles_errors = self._install_profiles_internal(
-            install_set_artefacts, install_dict, strict)
+            install_set_artifacts, install_dict, strict)
         if profiles_errors:
             error_specs.extend(profiles_errors)
 
@@ -326,8 +326,8 @@ class ModuleBase(object):
                     else:
                         for profile in latest_module.getProfiles():
                             upgrade_package_set.update(profile.getContent())
-                        for artefact in latest_module.getArtifacts():
-                            subj = hawkey.Subject(artefact)
+                        for artifact in latest_module.getArtifacts():
+                            subj = hawkey.Subject(artifact)
                             for nevra_obj in subj.get_nevra_possibilities(
                                     forms=[hawkey.FORM_NEVRA]):
                                 upgrade_package_set.add(nevra_obj.name)
-- 
2.26.2


From 0818bb80fc0846f602f338a2119671be97c47217 Mon Sep 17 00:00:00 2001
From: Jaroslav Mracek <jmracek@redhat.com>
Date: Thu, 12 Nov 2020 15:11:29 +0100
Subject: [PATCH 4/5] [doc] Add description of dnf module switch-to

---
 doc/command_ref.rst | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

diff --git a/doc/command_ref.rst b/doc/command_ref.rst
index 83879013..c12837ea 100644
--- a/doc/command_ref.rst
+++ b/doc/command_ref.rst
@@ -979,15 +979,31 @@ Module subcommands take :ref:`\<module-spec>\ <specifying_modules-label>`... arg
     In case no profile was provided, all default profiles get installed.
     Module streams get enabled accordingly.
 
-    This command cannot be used for switching module streams. It is recommended to remove all
-    installed content from the module and reset the module using the
-    :ref:`reset <module_reset_command-label>` command. After you reset the module, you can install
-    the other stream.
+    This command cannot be used for switching module streams. Use the
+    :ref:`dnf module switch-to <module_switch_to_command-label>` command for that.
 
 ``dnf [options] module update <module-spec>...``
     Update packages associated with an active module stream, optionally restricted to a profile.
     If the `profile_name` is provided, only the packages referenced by that profile will be updated.
 
+.. _module_switch_to_command-label:
+
+``dnf [options] module switch-to <module-spec>...``
+    Switch to or enable a module stream, change versions of installed packages to versions provided
+    by the new stream, and remove packages from the old stream that are no longer available. It also
+    updates installed profiles if they are available for the new stream. When a profile was
+    provided, it installs that profile and does not update any already installed profiles.
+
+    This command can be used as a stronger version of the
+    :ref:`dnf module enable <module_enable_command-label>` command, which not only enables modules,
+    but also does a `distrosync` to all modular packages in the enabled modules.
+
+    It can also be used as a stronger version of the
+    :ref:`dnf module install <module_install_command-label>` command, but it requires to specify
+    profiles that are supposed to be installed, because `switch-to` command does not use `default
+    profiles`. The `switch-to` command doesn't only install profiles, it also makes a `distrosync`
+    to all modular packages in the installed module.
+
 ``dnf [options] module remove <module-spec>...``
     Remove installed module profiles, including packages that were installed with the
     :ref:`dnf module install <module_install_command-label>` command. Will not remove packages
@@ -1010,10 +1026,8 @@ Module subcommands take :ref:`\<module-spec>\ <specifying_modules-label>`... arg
     of modular dependency issue the operation will be rejected. To perform the action anyway please use
     \-\ :ref:`-skip-broken <skip-broken_option-label>` option.
 
-    This command cannot be used for switching module streams. It is recommended to remove all
-    installed content from the module, and reset the module using the
-    :ref:`reset <module_reset_command-label>` command. After you reset the module, you can enable
-    the other stream.
+    This command cannot be used for switching module streams. Use the
+    :ref:`dnf module switch-to <module_switch_to_command-label>` command for that.
 
 .. _module_disable_command-label:
 
-- 
2.26.2


From 6b0b2b99e40c20706145e774626658825f5bc55d Mon Sep 17 00:00:00 2001
From: Jaroslav Mracek <jmracek@redhat.com>
Date: Wed, 25 Nov 2020 12:34:30 +0100
Subject: [PATCH 5/5] Do not use source rpms for module switch

It prevents misleading message from libsolv that it tries to install
source rpm.
---
 dnf/module/module_base.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/dnf/module/module_base.py b/dnf/module/module_base.py
index 7298c9a3..02d5d5a3 100644
--- a/dnf/module/module_base.py
+++ b/dnf/module/module_base.py
@@ -253,8 +253,10 @@ class ModuleBase(object):
         all_names.update(new_artifacts_names)
         all_names.update(active_artifacts_names)
         remove_query = self.base.sack.query().filterm(empty=True)
+        base_no_source_query = self.base.sack.query().filterm(arch__neq=['src', 'nosrc']).apply()
+
         for pkg_name in all_names:
-            query = self.base.sack.query().filterm(name=pkg_name)
+            query = base_no_source_query.filter(name=pkg_name)
             installed = query.installed()
             if not installed:
                 continue
-- 
2.26.2