From 6ed0458744090ab307da9d9118690372b2e66ca8 Mon Sep 17 00:00:00 2001 From: Jaroslav Mracek Date: Wed, 11 Nov 2020 12:47:21 +0100 Subject: [PATCH 1/5] Make module_base better industrialized for method reuse It will allow to use internal for module switch command. --- dnf/module/module_base.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/dnf/module/module_base.py b/dnf/module/module_base.py index 49c871c4..0da4fab1 100644 --- a/dnf/module/module_base.py +++ b/dnf/module/module_base.py @@ -323,7 +323,7 @@ class ModuleBase(object): assert len(streamDict) == 1 return moduleDict - def _resolve_specs_enable_update_sack(self, module_specs): + def _resolve_specs_enable(self, module_specs): no_match_specs = [] error_spec = [] module_dicts = {} @@ -339,6 +339,9 @@ class ModuleBase(object): error_spec.append(spec) logger.error(ucd(e)) logger.error(_("Unable to resolve argument {}").format(spec)) + return no_match_specs, error_spec, module_dicts + + def _update_sack(self): hot_fix_repos = [i.id for i in self.base.repos.iter_enabled() if i.module_hotfixes] try: solver_errors = self.base.sack.filter_modules( @@ -347,6 +350,10 @@ class ModuleBase(object): debugsolver=self.base.conf.debug_solver) except hawkey.Exception as e: raise dnf.exceptions.Error(ucd(e)) + return solver_errors + + def _enable_dependencies(self, module_dicts): + error_spec = [] for spec, (nsvcap, moduleDict) in module_dicts.items(): for streamDict in moduleDict.values(): for modules in streamDict.values(): @@ -357,6 +364,17 @@ class ModuleBase(object): error_spec.append(spec) logger.error(ucd(e)) logger.error(_("Unable to resolve argument {}").format(spec)) + return error_spec + + def _resolve_specs_enable_update_sack(self, module_specs): + no_match_specs, error_spec, module_dicts = self._resolve_specs_enable(module_specs) + + solver_errors = self._update_sack() + + dependency_error_spec = self._enable_dependencies(module_dicts) + if dependency_error_spec: + error_spec.extend(dependency_error_spec) + return no_match_specs, error_spec, solver_errors, module_dicts def _modules_reset_or_disable(self, module_specs, to_state): @@ -379,14 +397,7 @@ class ModuleBase(object): if to_state == STATE_DISABLED: self.base._moduleContainer.disable(name) - hot_fix_repos = [i.id for i in self.base.repos.iter_enabled() if i.module_hotfixes] - try: - solver_errors = self.base.sack.filter_modules( - self.base._moduleContainer, hot_fix_repos, self.base.conf.installroot, - self.base.conf.module_platform_id, update_only=True, - debugsolver=self.base.conf.debug_solver) - except hawkey.Exception as e: - raise dnf.exceptions.Error(ucd(e)) + solver_errors = self._update_sack() return no_match_specs, solver_errors def _get_package_name_set_and_remove_profiles(self, module_list, nsvcap, remove=False): -- 2.26.2 From e6473f4e6f17bb635e023b8905f29b318b8795bf Mon Sep 17 00:00:00 2001 From: Jaroslav Mracek Date: Wed, 11 Nov 2020 17:09:16 +0100 Subject: [PATCH 2/5] Add module switch-to support (RhBug:1792020) It is a combination of module rpm distrosync, module profile switch and module stream switch. = changelog = msg: Add new `module switch-to` command for switching content of module streams type: enhancement resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1792020 --- VERSION.cmake | 2 +- dnf.spec | 2 +- dnf/cli/commands/module.py | 24 ++++- dnf/module/module_base.py | 182 ++++++++++++++++++++++++++++++++----- 4 files changed, 185 insertions(+), 25 deletions(-) diff --git a/dnf/cli/commands/module.py b/dnf/cli/commands/module.py index 5a6c0069..4cdc915e 100644 --- a/dnf/cli/commands/module.py +++ b/dnf/cli/commands/module.py @@ -271,6 +271,28 @@ class ModuleCommand(commands.Command): logger.error(dnf.exceptions.MarkingErrors(no_match_group_specs=skipped_groups)) + class SwitchToSubCommand(SubCommand): + + aliases = ('switch-to',) + summary = _('switch a module to a stream and distrosync rpm packages') + + def configure(self): + demands = self.cli.demands + demands.available_repos = True + demands.sack_activation = True + demands.resolving = True + demands.root_user = True + self.base.conf.module_stream_switch = True + + def run_on_module(self): + try: + self.module_base.switch_to(self.opts.module_spec, strict=self.base.conf.strict) + except dnf.exceptions.MarkingErrors as e: + if self.base.conf.strict: + if e.no_match_group_specs or e.error_group_specs: + raise e + logger.error(str(e)) + class ProvidesSubCommand(SubCommand): aliases = ("provides", ) @@ -319,7 +341,7 @@ class ModuleCommand(commands.Command): SUBCMDS = {ListSubCommand, InfoSubCommand, EnableSubCommand, DisableSubCommand, ResetSubCommand, InstallSubCommand, UpdateSubCommand, - RemoveSubCommand, ProvidesSubCommand, RepoquerySubCommand} + RemoveSubCommand, SwitchToSubCommand, ProvidesSubCommand, RepoquerySubCommand} SUBCMDS_NOT_REQUIRED_ARG = {ListSubCommand} diff --git a/dnf/module/module_base.py b/dnf/module/module_base.py index 0da4fab1..03d54f72 100644 --- a/dnf/module/module_base.py +++ b/dnf/module/module_base.py @@ -140,31 +140,140 @@ class ModuleBase(object): if fail_safe_repo_used: raise dnf.exceptions.Error(_( "Installing module from Fail-Safe repository is not allowed")) - # Remove source packages they cannot be installed or upgraded - base_no_source_query = self.base.sack.query().filterm(arch__neq=['src', 'nosrc']).apply() - install_base_query = base_no_source_query.filter(nevra_strict=install_set_artefacts) + __, profiles_errors = self._install_profiles_internal( + install_set_artefacts, install_dict, strict) + if profiles_errors: + error_specs.extend(profiles_errors) - # add hot-fix packages - hot_fix_repos = [i.id for i in self.base.repos.iter_enabled() if i.module_hotfixes] - hotfix_packages = base_no_source_query.filter( - reponame=hot_fix_repos, name=install_dict.keys()) - install_base_query = install_base_query.union(hotfix_packages) + if no_match_specs or error_specs or solver_errors: + raise dnf.exceptions.MarkingErrors(no_match_group_specs=no_match_specs, + error_group_specs=error_specs, + module_depsolv_errors=solver_errors) - for pkg_name, set_specs in install_dict.items(): - query = install_base_query.filter(name=pkg_name) - if not query: - # package can also be non-modular or part of another stream - query = base_no_source_query.filter(name=pkg_name) - if not query: - for spec in set_specs: - logger.error(_("Unable to resolve argument {}").format(spec)) - logger.error(_("No match for package {}").format(pkg_name)) - error_specs.extend(set_specs) - continue - self.base._goal.group_members.add(pkg_name) + def switch_to(self, module_specs, strict=True): + # :api + no_match_specs, error_specs, module_dicts = self._resolve_specs_enable(module_specs) + # collect name of artifacts from new modules for distrosync + new_artifacts_names = set() + # collect name of artifacts from active modules for distrosync before sack update + active_artifacts_names = set() + src_arches = {"nosrc", "src"} + for spec, (nsvcap, moduledict) in module_dicts.items(): + for name in moduledict.keys(): + for module in self.base._moduleContainer.query(name, "", "", "", ""): + if self.base._moduleContainer.isModuleActive(module): + for artifact in module.getArtifacts(): + arch = artifact.rsplit(".", 1)[1] + if arch in src_arches: + continue + pkg_name = artifact.rsplit("-", 2)[0] + active_artifacts_names.add(pkg_name) + + solver_errors = self._update_sack() + + dependency_error_spec = self._enable_dependencies(module_dicts) + if dependency_error_spec: + error_specs.extend(dependency_error_spec) + + # + fail_safe_repo = hawkey.MODULE_FAIL_SAFE_REPO_NAME + install_dict = {} + install_set_artifacts = set() + fail_safe_repo_used = False + + # list of name: [profiles] for module profiles being removed + removed_profiles = self.base._moduleContainer.getRemovedProfiles() + + for spec, (nsvcap, moduledict) in module_dicts.items(): + for name, streamdict in moduledict.items(): + for stream, module_list in streamdict.items(): + install_module_list = [x for x in module_list + if self.base._moduleContainer.isModuleActive(x.getId())] + if not install_module_list: + "No active matches for argument '{0}' in module '{1}:{2}'" + logger.error(_("No active matches for argument '{0}' in module " + "'{1}:{2}'").format(spec, name, stream)) + error_specs.append(spec) + continue + profiles = [] + latest_module = self._get_latest(install_module_list) + if latest_module.getRepoID() == fail_safe_repo: + msg = _( + "Installing module '{0}' from Fail-Safe repository {1} is not allowed") + logger.critical(msg.format(latest_module.getNameStream(), fail_safe_repo)) + fail_safe_repo_used = True + if nsvcap.profile: + profiles.extend(latest_module.getProfiles(nsvcap.profile)) + if not profiles: + available_profiles = latest_module.getProfiles() + if available_profiles: + profile_names = ", ".join(sorted( + [profile.getName() for profile in available_profiles])) + msg = _("Unable to match profile for argument {}. Available " + "profiles for '{}:{}': {}").format( + spec, name, stream, profile_names) + else: + msg = _("Unable to match profile for argument {}").format(spec) + logger.error(msg) + no_match_specs.append(spec) + continue + elif name in removed_profiles: + + for profile in removed_profiles[name]: + module_profiles = latest_module.getProfiles(profile) + if not module_profiles: + logger.warning( + _("Installed profile '{0}' is not available in module " + "'{1}' stream '{2}'").format(profile, name, stream)) + continue + profiles.extend(module_profiles) + for profile in profiles: + self.base._moduleContainer.install(latest_module, profile.getName()) + for pkg_name in profile.getContent(): + install_dict.setdefault(pkg_name, set()).add(spec) + for module in install_module_list: + artifacts = module.getArtifacts() + install_set_artifacts.update(artifacts) + for artifact in artifacts: + arch = artifact.rsplit(".", 1)[1] + if arch in src_arches: + continue + pkg_name = artifact.rsplit("-", 2)[0] + new_artifacts_names.add(pkg_name) + if fail_safe_repo_used: + raise dnf.exceptions.Error(_( + "Installing module from Fail-Safe repository is not allowed")) + install_base_query, profiles_errors = self._install_profiles_internal( + install_set_artifacts, install_dict, strict) + if profiles_errors: + error_specs.extend(profiles_errors) + + # distrosync module name + all_names = set() + all_names.update(new_artifacts_names) + all_names.update(active_artifacts_names) + remove_query = self.base.sack.query().filterm(empty=True) + for pkg_name in all_names: + query = self.base.sack.query().filterm(name=pkg_name) + installed = query.installed() + if not installed: + continue + available = query.available() + if not available: + logger.warning(_("No packages available to distrosync for package name " + "'{}'").format(pkg_name)) + if pkg_name not in new_artifacts_names: + remove_query = remove_query.union(query) + continue + + only_new_module = query.intersection(install_base_query) + if only_new_module: + query = only_new_module sltr = dnf.selector.Selector(self.base.sack) sltr.set(pkg=query) - self.base._goal.install(select=sltr, optional=(not strict)) + self.base._goal.distupgrade(select=sltr) + self.base._remove_if_unneeded(remove_query) + if no_match_specs or error_specs or solver_errors: raise dnf.exceptions.MarkingErrors(no_match_group_specs=no_match_specs, error_group_specs=error_specs, @@ -183,7 +292,7 @@ class ModuleBase(object): fail_safe_repo = hawkey.MODULE_FAIL_SAFE_REPO_NAME fail_safe_repo_used = False - # Remove source packages they cannot be installed or upgraded + # Remove source packages because they cannot be installed or upgraded base_no_source_query = self.base.sack.query().filterm(arch__neq=['src', 'nosrc']).apply() for spec in module_specs: @@ -694,6 +803,35 @@ class ModuleBase(object): def _format_repoid(self, repo_name): return "{}\n".format(self.base.output.term.bold(repo_name)) + def _install_profiles_internal(self, install_set_artifacts, install_dict, strict): + # Remove source packages because they cannot be installed or upgraded + base_no_source_query = self.base.sack.query().filterm(arch__neq=['src', 'nosrc']).apply() + install_base_query = base_no_source_query.filter(nevra_strict=install_set_artifacts) + error_specs = [] + + # add hot-fix packages + hot_fix_repos = [i.id for i in self.base.repos.iter_enabled() if i.module_hotfixes] + hotfix_packages = base_no_source_query.filter( + reponame=hot_fix_repos, name=install_dict.keys()) + install_base_query = install_base_query.union(hotfix_packages) + + for pkg_name, set_specs in install_dict.items(): + query = install_base_query.filter(name=pkg_name) + if not query: + # package can also be non-modular or part of another stream + query = base_no_source_query.filter(name=pkg_name) + if not query: + for spec in set_specs: + logger.error(_("Unable to resolve argument {}").format(spec)) + logger.error(_("No match for package {}").format(pkg_name)) + error_specs.extend(set_specs) + continue + self.base._goal.group_members.add(pkg_name) + sltr = dnf.selector.Selector(self.base.sack) + sltr.set(pkg=query) + self.base._goal.install(select=sltr, optional=(not strict)) + return install_base_query, error_specs + def format_modular_solver_errors(errors): msg = dnf.util._format_resolve_problems(errors) -- 2.26.2 From df8c74679193bf27db584b3ad225997b2f5f4b87 Mon Sep 17 00:00:00 2001 From: Jaroslav Mracek Date: Thu, 12 Nov 2020 13:51:02 +0100 Subject: [PATCH 3/5] [minor] Rename all variables with artefact to artifact --- dnf/module/module_base.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dnf/module/module_base.py b/dnf/module/module_base.py index 03d54f72..7298c9a3 100644 --- a/dnf/module/module_base.py +++ b/dnf/module/module_base.py @@ -73,7 +73,7 @@ class ModuleBase(object): # fail_safe_repo = hawkey.MODULE_FAIL_SAFE_REPO_NAME install_dict = {} - install_set_artefacts = set() + install_set_artifacts = set() fail_safe_repo_used = False for spec, (nsvcap, moduledict) in module_dicts.items(): for name, streamdict in moduledict.items(): @@ -136,12 +136,12 @@ class ModuleBase(object): for pkg_name in profile.getContent(): install_dict.setdefault(pkg_name, set()).add(spec) for module in install_module_list: - install_set_artefacts.update(module.getArtifacts()) + install_set_artifacts.update(module.getArtifacts()) if fail_safe_repo_used: raise dnf.exceptions.Error(_( "Installing module from Fail-Safe repository is not allowed")) __, profiles_errors = self._install_profiles_internal( - install_set_artefacts, install_dict, strict) + install_set_artifacts, install_dict, strict) if profiles_errors: error_specs.extend(profiles_errors) @@ -326,8 +326,8 @@ class ModuleBase(object): else: for profile in latest_module.getProfiles(): upgrade_package_set.update(profile.getContent()) - for artefact in latest_module.getArtifacts(): - subj = hawkey.Subject(artefact) + for artifact in latest_module.getArtifacts(): + subj = hawkey.Subject(artifact) for nevra_obj in subj.get_nevra_possibilities( forms=[hawkey.FORM_NEVRA]): upgrade_package_set.add(nevra_obj.name) -- 2.26.2 From 0818bb80fc0846f602f338a2119671be97c47217 Mon Sep 17 00:00:00 2001 From: Jaroslav Mracek Date: Thu, 12 Nov 2020 15:11:29 +0100 Subject: [PATCH 4/5] [doc] Add description of dnf module switch-to --- doc/command_ref.rst | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/doc/command_ref.rst b/doc/command_ref.rst index 83879013..c12837ea 100644 --- a/doc/command_ref.rst +++ b/doc/command_ref.rst @@ -979,15 +979,31 @@ Module subcommands take :ref:`\\ `... arg In case no profile was provided, all default profiles get installed. Module streams get enabled accordingly. - This command cannot be used for switching module streams. It is recommended to remove all - installed content from the module and reset the module using the - :ref:`reset ` command. After you reset the module, you can install - the other stream. + This command cannot be used for switching module streams. Use the + :ref:`dnf module switch-to ` command for that. ``dnf [options] module update ...`` Update packages associated with an active module stream, optionally restricted to a profile. If the `profile_name` is provided, only the packages referenced by that profile will be updated. +.. _module_switch_to_command-label: + +``dnf [options] module switch-to ...`` + Switch to or enable a module stream, change versions of installed packages to versions provided + by the new stream, and remove packages from the old stream that are no longer available. It also + updates installed profiles if they are available for the new stream. When a profile was + provided, it installs that profile and does not update any already installed profiles. + + This command can be used as a stronger version of the + :ref:`dnf module enable ` command, which not only enables modules, + but also does a `distrosync` to all modular packages in the enabled modules. + + It can also be used as a stronger version of the + :ref:`dnf module install ` command, but it requires to specify + profiles that are supposed to be installed, because `switch-to` command does not use `default + profiles`. The `switch-to` command doesn't only install profiles, it also makes a `distrosync` + to all modular packages in the installed module. + ``dnf [options] module remove ...`` Remove installed module profiles, including packages that were installed with the :ref:`dnf module install ` command. Will not remove packages @@ -1010,10 +1026,8 @@ Module subcommands take :ref:`\\ `... arg of modular dependency issue the operation will be rejected. To perform the action anyway please use \-\ :ref:`-skip-broken ` option. - This command cannot be used for switching module streams. It is recommended to remove all - installed content from the module, and reset the module using the - :ref:`reset ` command. After you reset the module, you can enable - the other stream. + This command cannot be used for switching module streams. Use the + :ref:`dnf module switch-to ` command for that. .. _module_disable_command-label: -- 2.26.2 From 6b0b2b99e40c20706145e774626658825f5bc55d Mon Sep 17 00:00:00 2001 From: Jaroslav Mracek Date: Wed, 25 Nov 2020 12:34:30 +0100 Subject: [PATCH 5/5] Do not use source rpms for module switch It prevents misleading message from libsolv that it tries to install source rpm. --- dnf/module/module_base.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dnf/module/module_base.py b/dnf/module/module_base.py index 7298c9a3..02d5d5a3 100644 --- a/dnf/module/module_base.py +++ b/dnf/module/module_base.py @@ -253,8 +253,10 @@ class ModuleBase(object): all_names.update(new_artifacts_names) all_names.update(active_artifacts_names) remove_query = self.base.sack.query().filterm(empty=True) + base_no_source_query = self.base.sack.query().filterm(arch__neq=['src', 'nosrc']).apply() + for pkg_name in all_names: - query = self.base.sack.query().filterm(name=pkg_name) + query = base_no_source_query.filter(name=pkg_name) installed = query.installed() if not installed: continue -- 2.26.2