From 9ed390d08a9f2b66f4e352532fa526fc64e329d4 Mon Sep 17 00:00:00 2001 From: Marek Blaha Date: Tue, 28 Jul 2020 09:50:10 +0200 Subject: [PATCH 1/3] Remove unused loops attribute from DepSolveProgressCallBack --- dnf/cli/output.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dnf/cli/output.py b/dnf/cli/output.py index de188ffbd1..44d5f8b89f 100644 --- a/dnf/cli/output.py +++ b/dnf/cli/output.py @@ -1987,10 +1987,6 @@ def historyInfoCmdPkgsAltered(self, old, pats=[]): class DepSolveProgressCallBack(dnf.callback.Depsolve): """Provides text output callback functions for Dependency Solver callback.""" - def __init__(self): - """requires yum-cli log and errorlog functions as arguments""" - self.loops = 0 - def pkg_added(self, pkg, mode): """Print information about a package being added to the transaction set. @@ -2037,7 +2033,6 @@ def start(self): process. """ logger.debug(_('--> Starting dependency resolution')) - self.loops += 1 def end(self): """Output a message stating that dependency resolution has finished.""" From 0ee646f4965c597f2832ed3df9d9f0e6546dcc54 Mon Sep 17 00:00:00 2001 From: Marek Blaha Date: Wed, 21 Oct 2020 11:47:43 +0200 Subject: [PATCH 2/3] Remove unused parameter of _make_lists() --- dnf/cli/output.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dnf/cli/output.py b/dnf/cli/output.py index 44d5f8b89f..af8a968770 100644 --- a/dnf/cli/output.py +++ b/dnf/cli/output.py @@ -52,7 +52,8 @@ logger = logging.getLogger('dnf') -def _make_lists(transaction, goal): + +def _make_lists(transaction): b = dnf.util.Bunch({ 'downgraded': [], 'erased': [], @@ -1101,7 +1102,7 @@ def list_transaction(self, transaction, total_width=None): # in order to display module changes when RPM transaction is empty transaction = [] - list_bunch = _make_lists(transaction, self.base._goal) + list_bunch = _make_lists(transaction) pkglist_lines = [] data = {'n' : {}, 'v' : {}, 'r' : {}} a_wid = 0 # Arch can't get "that big" ... so always use the max. @@ -1488,7 +1489,7 @@ def _tsi_or_pkg_nevra_cmp(item1, item2): return (item1.arch > item2.arch) - (item1.arch < item2.arch) out = '' - list_bunch = _make_lists(transaction, self.base._goal) + list_bunch = _make_lists(transaction) skipped_conflicts, skipped_broken = self._skipped_packages( report_problems=False, transaction=transaction) From 865b7183453684de6a25e77fddf5a2d11fbffba8 Mon Sep 17 00:00:00 2001 From: Marek Blaha Date: Wed, 21 Oct 2020 17:59:46 +0200 Subject: [PATCH 3/3] Post transaction summary is logged for API users (RhBug:1855158) Post transaction summary is always logged into /var/log/dnf.log. When transaction is called from cli, the summary is also printed to stdout in columns (as previously). = changelog = msg: Packages installed/removed via DNF API are logged into dnf.log type: enhancement resolves: https://bugzilla.redhat.com/show_bug.cgi?id=1855158 --- dnf/base.py | 46 ++++++++++++- dnf/cli/cli.py | 8 ++- dnf/cli/output.py | 167 ++++++++-------------------------------------- dnf/util.py | 102 +++++++++++++++++++++++++++- 4 files changed, 177 insertions(+), 146 deletions(-) diff --git a/dnf/base.py b/dnf/base.py index 075e74265a..c0d7712605 100644 --- a/dnf/base.py +++ b/dnf/base.py @@ -28,12 +28,12 @@ import dnf import libdnf.transaction +from copy import deepcopy from dnf.comps import CompsQuery from dnf.i18n import _, P_, ucd from dnf.util import _parse_specs from dnf.db.history import SwdbInterface from dnf.yum import misc -from functools import reduce try: from collections.abc import Sequence except ImportError: @@ -549,7 +549,7 @@ def _ts(self): if self.conf.ignorearch: self._rpm_probfilter.add(rpm.RPMPROB_FILTER_IGNOREARCH) - probfilter = reduce(operator.or_, self._rpm_probfilter, 0) + probfilter = functools.reduce(operator.or_, self._rpm_probfilter, 0) self._priv_ts.setProbFilter(probfilter) return self._priv_ts @@ -890,6 +890,15 @@ def do_transaction(self, display=()): self._plugins.unload_removed_plugins(self.transaction) self._plugins.run_transaction() + # log post transaction summary + def _pto_callback(action, tsis): + msgs = [] + for tsi in tsis: + msgs.append('{}: {}'.format(action, str(tsi))) + return msgs + for msg in dnf.util._post_transaction_output(self, self.transaction, _pto_callback): + logger.debug(msg) + return tid def _trans_error_summary(self, errstring): @@ -1311,7 +1320,7 @@ def _do_package_lists(self, pkgnarrow='all', patterns=None, showdups=None, if patterns is None or len(patterns) == 0: return list_fn(None) yghs = map(list_fn, patterns) - return reduce(lambda a, b: a.merge_lists(b), yghs) + return functools.reduce(lambda a, b: a.merge_lists(b), yghs) def _list_pattern(self, pkgnarrow, pattern, showdups, ignore_case, reponame=None): @@ -2579,6 +2588,37 @@ def setup_loggers(self): """ self._logging._setup_from_dnf_conf(self.conf, file_loggers_only=True) + def _skipped_packages(self, report_problems, transaction): + """returns set of conflicting packages and set of packages with broken dependency that would + be additionally installed when --best and --allowerasing""" + if self._goal.actions & (hawkey.INSTALL | hawkey.UPGRADE | hawkey.UPGRADE_ALL): + best = True + else: + best = False + ng = deepcopy(self._goal) + params = {"allow_uninstall": self._allow_erasing, + "force_best": best, + "ignore_weak": True} + ret = ng.run(**params) + if not ret and report_problems: + msg = dnf.util._format_resolve_problems(ng.problem_rules()) + logger.warning(msg) + problem_conflicts = set(ng.problem_conflicts(available=True)) + problem_dependency = set(ng.problem_broken_dependency(available=True)) - problem_conflicts + + def _nevra(item): + return hawkey.NEVRA(name=item.name, epoch=item.epoch, version=item.version, + release=item.release, arch=item.arch) + + # Sometimes, pkg is not in transaction item, therefore, comparing by nevra + transaction_nevras = [_nevra(tsi) for tsi in transaction] + skipped_conflicts = set( + [pkg for pkg in problem_conflicts if _nevra(pkg) not in transaction_nevras]) + skipped_dependency = set( + [pkg for pkg in problem_dependency if _nevra(pkg) not in transaction_nevras]) + + return skipped_conflicts, skipped_dependency + def _msg_installed(pkg): name = ucd(pkg) diff --git a/dnf/cli/cli.py b/dnf/cli/cli.py index 0bc2c119d0..334000362c 100644 --- a/dnf/cli/cli.py +++ b/dnf/cli/cli.py @@ -252,8 +252,12 @@ def do_transaction(self, display=()): trans = None if trans: - msg = self.output.post_transaction_output(trans) - logger.info(msg) + # the post transaction summary is already written to log during + # Base.do_transaction() so here only print the messages to the + # user arranged in columns + print() + print('\n'.join(self.output.post_transaction_output(trans))) + print() for tsi in trans: if tsi.state == libdnf.transaction.TransactionItemState_ERROR: raise dnf.exceptions.Error(_('Transaction failed')) diff --git a/dnf/cli/output.py b/dnf/cli/output.py index af8a968770..6d729b63ba 100644 --- a/dnf/cli/output.py +++ b/dnf/cli/output.py @@ -21,9 +21,7 @@ from __future__ import print_function from __future__ import unicode_literals -from copy import deepcopy import fnmatch -import functools import hawkey import itertools import libdnf.transaction @@ -53,51 +51,6 @@ logger = logging.getLogger('dnf') -def _make_lists(transaction): - b = dnf.util.Bunch({ - 'downgraded': [], - 'erased': [], - 'erased_clean': [], - 'erased_dep': [], - 'installed': [], - 'installed_group': [], - 'installed_dep': [], - 'installed_weak': [], - 'reinstalled': [], - 'upgraded': [], - 'failed': [], - }) - - for tsi in transaction: - if tsi.state == libdnf.transaction.TransactionItemState_ERROR: - b.failed.append(tsi) - elif tsi.action == libdnf.transaction.TransactionItemAction_DOWNGRADE: - b.downgraded.append(tsi) - elif tsi.action == libdnf.transaction.TransactionItemAction_INSTALL: - if tsi.reason == libdnf.transaction.TransactionItemReason_GROUP: - b.installed_group.append(tsi) - elif tsi.reason == libdnf.transaction.TransactionItemReason_DEPENDENCY: - b.installed_dep.append(tsi) - elif tsi.reason == libdnf.transaction.TransactionItemReason_WEAK_DEPENDENCY: - b.installed_weak.append(tsi) - else: - # TransactionItemReason_USER - b.installed.append(tsi) - elif tsi.action == libdnf.transaction.TransactionItemAction_REINSTALL: - b.reinstalled.append(tsi) - elif tsi.action == libdnf.transaction.TransactionItemAction_REMOVE: - if tsi.reason == libdnf.transaction.TransactionItemReason_CLEAN: - b.erased_clean.append(tsi) - elif tsi.reason == libdnf.transaction.TransactionItemReason_DEPENDENCY: - b.erased_dep.append(tsi) - else: - b.erased.append(tsi) - elif tsi.action == libdnf.transaction.TransactionItemAction_UPGRADE: - b.upgraded.append(tsi) - - return b - - def _spread_in_columns(cols_count, label, lst): left = itertools.chain((label,), itertools.repeat('')) lst_length = len(lst) @@ -1057,37 +1010,6 @@ def list_group_transaction(self, comps, history, diff): out[0:0] = self._banner(col_data, (_('Group'), _('Packages'), '', '')) return '\n'.join(out) - def _skipped_packages(self, report_problems, transaction): - """returns set of conflicting packages and set of packages with broken dependency that would - be additionally installed when --best and --allowerasing""" - if self.base._goal.actions & (hawkey.INSTALL | hawkey.UPGRADE | hawkey.UPGRADE_ALL): - best = True - else: - best = False - ng = deepcopy(self.base._goal) - params = {"allow_uninstall": self.base._allow_erasing, - "force_best": best, - "ignore_weak": True} - ret = ng.run(**params) - if not ret and report_problems: - msg = dnf.util._format_resolve_problems(ng.problem_rules()) - logger.warning(msg) - problem_conflicts = set(ng.problem_conflicts(available=True)) - problem_dependency = set(ng.problem_broken_dependency(available=True)) - problem_conflicts - - def _nevra(item): - return hawkey.NEVRA(name=item.name, epoch=item.epoch, version=item.version, - release=item.release, arch=item.arch) - - # Sometimes, pkg is not in transaction item, therefore, comparing by nevra - transaction_nevras = [_nevra(tsi) for tsi in transaction] - skipped_conflicts = set( - [pkg for pkg in problem_conflicts if _nevra(pkg) not in transaction_nevras]) - skipped_dependency = set( - [pkg for pkg in problem_dependency if _nevra(pkg) not in transaction_nevras]) - - return skipped_conflicts, skipped_dependency - def list_transaction(self, transaction, total_width=None): """Return a string representation of the transaction in an easy-to-read format. @@ -1102,7 +1024,7 @@ def list_transaction(self, transaction, total_width=None): # in order to display module changes when RPM transaction is empty transaction = [] - list_bunch = _make_lists(transaction) + list_bunch = dnf.util._make_lists(transaction) pkglist_lines = [] data = {'n' : {}, 'v' : {}, 'r' : {}} a_wid = 0 # Arch can't get "that big" ... so always use the max. @@ -1271,7 +1193,7 @@ def format_line(group): # show skipped conflicting packages if not self.conf.best and self.base._goal.actions & forward_actions: lines = [] - skipped_conflicts, skipped_broken = self._skipped_packages( + skipped_conflicts, skipped_broken = self.base._skipped_packages( report_problems=True, transaction=transaction) skipped_broken = dict((str(pkg), pkg) for pkg in skipped_broken) for pkg in sorted(skipped_conflicts): @@ -1436,13 +1358,8 @@ def format_line(group): max_msg_count, count, msg_pkgs)) return ''.join(out) - def post_transaction_output(self, transaction): - """Returns a human-readable summary of the results of the - transaction. - :return: a string containing a human-readable summary of the - results of the transaction - """ + def _pto_callback(self, action, tsis): # Works a bit like calcColumns, but we never overflow a column we just # have a dynamic number of columns. def _fits_in_cols(msgs, num): @@ -1472,61 +1389,33 @@ def _fits_in_cols(msgs, num): col_lens[col] *= -1 return col_lens - def _tsi_or_pkg_nevra_cmp(item1, item2): - """Compares two transaction items or packages by nevra. - Used as a fallback when tsi does not contain package object. - """ - ret = (item1.name > item2.name) - (item1.name < item2.name) - if ret != 0: - return ret - nevra1 = hawkey.NEVRA(name=item1.name, epoch=item1.epoch, version=item1.version, - release=item1.release, arch=item1.arch) - nevra2 = hawkey.NEVRA(name=item2.name, epoch=item2.epoch, version=item2.version, - release=item2.release, arch=item2.arch) - ret = nevra1.evr_cmp(nevra2, self.sack) - if ret != 0: - return ret - return (item1.arch > item2.arch) - (item1.arch < item2.arch) - - out = '' - list_bunch = _make_lists(transaction) - - skipped_conflicts, skipped_broken = self._skipped_packages( - report_problems=False, transaction=transaction) - skipped = skipped_conflicts.union(skipped_broken) - - for (action, tsis) in [(_('Upgraded'), list_bunch.upgraded), - (_('Downgraded'), list_bunch.downgraded), - (_('Installed'), list_bunch.installed + - list_bunch.installed_group + - list_bunch.installed_weak + - list_bunch.installed_dep), - (_('Reinstalled'), list_bunch.reinstalled), - (_('Skipped'), skipped), - (_('Removed'), list_bunch.erased + - list_bunch.erased_dep + - list_bunch.erased_clean), - (_('Failed'), list_bunch.failed)]: - if not tsis: - continue - msgs = [] - out += '\n%s:\n' % action - for tsi in sorted(tsis, key=functools.cmp_to_key(_tsi_or_pkg_nevra_cmp)): - msgs.append(str(tsi)) - for num in (8, 7, 6, 5, 4, 3, 2): - cols = _fits_in_cols(msgs, num) - if cols: - break - if not cols: - cols = [-(self.term.columns - 2)] - while msgs: - current_msgs = msgs[:len(cols)] - out += ' ' - out += self.fmtColumns(zip(current_msgs, cols), end=u'\n') - msgs = msgs[len(cols):] - + if not tsis: + return '' + out = [] + msgs = [] + out.append('{}:'.format(action)) + for tsi in tsis: + msgs.append(str(tsi)) + for num in (8, 7, 6, 5, 4, 3, 2): + cols = _fits_in_cols(msgs, num) + if cols: + break + if not cols: + cols = [-(self.term.columns - 2)] + while msgs: + current_msgs = msgs[:len(cols)] + out.append(' {}'.format(self.fmtColumns(zip(current_msgs, cols)))) + msgs = msgs[len(cols):] return out + + def post_transaction_output(self, transaction): + """ + Return a human-readable summary of the transaction. Packages in sections + are arranged to columns. + """ + return dnf.util._post_transaction_output(self.base, transaction, self._pto_callback) + def setup_progress_callbacks(self): """Set up the progress callbacks and various output bars based on debug level. diff --git a/dnf/util.py b/dnf/util.py index 8cf362706d..0beb04424d 100644 --- a/dnf/util.py +++ b/dnf/util.py @@ -24,13 +24,14 @@ from .pycomp import PY3, basestring from dnf.i18n import _, ucd -from functools import reduce import argparse import dnf import dnf.callback import dnf.const import dnf.pycomp import errno +import functools +import hawkey import itertools import locale import logging @@ -41,6 +42,7 @@ import tempfile import time import libdnf.repo +import libdnf.transaction logger = logging.getLogger('dnf') @@ -195,7 +197,7 @@ def group_by_filter(fn, iterable): def splitter(acc, item): acc[not bool(fn(item))].append(item) return acc - return reduce(splitter, iterable, ([], [])) + return functools.reduce(splitter, iterable, ([], [])) def insert_if(item, iterable, condition): """Insert an item into an iterable by a condition.""" @@ -504,3 +506,99 @@ def __setattr__(self, what, val): def setter(item): setattr(item, what, val) return list(map(setter, self)) + + +def _make_lists(transaction): + b = Bunch({ + 'downgraded': [], + 'erased': [], + 'erased_clean': [], + 'erased_dep': [], + 'installed': [], + 'installed_group': [], + 'installed_dep': [], + 'installed_weak': [], + 'reinstalled': [], + 'upgraded': [], + 'failed': [], + }) + + for tsi in transaction: + if tsi.state == libdnf.transaction.TransactionItemState_ERROR: + b.failed.append(tsi) + elif tsi.action == libdnf.transaction.TransactionItemAction_DOWNGRADE: + b.downgraded.append(tsi) + elif tsi.action == libdnf.transaction.TransactionItemAction_INSTALL: + if tsi.reason == libdnf.transaction.TransactionItemReason_GROUP: + b.installed_group.append(tsi) + elif tsi.reason == libdnf.transaction.TransactionItemReason_DEPENDENCY: + b.installed_dep.append(tsi) + elif tsi.reason == libdnf.transaction.TransactionItemReason_WEAK_DEPENDENCY: + b.installed_weak.append(tsi) + else: + # TransactionItemReason_USER + b.installed.append(tsi) + elif tsi.action == libdnf.transaction.TransactionItemAction_REINSTALL: + b.reinstalled.append(tsi) + elif tsi.action == libdnf.transaction.TransactionItemAction_REMOVE: + if tsi.reason == libdnf.transaction.TransactionItemReason_CLEAN: + b.erased_clean.append(tsi) + elif tsi.reason == libdnf.transaction.TransactionItemReason_DEPENDENCY: + b.erased_dep.append(tsi) + else: + b.erased.append(tsi) + elif tsi.action == libdnf.transaction.TransactionItemAction_UPGRADE: + b.upgraded.append(tsi) + + return b + + +def _post_transaction_output(base, transaction, action_callback): + """Returns a human-readable summary of the results of the + transaction. + + :param action_callback: function generating output for specific action. It + takes two parameters - action as a string and list of affected packages for + this action + :return: a list of lines containing a human-readable summary of the + results of the transaction + """ + def _tsi_or_pkg_nevra_cmp(item1, item2): + """Compares two transaction items or packages by nevra. + Used as a fallback when tsi does not contain package object. + """ + ret = (item1.name > item2.name) - (item1.name < item2.name) + if ret != 0: + return ret + nevra1 = hawkey.NEVRA(name=item1.name, epoch=item1.epoch, version=item1.version, + release=item1.release, arch=item1.arch) + nevra2 = hawkey.NEVRA(name=item2.name, epoch=item2.epoch, version=item2.version, + release=item2.release, arch=item2.arch) + ret = nevra1.evr_cmp(nevra2, base.sack) + if ret != 0: + return ret + return (item1.arch > item2.arch) - (item1.arch < item2.arch) + + list_bunch = dnf.util._make_lists(transaction) + + skipped_conflicts, skipped_broken = base._skipped_packages( + report_problems=False, transaction=transaction) + skipped = skipped_conflicts.union(skipped_broken) + + out = [] + for (action, tsis) in [(_('Upgraded'), list_bunch.upgraded), + (_('Downgraded'), list_bunch.downgraded), + (_('Installed'), list_bunch.installed + + list_bunch.installed_group + + list_bunch.installed_weak + + list_bunch.installed_dep), + (_('Reinstalled'), list_bunch.reinstalled), + (_('Skipped'), skipped), + (_('Removed'), list_bunch.erased + + list_bunch.erased_dep + + list_bunch.erased_clean), + (_('Failed'), list_bunch.failed)]: + out.extend(action_callback( + action, sorted(tsis, key=functools.cmp_to_key(_tsi_or_pkg_nevra_cmp)))) + + return out