From bfd04883c44ad31407f0b5a48e06fa597c46d727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hr=C3=A1zk=C3=BD?= Date: Fri, 12 Jul 2019 17:14:00 +0200 Subject: [PATCH 1/6] repoquery: move the alldeps/exactdeps check to configure() Moves the check for alldeps/exactdeps switches to the configure() method along with the other checks. Raises dnf.cli.CliError instead of dnf.exceptions.Error. --- dnf/cli/commands/repoquery.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dnf/cli/commands/repoquery.py b/dnf/cli/commands/repoquery.py index 7334ddcd90..dc09bf4403 100644 --- a/dnf/cli/commands/repoquery.py +++ b/dnf/cli/commands/repoquery.py @@ -303,6 +303,12 @@ def configure(self): "(optionally with '--alldeps', but not with '--exactdeps'), or with " "'--requires --resolve'")) + if self.opts.alldeps or self.opts.exactdeps: + if not (self.opts.whatrequires or self.opts.whatdepends): + raise dnf.cli.CliError( + _("argument {} requires --whatrequires or --whatdepends option".format( + '--alldeps' if self.opts.alldeps else '--exactdeps'))) + if self.opts.srpm: self.base.repos.enable_source_repos() @@ -496,10 +502,6 @@ def run(self): else: q.filterm(file__glob=self.opts.whatprovides) if self.opts.alldeps or self.opts.exactdeps: - if not (self.opts.whatrequires or self.opts.whatdepends): - raise dnf.exceptions.Error( - _("argument {} requires --whatrequires or --whatdepends option".format( - '--alldeps' if self.opts.alldeps else '--exactdeps'))) if self.opts.alldeps: q = self.by_all_deps(self.opts.whatrequires, self.opts.whatdepends, q) else: From d9492fef24989085011047dd561fef1dfa1e31d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hr=C3=A1zk=C3=BD?= Date: Fri, 12 Jul 2019 17:22:05 +0200 Subject: [PATCH 2/6] repoquery: fix calling by_all_deps() for --tree Fixes a bug introduced in bd00c044, where the by_all_deps() arguments were changed to an array, but in tree_seed() it still passes a single string. Untested, I have actually no idea what the code does there, it looks kind of suspect. --- dnf/cli/commands/repoquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dnf/cli/commands/repoquery.py b/dnf/cli/commands/repoquery.py index dc09bf4403..db174b3bba 100644 --- a/dnf/cli/commands/repoquery.py +++ b/dnf/cli/commands/repoquery.py @@ -674,7 +674,7 @@ def tree_seed(self, query, aquery, opts, level=-1, usedpkgs=None): ar[querypkg.name + "." + querypkg.arch] = querypkg pkgquery = self.base.sack.query().filterm(pkg=list(ar.values())) else: - pkgquery = self.by_all_deps(pkg.name, None, aquery) if opts.alldeps \ + pkgquery = self.by_all_deps((pkg.name, ), None, aquery) if opts.alldeps \ else aquery.filter(requires__glob=pkg.name) self.tree_seed(pkgquery, aquery, opts, level + 1, usedpkgs) From f2f31452876b37b54150cba1bf08fc46ecaf72f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hr=C3=A1zk=C3=BD?= Date: Fri, 12 Jul 2019 17:44:38 +0200 Subject: [PATCH 3/6] repoquery: simplify handling of whatrequires and whatdepends Separates whatrequires and whatdepends handling into two separate code branches, the original was overly complex, confusing and presented multiple unnecessary corner cases. The by_all_deps() arguments now also have a much better defined role. --- dnf/cli/commands/repoquery.py | 59 +++++++++++++++++------------------ 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/dnf/cli/commands/repoquery.py b/dnf/cli/commands/repoquery.py index db174b3bba..f5e951c9c8 100644 --- a/dnf/cli/commands/repoquery.py +++ b/dnf/cli/commands/repoquery.py @@ -350,7 +350,7 @@ def build_format_fn(self, opts, pkg): raise dnf.exceptions.Error(str(e)) def _get_recursive_deps_query(self, query_in, query_select, done=None, recursive=False, - all_deps=False): + all_dep_types=False): done = done if done else self.base.sack.query().filterm(empty=True) t = self.base.sack.query().filterm(empty=True) set_requires = set() @@ -360,7 +360,7 @@ def _get_recursive_deps_query(self, query_in, query_select, done=None, recursive pkg_provides = pkg.provides set_requires.update(pkg_provides) set_requires.update(pkg.files) - if all_deps: + if all_dep_types: set_all_deps.update(pkg_provides) t = t.union(query_in.filter(requires=set_requires)) @@ -373,29 +373,29 @@ def _get_recursive_deps_query(self, query_in, query_select, done=None, recursive query_select = t.difference(done) if query_select: done = self._get_recursive_deps_query(query_in, query_select, done=t.union(done), - recursive=recursive, all_deps=all_deps) + recursive=recursive, + all_dep_types=all_dep_types) return t.union(done) - def by_all_deps(self, requires_name, depends_name, query): - names = requires_name or depends_name + def by_all_deps(self, names, query, all_dep_types=False): defaultquery = self.base.sack.query().filterm(empty=True) for name in names: defaultquery = defaultquery.union(query.intersection( dnf.subject.Subject(name).get_best_query(self.base.sack, with_provides=False, with_filenames=False))) requiresquery = query.filter(requires__glob=names) - if depends_name: - requiresquery = requiresquery.union(query.filter(recommends__glob=depends_name)) - requiresquery = requiresquery.union(query.filter(enhances__glob=depends_name)) - requiresquery = requiresquery.union(query.filter(supplements__glob=depends_name)) - requiresquery = requiresquery.union(query.filter(suggests__glob=depends_name)) + if all_dep_types: + requiresquery = requiresquery.union(query.filter(recommends__glob=names)) + requiresquery = requiresquery.union(query.filter(enhances__glob=names)) + requiresquery = requiresquery.union(query.filter(supplements__glob=names)) + requiresquery = requiresquery.union(query.filter(suggests__glob=names)) done = requiresquery.union(self._get_recursive_deps_query(query, defaultquery, - all_deps=depends_name)) + all_dep_types=all_dep_types)) if self.opts.recursive: done = done.union(self._get_recursive_deps_query(query, done, recursive=self.opts.recursive, - all_deps=depends_name)) + all_dep_types=all_dep_types)) return done def _get_recursive_providers_query(self, query_in, providers, done=None): @@ -501,24 +501,23 @@ def run(self): q = query_for_provide else: q.filterm(file__glob=self.opts.whatprovides) - if self.opts.alldeps or self.opts.exactdeps: - if self.opts.alldeps: - q = self.by_all_deps(self.opts.whatrequires, self.opts.whatdepends, q) + + if self.opts.whatrequires: + if (self.opts.exactdeps): + q.filterm(requires__glob=self.opts.whatrequires) else: - if self.opts.whatrequires: - q.filterm(requires__glob=self.opts.whatrequires) - else: - dependsquery = q.filter(requires__glob=self.opts.whatdepends) - dependsquery = dependsquery.union( - q.filter(recommends__glob=self.opts.whatdepends)) - dependsquery = dependsquery.union( - q.filter(enhances__glob=self.opts.whatdepends)) - dependsquery = dependsquery.union( - q.filter(supplements__glob=self.opts.whatdepends)) - q = dependsquery.union(q.filter(suggests__glob=self.opts.whatdepends)) - - elif self.opts.whatrequires or self.opts.whatdepends: - q = self.by_all_deps(self.opts.whatrequires, self.opts.whatdepends, q) + q = self.by_all_deps(self.opts.whatrequires, q) + + if self.opts.whatdepends: + if (self.opts.exactdeps): + dependsquery = q.filter(requires__glob=self.opts.whatdepends) + dependsquery = dependsquery.union(q.filter(recommends__glob=self.opts.whatdepends)) + dependsquery = dependsquery.union(q.filter(enhances__glob=self.opts.whatdepends)) + dependsquery = dependsquery.union(q.filter(supplements__glob=self.opts.whatdepends)) + q = dependsquery.union(q.filter(suggests__glob=self.opts.whatdepends)) + else: + q = self.by_all_deps(self.opts.whatdepends, q, True) + if self.opts.whatrecommends: q.filterm(recommends__glob=self.opts.whatrecommends) if self.opts.whatenhances: @@ -674,7 +673,7 @@ def tree_seed(self, query, aquery, opts, level=-1, usedpkgs=None): ar[querypkg.name + "." + querypkg.arch] = querypkg pkgquery = self.base.sack.query().filterm(pkg=list(ar.values())) else: - pkgquery = self.by_all_deps((pkg.name, ), None, aquery) if opts.alldeps \ + pkgquery = self.by_all_deps((pkg.name, ), aquery) if opts.alldeps \ else aquery.filter(requires__glob=pkg.name) self.tree_seed(pkgquery, aquery, opts, level + 1, usedpkgs) From a1bdbfa498438c2710aecd607a82face05a1bdfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hr=C3=A1zk=C3=BD?= Date: Tue, 9 Jul 2019 17:30:58 +0200 Subject: [PATCH 4/6] repoquery: rework "alldeps" dependency search (RhBug:1534123,1698034) Uses the new feature of filtering by solvables to properly resolve rich dependencies for packages. The new code now, along with serching for the arguments as reldeps, also searches for them as NEVRAs and then looks up the dependencies for the packages it found. https://bugzilla.redhat.com/show_bug.cgi?id=1534123 https://bugzilla.redhat.com/show_bug.cgi?id=1698034 --- dnf.spec | 2 +- dnf/cli/commands/repoquery.py | 93 +++++++++++++++++++---------------- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/dnf.spec b/dnf.spec index 79e4d27dab..578baccac4 100644 --- a/dnf.spec +++ b/dnf.spec @@ -1,5 +1,5 @@ # default dependencies -%global hawkey_version 0.41.0 +%global hawkey_version 0.44.0 %global libcomps_version 0.1.8 %global libmodulemd_version 1.4.0 %global rpm_version 4.14.0 diff --git a/dnf/cli/commands/repoquery.py b/dnf/cli/commands/repoquery.py index f5e951c9c8..d86d32ffcd 100644 --- a/dnf/cli/commands/repoquery.py +++ b/dnf/cli/commands/repoquery.py @@ -349,54 +349,61 @@ def build_format_fn(self, opts, pkg): # there don't exist on the dnf Package object. raise dnf.exceptions.Error(str(e)) - def _get_recursive_deps_query(self, query_in, query_select, done=None, recursive=False, - all_dep_types=False): - done = done if done else self.base.sack.query().filterm(empty=True) - t = self.base.sack.query().filterm(empty=True) - set_requires = set() - set_all_deps = set() - - for pkg in query_select.run(): - pkg_provides = pkg.provides - set_requires.update(pkg_provides) - set_requires.update(pkg.files) - if all_dep_types: - set_all_deps.update(pkg_provides) - - t = t.union(query_in.filter(requires=set_requires)) - if set_all_deps: - t = t.union(query_in.filter(recommends=set_all_deps)) - t = t.union(query_in.filter(enhances=set_all_deps)) - t = t.union(query_in.filter(supplements=set_all_deps)) - t = t.union(query_in.filter(suggests=set_all_deps)) - if recursive: - query_select = t.difference(done) - if query_select: - done = self._get_recursive_deps_query(query_in, query_select, done=t.union(done), - recursive=recursive, - all_dep_types=all_dep_types) - return t.union(done) + def _resolve_nevras(self, nevras, base_query): + resolved_nevras_query = self.base.sack.query().filterm(empty=True) + for nevra in nevras: + resolved_nevras_query = resolved_nevras_query.union(base_query.intersection( + dnf.subject.Subject(nevra).get_best_query( + self.base.sack, + with_provides=False, + with_filenames=False + ) + )) + + return resolved_nevras_query + + def _do_recursive_deps(self, query_in, query_select, done=None): + done = done if done else query_select + + query_required = query_in.filter(requires=query_select) + + query_select = query_required.difference(done) + done = query_required.union(done) + + if query_select: + done = self._do_recursive_deps(query_in, query_select, done=done) + + return done def by_all_deps(self, names, query, all_dep_types=False): - defaultquery = self.base.sack.query().filterm(empty=True) - for name in names: - defaultquery = defaultquery.union(query.intersection( - dnf.subject.Subject(name).get_best_query(self.base.sack, with_provides=False, - with_filenames=False))) - requiresquery = query.filter(requires__glob=names) + # in case of arguments being NEVRAs, resolve them to packages + resolved_nevras_query = self._resolve_nevras(names, query) + + # filter the arguments directly as reldeps + depquery = query.filter(requires__glob=names) + + # filter the resolved NEVRAs as packages + depquery = depquery.union(query.filter(requires=resolved_nevras_query)) + if all_dep_types: - requiresquery = requiresquery.union(query.filter(recommends__glob=names)) - requiresquery = requiresquery.union(query.filter(enhances__glob=names)) - requiresquery = requiresquery.union(query.filter(supplements__glob=names)) - requiresquery = requiresquery.union(query.filter(suggests__glob=names)) + # TODO this is very inefficient, as it resolves the `names` glob to + # reldeps four more times, which in a reasonably wide glob like + # `dnf repoquery --whatdepends "libdnf*"` can take roughly 50% of + # the total execution time. + depquery = depquery.union(query.filter(recommends__glob=names)) + depquery = depquery.union(query.filter(enhances__glob=names)) + depquery = depquery.union(query.filter(supplements__glob=names)) + depquery = depquery.union(query.filter(suggests__glob=names)) + + depquery = depquery.union(query.filter(recommends=resolved_nevras_query)) + depquery = depquery.union(query.filter(enhances=resolved_nevras_query)) + depquery = depquery.union(query.filter(supplements=resolved_nevras_query)) + depquery = depquery.union(query.filter(suggests=resolved_nevras_query)) - done = requiresquery.union(self._get_recursive_deps_query(query, defaultquery, - all_dep_types=all_dep_types)) if self.opts.recursive: - done = done.union(self._get_recursive_deps_query(query, done, - recursive=self.opts.recursive, - all_dep_types=all_dep_types)) - return done + depquery = self._do_recursive_deps(query, depquery) + + return depquery def _get_recursive_providers_query(self, query_in, providers, done=None): done = done if done else self.base.sack.query().filterm(empty=True) From 60cd14aae7fe3d9ae75d5e5665fc311ef3b04402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hr=C3=A1zk=C3=BD?= Date: Fri, 1 Nov 2019 18:33:13 +0100 Subject: [PATCH 5/6] repoquery: resolve NEVRAs to packages for most of --what* arguments (RhBug:1534123,1698034) To properly list the conflicts, suggests, recommends, enhances and supplements dependencies of packages defined through provides or even rich dependencies, it is required to resolve them to packages and query for them, because that's where the dependencies are defined. https://bugzilla.redhat.com/show_bug.cgi?id=1534123 https://bugzilla.redhat.com/show_bug.cgi?id=1698034 --- dnf/cli/commands/repoquery.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/dnf/cli/commands/repoquery.py b/dnf/cli/commands/repoquery.py index d86d32ffcd..d0521432be 100644 --- a/dnf/cli/commands/repoquery.py +++ b/dnf/cli/commands/repoquery.py @@ -499,7 +499,8 @@ def run(self): if self.opts.file: q.filterm(file__glob=self.opts.file) if self.opts.whatconflicts: - q.filterm(conflicts=self.opts.whatconflicts) + rels = q.filter(conflicts__glob=self.opts.whatconflicts) + q = rels.union(q.filter(conflicts=self._resolve_nevras(self.opts.whatconflicts, q))) if self.opts.whatobsoletes: q.filterm(obsoletes=self.opts.whatobsoletes) if self.opts.whatprovides: @@ -526,13 +527,18 @@ def run(self): q = self.by_all_deps(self.opts.whatdepends, q, True) if self.opts.whatrecommends: - q.filterm(recommends__glob=self.opts.whatrecommends) + rels = q.filter(recommends__glob=self.opts.whatrecommends) + q = rels.union(q.filter(recommends=self._resolve_nevras(self.opts.whatrecommends, q))) if self.opts.whatenhances: - q.filterm(enhances__glob=self.opts.whatenhances) + rels = q.filter(enhances__glob=self.opts.whatenhances) + q = rels.union(q.filter(enhances=self._resolve_nevras(self.opts.whatenhances, q))) if self.opts.whatsupplements: - q.filterm(supplements__glob=self.opts.whatsupplements) + rels = q.filter(supplements__glob=self.opts.whatsupplements) + q = rels.union(q.filter(supplements=self._resolve_nevras(self.opts.whatsupplements, q))) if self.opts.whatsuggests: - q.filterm(suggests__glob=self.opts.whatsuggests) + rels = q.filter(suggests__glob=self.opts.whatsuggests) + q = rels.union(q.filter(suggests=self._resolve_nevras(self.opts.whatsuggests, q))) + if self.opts.latest_limit: q = q.latest(self.opts.latest_limit) # reduce a query to security upgrades if they are specified From 757f9ad4f26c5a0fee9a6a3620eea94dd6004a73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Hr=C3=A1zk=C3=BD?= Date: Tue, 14 Jan 2020 16:44:07 +0100 Subject: [PATCH 6/6] [doc] Update documentation for the query filter API Add the new types of arguments supported for dependency filtering. --- doc/api_queries.rst | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/api_queries.rst b/doc/api_queries.rst index 5c8804e52a..98907d95a6 100644 --- a/doc/api_queries.rst +++ b/doc/api_queries.rst @@ -101,17 +101,20 @@ release string match against packages' releases reponame string match against packages repositories' names version string match against packages' versions - obsoletes Query match packages that obsolete any package from query pkg Query match against packages in query pkg* list match against hawkey.Packages in list provides string match against packages' provides provides* Hawkey.Reldep match against packages' provides - requires string match against packages' requirements - requires* Hawkey.Reldep match against packages' requirements + string match against packages' + * Hawkey.Reldep match a reldep against packages' + * Query match the result of a query against packages' + * list(Package) match the list of hawkey.Packages against packages' sourcerpm string match against packages' source rpm upgrades boolean see :meth:`upgrades`. Defaults to ``False``. =============== ============== ====================================================== + ```` can be any of: requires, conflicts, obsoletes, enhances, recomments, suggests, supplements + \* The key can also accept a list of values with specified type. The key name can be supplemented with a relation-specifying suffix, separated by ``__``: @@ -133,6 +136,8 @@ q = base.sack.query().filter(name__substr="club") + Note that using packages or queries for dependency filtering performs a more advanced resolution than using a string or a reldep. When a package list or a query is used, rich dependencies are resolved in a more precise way than what is possible when a string or a reldep is used. + .. method:: filterm(\*\*kwargs) Similar to :meth:`dnf.query.Query.filter` but it modifies the query in place.