From a777ff01c79d5e0e2cf3ae7b0652795577253bc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
Date: Thu, 14 Jan 2021 09:58:30 +0100
Subject: [PATCH 1/3] Fix recreate script
---
tests/repos/rpm/recreate | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/repos/rpm/recreate b/tests/repos/rpm/recreate
index da348d9799..0fbb9396bd 100755
--- a/tests/repos/rpm/recreate
+++ b/tests/repos/rpm/recreate
@@ -1,6 +1,6 @@
#!/bin/bash
-THISDIR="$( readlink -f "$( dirname "$0 )" )"
+THISDIR="$( readlink -f "$( dirname "$0" )" )"
cd "$THISDIR"
git rm -rf repodata/
createrepo --no-database -o . ..
From 5d4c0266f6967c7cd5f0e675b13fa3e9b395e4dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
Date: Thu, 14 Jan 2021 10:28:53 +0100
Subject: [PATCH 2/3] Add unit test for fill_sack_from_repos_in_cache
(RhBug:1865803)
https://bugzilla.redhat.com/show_bug.cgi?id=1865803
---
tests/test_fill_sack_from_repos_in_cache.py | 262 ++++++++++++++++++++
1 file changed, 262 insertions(+)
create mode 100644 tests/test_fill_sack_from_repos_in_cache.py
diff --git a/tests/test_fill_sack_from_repos_in_cache.py b/tests/test_fill_sack_from_repos_in_cache.py
new file mode 100644
index 0000000000..24b0d4598d
--- /dev/null
+++ b/tests/test_fill_sack_from_repos_in_cache.py
@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2021 Red Hat, Inc.
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# the GNU General Public License v.2, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY expressed or implied, including the implied warranties of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details. You should have received a copy of the
+# GNU General Public License along with this program; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the
+# source code or documentation are not subject to the GNU General Public
+# License and may only be used or replicated with the express permission of
+# Red Hat, Inc.
+#
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import os
+import tempfile
+import glob
+import shutil
+import unittest
+
+import dnf.exceptions
+import dnf.repo
+import dnf.sack
+
+import hawkey
+
+import tests.support
+from tests.support import mock
+
+TEST_REPO_NAME = "test-repo"
+
+
+class FillSackFromReposInCacheTest(unittest.TestCase):
+ def _create_cache_for_repo(self, repopath, tmpdir):
+ conf = dnf.conf.MainConf()
+ conf.cachedir = os.path.join(tmpdir, "cache")
+
+ base = dnf.Base(conf=conf)
+
+ repoconf = dnf.repo.Repo(TEST_REPO_NAME, base.conf)
+ repoconf.baseurl = repopath
+ repoconf.enable()
+
+ base.repos.add(repoconf)
+
+ base.fill_sack(load_system_repo=False)
+ base.close()
+
+ def _setUp_from_repo_path(self, original_repo_path):
+ self.tmpdir = tempfile.mkdtemp(prefix="dnf_test_")
+
+ self.repo_copy_path = os.path.join(self.tmpdir, "repo")
+ shutil.copytree(original_repo_path, self.repo_copy_path)
+
+ self._create_cache_for_repo(self.repo_copy_path, self.tmpdir)
+
+ # Just to be sure remove repo (it shouldn't be used)
+ shutil.rmtree(self.repo_copy_path)
+
+ # Prepare base for the actual test
+ conf = dnf.conf.MainConf()
+ conf.cachedir = os.path.join(self.tmpdir, "cache")
+ self.test_base = dnf.Base(conf=conf)
+ repoconf = dnf.repo.Repo(TEST_REPO_NAME, conf)
+ repoconf.baseurl = self.repo_copy_path
+ repoconf.enable()
+ self.test_base.repos.add(repoconf)
+
+ def tearDown(self):
+ self.test_base.close()
+ shutil.rmtree(self.tmpdir)
+
+ def test_with_solv_solvx_repomd(self):
+ self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+ # Remove xml metadata except repomd
+ # repomd.xml is not compressed and doesn't end with .gz
+ repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+ for f in repodata_without_repomd:
+ os.remove(f)
+
+ # Now we only have cache with just solv, solvx files and repomd.xml
+
+ self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+ q = self.test_base.sack.query()
+ packages = q.run()
+ self.assertEqual(len(packages), 9)
+ self.assertEqual(packages[0].evr, "4-4")
+
+ # Use *-updateinfo.solvx
+ adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT)
+ adv_titles = set()
+ for pkg in adv_pkgs:
+ adv_titles.add(pkg.get_advisory(self.test_base.sack).title)
+ self.assertEqual(len(adv_titles), 3)
+
+ def test_with_just_solv_repomd(self):
+ self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+ # Remove xml metadata except repomd
+ # repomd.xml is not compressed and doesn't end with .gz
+ repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+ for f in repodata_without_repomd:
+ os.remove(f)
+
+ # Remove solvx files
+ solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solvx"))
+ for f in solvx:
+ os.remove(f)
+
+ # Now we only have cache with just solv files and repomd.xml
+
+ self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+ q = self.test_base.sack.query()
+ packages = q.run()
+ self.assertEqual(len(packages), 9)
+ self.assertEqual(packages[0].evr, "4-4")
+
+ # No *-updateinfo.solvx -> we get no advisory packages
+ adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT)
+ self.assertEqual(len(adv_pkgs), 0)
+
+ def test_with_xml_metadata(self):
+ self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+ # Remove all solv and solvx files
+ solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*"))
+ for f in solvx:
+ os.remove(f)
+
+ # Now we only have cache with just xml metadata
+
+ self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+ q = self.test_base.sack.query()
+ packages = q.run()
+ self.assertEqual(len(packages), 9)
+ self.assertEqual(packages[0].evr, "4-4")
+
+ def test_exception_without_repomd(self):
+ self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+ # Remove xml metadata
+ repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*"))
+ for f in repodata_without_repomd:
+ os.remove(f)
+
+ # Now we only have cache with just solv and solvx files
+ # Since we don't have repomd we cannot verify checksums -> fail (exception)
+
+ self.assertRaises(dnf.exceptions.RepoError,
+ self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)
+
+ def test_exception_with_just_repomd(self):
+ self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+ # Remove xml metadata except repomd
+ # repomd.xml is not compressed and doesn't end with .gz
+ repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+ for f in repodata_without_repomd:
+ os.remove(f)
+
+ # Remove all solv and solvx files
+ solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*"))
+ for f in solvx:
+ os.remove(f)
+
+ # Now we only have cache with just repomd
+ # repomd is not enough, it doesn't contain the metadata it self -> fail (exception)
+
+ self.assertRaises(dnf.exceptions.RepoError,
+ self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)
+
+ def test_exception_with_checksum_mismatch_and_only_repomd(self):
+ self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+ # Remove xml metadata except repomd
+ # repomd.xml is not compressed and doesn't end with .gz
+ repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+ for f in repodata_without_repomd:
+ os.remove(f)
+
+ # Modify checksum of solv file so it doesn't match with repomd
+ solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0]
+ with open(solv, "a") as opensolv:
+ opensolv.write("appended text to change checksum")
+
+ # Now we only have cache with solvx, modified solv file and just repomd
+ # Since we don't have original xml metadata we cannot regenerate solv -> fail (exception)
+
+ self.assertRaises(dnf.exceptions.RepoError,
+ self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)
+
+ def test_checksum_mistmatch_regenerates_solv(self):
+ self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+ # Modify checksum of solv file so it doesn't match with repomd
+ solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0]
+ with open(solv, "a") as opensolv:
+ opensolv.write("appended text to change checksum")
+
+ # Now we only have cache with solvx, modified solv file and xml metadata.
+ # Checksum mistmatch causes regeneration of solv file and repo works.
+
+ self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+ q = self.test_base.sack.query()
+ packages = q.run()
+ self.assertEqual(len(packages), 9)
+ self.assertEqual(packages[0].evr, "4-4")
+
+ def test_with_modules_yaml(self):
+ self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)),
+ "modules/modules/_all/x86_64"))
+
+ # Now we have full cache (also with modules.yaml)
+
+ self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+ q = self.test_base.sack.query()
+ packages = q.run()
+ self.assertEqual(len(packages), 8)
+ self.assertEqual(packages[0].evr, "2.02-0.40")
+
+ self.module_base = dnf.module.module_base.ModuleBase(self.test_base)
+ modules, _ = self.module_base._get_modules("base-runtime*")
+ self.assertEqual(len(modules), 3)
+ self.assertEqual(modules[0].getFullIdentifier(), "base-runtime:f26:1::")
+
+ def test_with_modular_repo_without_modules_yaml(self):
+ self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)),
+ "modules/modules/_all/x86_64"))
+
+ # Remove xml and yaml metadata except repomd
+ # repomd.xml is not compressed and doesn't end with .gz
+ repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+ for f in repodata_without_repomd:
+ os.remove(f)
+
+ # Now we have just solv, *-filenames.solvx and repomd.xml (modules.yaml are not processed into *-modules.solvx)
+
+ self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+ q = self.test_base.sack.query()
+ packages = q.run()
+ # We have many more packages because they are not hidden by modules
+ self.assertEqual(len(packages), 44)
+ self.assertEqual(packages[0].evr, "10.0-7")
+
+ self.module_base = dnf.module.module_base.ModuleBase(self.test_base)
+ modules, _ = self.module_base._get_modules("base-runtime*")
+ self.assertEqual(len(modules), 0)
From de6177dba3dc20191e275eec14672570a0c4f4a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
Date: Thu, 14 Jan 2021 12:29:06 +0100
Subject: [PATCH 3/3] Add docs and examples for fill_sack_from_repos_in_cache
(RhBug:1865803)
https://bugzilla.redhat.com/show_bug.cgi?id=1865803
---
doc/api_base.rst | 41 +++++++++++++++++++++++++++++++++++++++++
1 file changed, 41 insertions(+)
diff --git a/doc/api_base.rst b/doc/api_base.rst
index 24ecb50e43..f0b1992e88 100644
--- a/doc/api_base.rst
+++ b/doc/api_base.rst
@@ -111,6 +111,47 @@
print("id: {}".format(repo.id))
print("baseurl: {}".format(repo.baseurl))
+ .. method:: fill_sack_from_repos_in_cache(load_system_repo=True)
+
+ Prepare Sack and Goal objects and load all enabled repositories from cache only, it doesn't download anything and it doesn't check if metadata are expired.
+ To successfully load a repository cache it requires repond.xml plus metadata (xml, yaml) or repond.xml plus generated cache files (solv, solvx).
+ If there is not enough metadata given repo is either skipped or it throws a :exc:`dnf.exceptions.RepoError` exception depending on :attr:`dnf.conf.Conf.skip_if_unavailable` configuration.
+
+ All additional metadata are loaded if present but are not generally required. Note that some metadata like updateinfo.xml get processed into a solvx cache file and its sufficient to have either xml or solvx. Module metadata represented by modules.yaml are not processed therefore they are needed when they are defined in repomd.xml.
+
+ Example of loading all configured repositories from cache and printing available packages' names::
+
+ #!/usr/bin/python3
+ import dnf
+
+ with dnf.Base() as base:
+ base.read_all_repos()
+
+ base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+ query = base.sack.query().available()
+ for pkg in query.run():
+ print(pkg.name)
+
+ Example of loading a single repository and printing available packages' names without reading repository configuration::
+
+ #!/usr/bin/python3
+ import dnf
+
+ with dnf.Base() as base:
+ repo = dnf.repo.Repo("rawhide", base.conf)
+
+ # Repository cache is also identified by its source therefore to find it you need to
+ # set metalink, mirrorlist or baseurl to the same value from which it was created.
+ repo.metalink = "https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=x86_64"
+
+ base.repos.add(repo)
+
+ base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+ query = base.sack.query().available()
+ for pkg in query.run():
+ print(pkg.name)
.. method:: do_transaction([display])