From a777ff01c79d5e0e2cf3ae7b0652795577253bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Thu, 14 Jan 2021 09:58:30 +0100 Subject: [PATCH 1/3] Fix recreate script --- tests/repos/rpm/recreate | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/repos/rpm/recreate b/tests/repos/rpm/recreate index da348d9799..0fbb9396bd 100755 --- a/tests/repos/rpm/recreate +++ b/tests/repos/rpm/recreate @@ -1,6 +1,6 @@ #!/bin/bash -THISDIR="$( readlink -f "$( dirname "$0 )" )" +THISDIR="$( readlink -f "$( dirname "$0" )" )" cd "$THISDIR" git rm -rf repodata/ createrepo --no-database -o . .. From 5d4c0266f6967c7cd5f0e675b13fa3e9b395e4dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Thu, 14 Jan 2021 10:28:53 +0100 Subject: [PATCH 2/3] Add unit test for fill_sack_from_repos_in_cache (RhBug:1865803) https://bugzilla.redhat.com/show_bug.cgi?id=1865803 --- tests/test_fill_sack_from_repos_in_cache.py | 262 ++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 tests/test_fill_sack_from_repos_in_cache.py diff --git a/tests/test_fill_sack_from_repos_in_cache.py b/tests/test_fill_sack_from_repos_in_cache.py new file mode 100644 index 0000000000..24b0d4598d --- /dev/null +++ b/tests/test_fill_sack_from_repos_in_cache.py @@ -0,0 +1,262 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2012-2021 Red Hat, Inc. +# +# This copyrighted material is made available to anyone wishing to use, +# modify, copy, or redistribute it subject to the terms and conditions of +# the GNU General Public License v.2, or (at your option) any later version. +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY expressed or implied, including the implied warranties of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. You should have received a copy of the +# GNU General Public License along with this program; if not, write to the +# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. Any Red Hat trademarks that are incorporated in the +# source code or documentation are not subject to the GNU General Public +# License and may only be used or replicated with the express permission of +# Red Hat, Inc. +# + +from __future__ import absolute_import +from __future__ import unicode_literals + +import os +import tempfile +import glob +import shutil +import unittest + +import dnf.exceptions +import dnf.repo +import dnf.sack + +import hawkey + +import tests.support +from tests.support import mock + +TEST_REPO_NAME = "test-repo" + + +class FillSackFromReposInCacheTest(unittest.TestCase): + def _create_cache_for_repo(self, repopath, tmpdir): + conf = dnf.conf.MainConf() + conf.cachedir = os.path.join(tmpdir, "cache") + + base = dnf.Base(conf=conf) + + repoconf = dnf.repo.Repo(TEST_REPO_NAME, base.conf) + repoconf.baseurl = repopath + repoconf.enable() + + base.repos.add(repoconf) + + base.fill_sack(load_system_repo=False) + base.close() + + def _setUp_from_repo_path(self, original_repo_path): + self.tmpdir = tempfile.mkdtemp(prefix="dnf_test_") + + self.repo_copy_path = os.path.join(self.tmpdir, "repo") + shutil.copytree(original_repo_path, self.repo_copy_path) + + self._create_cache_for_repo(self.repo_copy_path, self.tmpdir) + + # Just to be sure remove repo (it shouldn't be used) + shutil.rmtree(self.repo_copy_path) + + # Prepare base for the actual test + conf = dnf.conf.MainConf() + conf.cachedir = os.path.join(self.tmpdir, "cache") + self.test_base = dnf.Base(conf=conf) + repoconf = dnf.repo.Repo(TEST_REPO_NAME, conf) + repoconf.baseurl = self.repo_copy_path + repoconf.enable() + self.test_base.repos.add(repoconf) + + def tearDown(self): + self.test_base.close() + shutil.rmtree(self.tmpdir) + + def test_with_solv_solvx_repomd(self): + self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm")) + + # Remove xml metadata except repomd + # repomd.xml is not compressed and doesn't end with .gz + repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz")) + for f in repodata_without_repomd: + os.remove(f) + + # Now we only have cache with just solv, solvx files and repomd.xml + + self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False) + + q = self.test_base.sack.query() + packages = q.run() + self.assertEqual(len(packages), 9) + self.assertEqual(packages[0].evr, "4-4") + + # Use *-updateinfo.solvx + adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT) + adv_titles = set() + for pkg in adv_pkgs: + adv_titles.add(pkg.get_advisory(self.test_base.sack).title) + self.assertEqual(len(adv_titles), 3) + + def test_with_just_solv_repomd(self): + self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm")) + + # Remove xml metadata except repomd + # repomd.xml is not compressed and doesn't end with .gz + repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz")) + for f in repodata_without_repomd: + os.remove(f) + + # Remove solvx files + solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solvx")) + for f in solvx: + os.remove(f) + + # Now we only have cache with just solv files and repomd.xml + + self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False) + + q = self.test_base.sack.query() + packages = q.run() + self.assertEqual(len(packages), 9) + self.assertEqual(packages[0].evr, "4-4") + + # No *-updateinfo.solvx -> we get no advisory packages + adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT) + self.assertEqual(len(adv_pkgs), 0) + + def test_with_xml_metadata(self): + self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm")) + + # Remove all solv and solvx files + solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*")) + for f in solvx: + os.remove(f) + + # Now we only have cache with just xml metadata + + self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False) + + q = self.test_base.sack.query() + packages = q.run() + self.assertEqual(len(packages), 9) + self.assertEqual(packages[0].evr, "4-4") + + def test_exception_without_repomd(self): + self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm")) + + # Remove xml metadata + repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*")) + for f in repodata_without_repomd: + os.remove(f) + + # Now we only have cache with just solv and solvx files + # Since we don't have repomd we cannot verify checksums -> fail (exception) + + self.assertRaises(dnf.exceptions.RepoError, + self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False) + + def test_exception_with_just_repomd(self): + self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm")) + + # Remove xml metadata except repomd + # repomd.xml is not compressed and doesn't end with .gz + repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz")) + for f in repodata_without_repomd: + os.remove(f) + + # Remove all solv and solvx files + solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*")) + for f in solvx: + os.remove(f) + + # Now we only have cache with just repomd + # repomd is not enough, it doesn't contain the metadata it self -> fail (exception) + + self.assertRaises(dnf.exceptions.RepoError, + self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False) + + def test_exception_with_checksum_mismatch_and_only_repomd(self): + self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm")) + + # Remove xml metadata except repomd + # repomd.xml is not compressed and doesn't end with .gz + repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz")) + for f in repodata_without_repomd: + os.remove(f) + + # Modify checksum of solv file so it doesn't match with repomd + solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0] + with open(solv, "a") as opensolv: + opensolv.write("appended text to change checksum") + + # Now we only have cache with solvx, modified solv file and just repomd + # Since we don't have original xml metadata we cannot regenerate solv -> fail (exception) + + self.assertRaises(dnf.exceptions.RepoError, + self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False) + + def test_checksum_mistmatch_regenerates_solv(self): + self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm")) + + # Modify checksum of solv file so it doesn't match with repomd + solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0] + with open(solv, "a") as opensolv: + opensolv.write("appended text to change checksum") + + # Now we only have cache with solvx, modified solv file and xml metadata. + # Checksum mistmatch causes regeneration of solv file and repo works. + + self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False) + + q = self.test_base.sack.query() + packages = q.run() + self.assertEqual(len(packages), 9) + self.assertEqual(packages[0].evr, "4-4") + + def test_with_modules_yaml(self): + self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), + "modules/modules/_all/x86_64")) + + # Now we have full cache (also with modules.yaml) + + self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False) + + q = self.test_base.sack.query() + packages = q.run() + self.assertEqual(len(packages), 8) + self.assertEqual(packages[0].evr, "2.02-0.40") + + self.module_base = dnf.module.module_base.ModuleBase(self.test_base) + modules, _ = self.module_base._get_modules("base-runtime*") + self.assertEqual(len(modules), 3) + self.assertEqual(modules[0].getFullIdentifier(), "base-runtime:f26:1::") + + def test_with_modular_repo_without_modules_yaml(self): + self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), + "modules/modules/_all/x86_64")) + + # Remove xml and yaml metadata except repomd + # repomd.xml is not compressed and doesn't end with .gz + repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz")) + for f in repodata_without_repomd: + os.remove(f) + + # Now we have just solv, *-filenames.solvx and repomd.xml (modules.yaml are not processed into *-modules.solvx) + + self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False) + + q = self.test_base.sack.query() + packages = q.run() + # We have many more packages because they are not hidden by modules + self.assertEqual(len(packages), 44) + self.assertEqual(packages[0].evr, "10.0-7") + + self.module_base = dnf.module.module_base.ModuleBase(self.test_base) + modules, _ = self.module_base._get_modules("base-runtime*") + self.assertEqual(len(modules), 0) From de6177dba3dc20191e275eec14672570a0c4f4a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= Date: Thu, 14 Jan 2021 12:29:06 +0100 Subject: [PATCH 3/3] Add docs and examples for fill_sack_from_repos_in_cache (RhBug:1865803) https://bugzilla.redhat.com/show_bug.cgi?id=1865803 --- doc/api_base.rst | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/doc/api_base.rst b/doc/api_base.rst index 24ecb50e43..f0b1992e88 100644 --- a/doc/api_base.rst +++ b/doc/api_base.rst @@ -111,6 +111,47 @@ print("id: {}".format(repo.id)) print("baseurl: {}".format(repo.baseurl)) + .. method:: fill_sack_from_repos_in_cache(load_system_repo=True) + + Prepare Sack and Goal objects and load all enabled repositories from cache only, it doesn't download anything and it doesn't check if metadata are expired. + To successfully load a repository cache it requires repond.xml plus metadata (xml, yaml) or repond.xml plus generated cache files (solv, solvx). + If there is not enough metadata given repo is either skipped or it throws a :exc:`dnf.exceptions.RepoError` exception depending on :attr:`dnf.conf.Conf.skip_if_unavailable` configuration. + + All additional metadata are loaded if present but are not generally required. Note that some metadata like updateinfo.xml get processed into a solvx cache file and its sufficient to have either xml or solvx. Module metadata represented by modules.yaml are not processed therefore they are needed when they are defined in repomd.xml. + + Example of loading all configured repositories from cache and printing available packages' names:: + + #!/usr/bin/python3 + import dnf + + with dnf.Base() as base: + base.read_all_repos() + + base.fill_sack_from_repos_in_cache(load_system_repo=False) + + query = base.sack.query().available() + for pkg in query.run(): + print(pkg.name) + + Example of loading a single repository and printing available packages' names without reading repository configuration:: + + #!/usr/bin/python3 + import dnf + + with dnf.Base() as base: + repo = dnf.repo.Repo("rawhide", base.conf) + + # Repository cache is also identified by its source therefore to find it you need to + # set metalink, mirrorlist or baseurl to the same value from which it was created. + repo.metalink = "https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=x86_64" + + base.repos.add(repo) + + base.fill_sack_from_repos_in_cache(load_system_repo=False) + + query = base.sack.query().available() + for pkg in query.run(): + print(pkg.name) .. method:: do_transaction([display])