richardphibel / rpms / dnf

Forked from rpms/dnf 2 years ago
Clone
Blob Blame History Raw
From a777ff01c79d5e0e2cf3ae7b0652795577253bc3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
Date: Thu, 14 Jan 2021 09:58:30 +0100
Subject: [PATCH 1/3] Fix recreate script

---
 tests/repos/rpm/recreate | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/repos/rpm/recreate b/tests/repos/rpm/recreate
index da348d9799..0fbb9396bd 100755
--- a/tests/repos/rpm/recreate
+++ b/tests/repos/rpm/recreate
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-THISDIR="$( readlink -f "$( dirname "$0 )" )"
+THISDIR="$( readlink -f "$( dirname "$0" )" )"
 cd "$THISDIR"
 git rm -rf repodata/
 createrepo --no-database -o . ..

From 5d4c0266f6967c7cd5f0e675b13fa3e9b395e4dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
Date: Thu, 14 Jan 2021 10:28:53 +0100
Subject: [PATCH 2/3] Add unit test for fill_sack_from_repos_in_cache
 (RhBug:1865803)

https://bugzilla.redhat.com/show_bug.cgi?id=1865803
---
 tests/test_fill_sack_from_repos_in_cache.py | 262 ++++++++++++++++++++
 1 file changed, 262 insertions(+)
 create mode 100644 tests/test_fill_sack_from_repos_in_cache.py

diff --git a/tests/test_fill_sack_from_repos_in_cache.py b/tests/test_fill_sack_from_repos_in_cache.py
new file mode 100644
index 0000000000..24b0d4598d
--- /dev/null
+++ b/tests/test_fill_sack_from_repos_in_cache.py
@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2012-2021 Red Hat, Inc.
+#
+# This copyrighted material is made available to anyone wishing to use,
+# modify, copy, or redistribute it subject to the terms and conditions of
+# the GNU General Public License v.2, or (at your option) any later version.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY expressed or implied, including the implied warranties of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
+# Public License for more details.  You should have received a copy of the
+# GNU General Public License along with this program; if not, write to the
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
+# source code or documentation are not subject to the GNU General Public
+# License and may only be used or replicated with the express permission of
+# Red Hat, Inc.
+#
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import os
+import tempfile
+import glob
+import shutil
+import unittest
+
+import dnf.exceptions
+import dnf.repo
+import dnf.sack
+
+import hawkey
+
+import tests.support
+from tests.support import mock
+
+TEST_REPO_NAME = "test-repo"
+
+
+class FillSackFromReposInCacheTest(unittest.TestCase):
+    def _create_cache_for_repo(self, repopath, tmpdir):
+        conf = dnf.conf.MainConf()
+        conf.cachedir = os.path.join(tmpdir, "cache")
+
+        base = dnf.Base(conf=conf)
+
+        repoconf = dnf.repo.Repo(TEST_REPO_NAME, base.conf)
+        repoconf.baseurl = repopath
+        repoconf.enable()
+
+        base.repos.add(repoconf)
+
+        base.fill_sack(load_system_repo=False)
+        base.close()
+
+    def _setUp_from_repo_path(self, original_repo_path):
+        self.tmpdir = tempfile.mkdtemp(prefix="dnf_test_")
+
+        self.repo_copy_path = os.path.join(self.tmpdir, "repo")
+        shutil.copytree(original_repo_path, self.repo_copy_path)
+
+        self._create_cache_for_repo(self.repo_copy_path, self.tmpdir)
+
+        # Just to be sure remove repo (it shouldn't be used)
+        shutil.rmtree(self.repo_copy_path)
+
+        # Prepare base for the actual test
+        conf = dnf.conf.MainConf()
+        conf.cachedir = os.path.join(self.tmpdir, "cache")
+        self.test_base = dnf.Base(conf=conf)
+        repoconf = dnf.repo.Repo(TEST_REPO_NAME, conf)
+        repoconf.baseurl = self.repo_copy_path
+        repoconf.enable()
+        self.test_base.repos.add(repoconf)
+
+    def tearDown(self):
+        self.test_base.close()
+        shutil.rmtree(self.tmpdir)
+
+    def test_with_solv_solvx_repomd(self):
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+        # Remove xml metadata except repomd
+        # repomd.xml is not compressed and doesn't end with .gz
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+        for f in repodata_without_repomd:
+            os.remove(f)
+
+        # Now we only have cache with just solv, solvx files and repomd.xml
+
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+        q = self.test_base.sack.query()
+        packages = q.run()
+        self.assertEqual(len(packages), 9)
+        self.assertEqual(packages[0].evr, "4-4")
+
+        # Use *-updateinfo.solvx
+        adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT)
+        adv_titles = set()
+        for pkg in adv_pkgs:
+            adv_titles.add(pkg.get_advisory(self.test_base.sack).title)
+        self.assertEqual(len(adv_titles), 3)
+
+    def test_with_just_solv_repomd(self):
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+        # Remove xml metadata except repomd
+        # repomd.xml is not compressed and doesn't end with .gz
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+        for f in repodata_without_repomd:
+            os.remove(f)
+
+        # Remove solvx files
+        solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solvx"))
+        for f in solvx:
+            os.remove(f)
+
+        # Now we only have cache with just solv files and repomd.xml
+
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+        q = self.test_base.sack.query()
+        packages = q.run()
+        self.assertEqual(len(packages), 9)
+        self.assertEqual(packages[0].evr, "4-4")
+
+        # No *-updateinfo.solvx -> we get no advisory packages
+        adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT)
+        self.assertEqual(len(adv_pkgs), 0)
+
+    def test_with_xml_metadata(self):
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+        # Remove all solv and solvx files
+        solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*"))
+        for f in solvx:
+            os.remove(f)
+
+        # Now we only have cache with just xml metadata
+
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+        q = self.test_base.sack.query()
+        packages = q.run()
+        self.assertEqual(len(packages), 9)
+        self.assertEqual(packages[0].evr, "4-4")
+
+    def test_exception_without_repomd(self):
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+        # Remove xml metadata
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*"))
+        for f in repodata_without_repomd:
+            os.remove(f)
+
+        # Now we only have cache with just solv and solvx files
+        # Since we don't have repomd we cannot verify checksums -> fail (exception)
+
+        self.assertRaises(dnf.exceptions.RepoError,
+                          self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)
+
+    def test_exception_with_just_repomd(self):
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+        # Remove xml metadata except repomd
+        # repomd.xml is not compressed and doesn't end with .gz
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+        for f in repodata_without_repomd:
+            os.remove(f)
+
+        # Remove all solv and solvx files
+        solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*"))
+        for f in solvx:
+            os.remove(f)
+
+        # Now we only have cache with just repomd
+        # repomd is not enough, it doesn't contain the metadata it self -> fail (exception)
+
+        self.assertRaises(dnf.exceptions.RepoError,
+                          self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)
+
+    def test_exception_with_checksum_mismatch_and_only_repomd(self):
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+        # Remove xml metadata except repomd
+        # repomd.xml is not compressed and doesn't end with .gz
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+        for f in repodata_without_repomd:
+            os.remove(f)
+
+        # Modify checksum of solv file so it doesn't match with repomd
+        solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0]
+        with open(solv, "a") as opensolv:
+            opensolv.write("appended text to change checksum")
+
+        # Now we only have cache with solvx, modified solv file and just repomd
+        # Since we don't have original xml metadata we cannot regenerate solv -> fail (exception)
+
+        self.assertRaises(dnf.exceptions.RepoError,
+                          self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)
+
+    def test_checksum_mistmatch_regenerates_solv(self):
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
+
+        # Modify checksum of solv file so it doesn't match with repomd
+        solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0]
+        with open(solv, "a") as opensolv:
+            opensolv.write("appended text to change checksum")
+
+        # Now we only have cache with solvx, modified solv file and xml metadata.
+        # Checksum mistmatch causes regeneration of solv file and repo works.
+
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+        q = self.test_base.sack.query()
+        packages = q.run()
+        self.assertEqual(len(packages), 9)
+        self.assertEqual(packages[0].evr, "4-4")
+
+    def test_with_modules_yaml(self):
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                                                "modules/modules/_all/x86_64"))
+
+        # Now we have full cache (also with modules.yaml)
+
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+        q = self.test_base.sack.query()
+        packages = q.run()
+        self.assertEqual(len(packages), 8)
+        self.assertEqual(packages[0].evr, "2.02-0.40")
+
+        self.module_base = dnf.module.module_base.ModuleBase(self.test_base)
+        modules, _ = self.module_base._get_modules("base-runtime*")
+        self.assertEqual(len(modules), 3)
+        self.assertEqual(modules[0].getFullIdentifier(), "base-runtime:f26:1::")
+
+    def test_with_modular_repo_without_modules_yaml(self):
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                                                "modules/modules/_all/x86_64"))
+
+        # Remove xml and yaml metadata except repomd
+        # repomd.xml is not compressed and doesn't end with .gz
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
+        for f in repodata_without_repomd:
+            os.remove(f)
+
+        # Now we have just solv, *-filenames.solvx and repomd.xml (modules.yaml are not processed into *-modules.solvx)
+
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+        q = self.test_base.sack.query()
+        packages = q.run()
+        # We have many more packages because they are not hidden by modules
+        self.assertEqual(len(packages), 44)
+        self.assertEqual(packages[0].evr, "10.0-7")
+
+        self.module_base = dnf.module.module_base.ModuleBase(self.test_base)
+        modules, _ = self.module_base._get_modules("base-runtime*")
+        self.assertEqual(len(modules), 0)

From de6177dba3dc20191e275eec14672570a0c4f4a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
Date: Thu, 14 Jan 2021 12:29:06 +0100
Subject: [PATCH 3/3] Add docs and examples for fill_sack_from_repos_in_cache
 (RhBug:1865803)

https://bugzilla.redhat.com/show_bug.cgi?id=1865803
---
 doc/api_base.rst | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/doc/api_base.rst b/doc/api_base.rst
index 24ecb50e43..f0b1992e88 100644
--- a/doc/api_base.rst
+++ b/doc/api_base.rst
@@ -111,6 +111,47 @@
             print("id: {}".format(repo.id))
             print("baseurl: {}".format(repo.baseurl))
 
+  .. method:: fill_sack_from_repos_in_cache(load_system_repo=True)
+
+    Prepare Sack and Goal objects and load all enabled repositories from cache only, it doesn't download anything and it doesn't check if metadata are expired.
+    To successfully load a repository cache it requires repond.xml plus metadata (xml, yaml) or repond.xml plus generated cache files (solv, solvx).
+    If there is not enough metadata given repo is either skipped or it throws a :exc:`dnf.exceptions.RepoError` exception depending on :attr:`dnf.conf.Conf.skip_if_unavailable` configuration.
+
+    All additional metadata are loaded if present but are not generally required. Note that some metadata like updateinfo.xml get processed into a solvx cache file and its sufficient to have either xml or solvx. Module metadata represented by modules.yaml are not processed therefore they are needed when they are defined in repomd.xml.
+
+    Example of loading all configured repositories from cache and printing available packages' names::
+
+        #!/usr/bin/python3
+        import dnf
+
+        with dnf.Base() as base:
+            base.read_all_repos()
+
+            base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+            query = base.sack.query().available()
+            for pkg in query.run():
+                print(pkg.name)
+
+    Example of loading a single repository and printing available packages' names without reading repository configuration::
+
+        #!/usr/bin/python3
+        import dnf
+
+        with dnf.Base() as base:
+            repo = dnf.repo.Repo("rawhide", base.conf)
+
+            # Repository cache is also identified by its source therefore to find it you need to
+            # set metalink, mirrorlist or baseurl to the same value from which it was created.
+            repo.metalink = "https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=x86_64"
+
+            base.repos.add(repo)
+
+            base.fill_sack_from_repos_in_cache(load_system_repo=False)
+
+            query = base.sack.query().available()
+            for pkg in query.run():
+                print(pkg.name)
 
   .. method:: do_transaction([display])