richardphibel / rpms / dnf

Forked from rpms/dnf 2 years ago
Clone

Blame SOURCES/0015-Add-tests-and-docs-for-fill-sack-from-repos-in-cache-RhBug-1865803.patch

31f77b
From a777ff01c79d5e0e2cf3ae7b0652795577253bc3 Mon Sep 17 00:00:00 2001
31f77b
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
31f77b
Date: Thu, 14 Jan 2021 09:58:30 +0100
31f77b
Subject: [PATCH 1/3] Fix recreate script
31f77b
31f77b
---
31f77b
 tests/repos/rpm/recreate | 2 +-
31f77b
 1 file changed, 1 insertion(+), 1 deletion(-)
31f77b
31f77b
diff --git a/tests/repos/rpm/recreate b/tests/repos/rpm/recreate
31f77b
index da348d9799..0fbb9396bd 100755
31f77b
--- a/tests/repos/rpm/recreate
31f77b
+++ b/tests/repos/rpm/recreate
31f77b
@@ -1,6 +1,6 @@
31f77b
 #!/bin/bash
31f77b
 
31f77b
-THISDIR="$( readlink -f "$( dirname "$0 )" )"
31f77b
+THISDIR="$( readlink -f "$( dirname "$0" )" )"
31f77b
 cd "$THISDIR"
31f77b
 git rm -rf repodata/
31f77b
 createrepo --no-database -o . ..
31f77b
31f77b
From 5d4c0266f6967c7cd5f0e675b13fa3e9b395e4dd Mon Sep 17 00:00:00 2001
31f77b
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
31f77b
Date: Thu, 14 Jan 2021 10:28:53 +0100
31f77b
Subject: [PATCH 2/3] Add unit test for fill_sack_from_repos_in_cache
31f77b
 (RhBug:1865803)
31f77b
31f77b
https://bugzilla.redhat.com/show_bug.cgi?id=1865803
31f77b
---
31f77b
 tests/test_fill_sack_from_repos_in_cache.py | 262 ++++++++++++++++++++
31f77b
 1 file changed, 262 insertions(+)
31f77b
 create mode 100644 tests/test_fill_sack_from_repos_in_cache.py
31f77b
31f77b
diff --git a/tests/test_fill_sack_from_repos_in_cache.py b/tests/test_fill_sack_from_repos_in_cache.py
31f77b
new file mode 100644
31f77b
index 0000000000..24b0d4598d
31f77b
--- /dev/null
31f77b
+++ b/tests/test_fill_sack_from_repos_in_cache.py
31f77b
@@ -0,0 +1,262 @@
31f77b
+# -*- coding: utf-8 -*-
31f77b
+
31f77b
+# Copyright (C) 2012-2021 Red Hat, Inc.
31f77b
+#
31f77b
+# This copyrighted material is made available to anyone wishing to use,
31f77b
+# modify, copy, or redistribute it subject to the terms and conditions of
31f77b
+# the GNU General Public License v.2, or (at your option) any later version.
31f77b
+# This program is distributed in the hope that it will be useful, but WITHOUT
31f77b
+# ANY WARRANTY expressed or implied, including the implied warranties of
31f77b
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General
31f77b
+# Public License for more details.  You should have received a copy of the
31f77b
+# GNU General Public License along with this program; if not, write to the
31f77b
+# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
31f77b
+# 02110-1301, USA.  Any Red Hat trademarks that are incorporated in the
31f77b
+# source code or documentation are not subject to the GNU General Public
31f77b
+# License and may only be used or replicated with the express permission of
31f77b
+# Red Hat, Inc.
31f77b
+#
31f77b
+
31f77b
+from __future__ import absolute_import
31f77b
+from __future__ import unicode_literals
31f77b
+
31f77b
+import os
31f77b
+import tempfile
31f77b
+import glob
31f77b
+import shutil
31f77b
+import unittest
31f77b
+
31f77b
+import dnf.exceptions
31f77b
+import dnf.repo
31f77b
+import dnf.sack
31f77b
+
31f77b
+import hawkey
31f77b
+
31f77b
+import tests.support
31f77b
+from tests.support import mock
31f77b
+
31f77b
+TEST_REPO_NAME = "test-repo"
31f77b
+
31f77b
+
31f77b
+class FillSackFromReposInCacheTest(unittest.TestCase):
31f77b
+    def _create_cache_for_repo(self, repopath, tmpdir):
31f77b
+        conf = dnf.conf.MainConf()
31f77b
+        conf.cachedir = os.path.join(tmpdir, "cache")
31f77b
+
31f77b
+        base = dnf.Base(conf=conf)
31f77b
+
31f77b
+        repoconf = dnf.repo.Repo(TEST_REPO_NAME, base.conf)
31f77b
+        repoconf.baseurl = repopath
31f77b
+        repoconf.enable()
31f77b
+
31f77b
+        base.repos.add(repoconf)
31f77b
+
31f77b
+        base.fill_sack(load_system_repo=False)
31f77b
+        base.close()
31f77b
+
31f77b
+    def _setUp_from_repo_path(self, original_repo_path):
31f77b
+        self.tmpdir = tempfile.mkdtemp(prefix="dnf_test_")
31f77b
+
31f77b
+        self.repo_copy_path = os.path.join(self.tmpdir, "repo")
31f77b
+        shutil.copytree(original_repo_path, self.repo_copy_path)
31f77b
+
31f77b
+        self._create_cache_for_repo(self.repo_copy_path, self.tmpdir)
31f77b
+
31f77b
+        # Just to be sure remove repo (it shouldn't be used)
31f77b
+        shutil.rmtree(self.repo_copy_path)
31f77b
+
31f77b
+        # Prepare base for the actual test
31f77b
+        conf = dnf.conf.MainConf()
31f77b
+        conf.cachedir = os.path.join(self.tmpdir, "cache")
31f77b
+        self.test_base = dnf.Base(conf=conf)
31f77b
+        repoconf = dnf.repo.Repo(TEST_REPO_NAME, conf)
31f77b
+        repoconf.baseurl = self.repo_copy_path
31f77b
+        repoconf.enable()
31f77b
+        self.test_base.repos.add(repoconf)
31f77b
+
31f77b
+    def tearDown(self):
31f77b
+        self.test_base.close()
31f77b
+        shutil.rmtree(self.tmpdir)
31f77b
+
31f77b
+    def test_with_solv_solvx_repomd(self):
31f77b
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
31f77b
+
31f77b
+        # Remove xml metadata except repomd
31f77b
+        # repomd.xml is not compressed and doesn't end with .gz
31f77b
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
31f77b
+        for f in repodata_without_repomd:
31f77b
+            os.remove(f)
31f77b
+
31f77b
+        # Now we only have cache with just solv, solvx files and repomd.xml
31f77b
+
31f77b
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
31f77b
+
31f77b
+        q = self.test_base.sack.query()
31f77b
+        packages = q.run()
31f77b
+        self.assertEqual(len(packages), 9)
31f77b
+        self.assertEqual(packages[0].evr, "4-4")
31f77b
+
31f77b
+        # Use *-updateinfo.solvx
31f77b
+        adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT)
31f77b
+        adv_titles = set()
31f77b
+        for pkg in adv_pkgs:
31f77b
+            adv_titles.add(pkg.get_advisory(self.test_base.sack).title)
31f77b
+        self.assertEqual(len(adv_titles), 3)
31f77b
+
31f77b
+    def test_with_just_solv_repomd(self):
31f77b
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
31f77b
+
31f77b
+        # Remove xml metadata except repomd
31f77b
+        # repomd.xml is not compressed and doesn't end with .gz
31f77b
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
31f77b
+        for f in repodata_without_repomd:
31f77b
+            os.remove(f)
31f77b
+
31f77b
+        # Remove solvx files
31f77b
+        solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solvx"))
31f77b
+        for f in solvx:
31f77b
+            os.remove(f)
31f77b
+
31f77b
+        # Now we only have cache with just solv files and repomd.xml
31f77b
+
31f77b
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
31f77b
+
31f77b
+        q = self.test_base.sack.query()
31f77b
+        packages = q.run()
31f77b
+        self.assertEqual(len(packages), 9)
31f77b
+        self.assertEqual(packages[0].evr, "4-4")
31f77b
+
31f77b
+        # No *-updateinfo.solvx -> we get no advisory packages
31f77b
+        adv_pkgs = q.get_advisory_pkgs(hawkey.LT | hawkey.EQ | hawkey.GT)
31f77b
+        self.assertEqual(len(adv_pkgs), 0)
31f77b
+
31f77b
+    def test_with_xml_metadata(self):
31f77b
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
31f77b
+
31f77b
+        # Remove all solv and solvx files
31f77b
+        solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*"))
31f77b
+        for f in solvx:
31f77b
+            os.remove(f)
31f77b
+
31f77b
+        # Now we only have cache with just xml metadata
31f77b
+
31f77b
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
31f77b
+
31f77b
+        q = self.test_base.sack.query()
31f77b
+        packages = q.run()
31f77b
+        self.assertEqual(len(packages), 9)
31f77b
+        self.assertEqual(packages[0].evr, "4-4")
31f77b
+
31f77b
+    def test_exception_without_repomd(self):
31f77b
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
31f77b
+
31f77b
+        # Remove xml metadata
31f77b
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*"))
31f77b
+        for f in repodata_without_repomd:
31f77b
+            os.remove(f)
31f77b
+
31f77b
+        # Now we only have cache with just solv and solvx files
31f77b
+        # Since we don't have repomd we cannot verify checksums -> fail (exception)
31f77b
+
31f77b
+        self.assertRaises(dnf.exceptions.RepoError,
31f77b
+                          self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)
31f77b
+
31f77b
+    def test_exception_with_just_repomd(self):
31f77b
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
31f77b
+
31f77b
+        # Remove xml metadata except repomd
31f77b
+        # repomd.xml is not compressed and doesn't end with .gz
31f77b
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
31f77b
+        for f in repodata_without_repomd:
31f77b
+            os.remove(f)
31f77b
+
31f77b
+        # Remove all solv and solvx files
31f77b
+        solvx = glob.glob(os.path.join(self.tmpdir, "cache/*.solv*"))
31f77b
+        for f in solvx:
31f77b
+            os.remove(f)
31f77b
+
31f77b
+        # Now we only have cache with just repomd
31f77b
+        # repomd is not enough, it doesn't contain the metadata it self -> fail (exception)
31f77b
+
31f77b
+        self.assertRaises(dnf.exceptions.RepoError,
31f77b
+                          self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)
31f77b
+
31f77b
+    def test_exception_with_checksum_mismatch_and_only_repomd(self):
31f77b
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
31f77b
+
31f77b
+        # Remove xml metadata except repomd
31f77b
+        # repomd.xml is not compressed and doesn't end with .gz
31f77b
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
31f77b
+        for f in repodata_without_repomd:
31f77b
+            os.remove(f)
31f77b
+
31f77b
+        # Modify checksum of solv file so it doesn't match with repomd
31f77b
+        solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0]
31f77b
+        with open(solv, "a") as opensolv:
31f77b
+            opensolv.write("appended text to change checksum")
31f77b
+
31f77b
+        # Now we only have cache with solvx, modified solv file and just repomd
31f77b
+        # Since we don't have original xml metadata we cannot regenerate solv -> fail (exception)
31f77b
+
31f77b
+        self.assertRaises(dnf.exceptions.RepoError,
31f77b
+                          self.test_base.fill_sack_from_repos_in_cache, load_system_repo=False)
31f77b
+
31f77b
+    def test_checksum_mistmatch_regenerates_solv(self):
31f77b
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)), "repos/rpm"))
31f77b
+
31f77b
+        # Modify checksum of solv file so it doesn't match with repomd
31f77b
+        solv = glob.glob(os.path.join(self.tmpdir, "cache/*.solv"))[0]
31f77b
+        with open(solv, "a") as opensolv:
31f77b
+            opensolv.write("appended text to change checksum")
31f77b
+
31f77b
+        # Now we only have cache with solvx, modified solv file and xml metadata.
31f77b
+        # Checksum mistmatch causes regeneration of solv file and repo works.
31f77b
+
31f77b
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
31f77b
+
31f77b
+        q = self.test_base.sack.query()
31f77b
+        packages = q.run()
31f77b
+        self.assertEqual(len(packages), 9)
31f77b
+        self.assertEqual(packages[0].evr, "4-4")
31f77b
+
31f77b
+    def test_with_modules_yaml(self):
31f77b
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)),
31f77b
+                                                "modules/modules/_all/x86_64"))
31f77b
+
31f77b
+        # Now we have full cache (also with modules.yaml)
31f77b
+
31f77b
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
31f77b
+
31f77b
+        q = self.test_base.sack.query()
31f77b
+        packages = q.run()
31f77b
+        self.assertEqual(len(packages), 8)
31f77b
+        self.assertEqual(packages[0].evr, "2.02-0.40")
31f77b
+
31f77b
+        self.module_base = dnf.module.module_base.ModuleBase(self.test_base)
31f77b
+        modules, _ = self.module_base._get_modules("base-runtime*")
31f77b
+        self.assertEqual(len(modules), 3)
31f77b
+        self.assertEqual(modules[0].getFullIdentifier(), "base-runtime:f26:1::")
31f77b
+
31f77b
+    def test_with_modular_repo_without_modules_yaml(self):
31f77b
+        self._setUp_from_repo_path(os.path.join(os.path.abspath(os.path.dirname(__file__)),
31f77b
+                                                "modules/modules/_all/x86_64"))
31f77b
+
31f77b
+        # Remove xml and yaml metadata except repomd
31f77b
+        # repomd.xml is not compressed and doesn't end with .gz
31f77b
+        repodata_without_repomd = glob.glob(os.path.join(self.tmpdir, "cache/test-repo-*/repodata/*.gz"))
31f77b
+        for f in repodata_without_repomd:
31f77b
+            os.remove(f)
31f77b
+
31f77b
+        # Now we have just solv, *-filenames.solvx and repomd.xml (modules.yaml are not processed into *-modules.solvx)
31f77b
+
31f77b
+        self.test_base.fill_sack_from_repos_in_cache(load_system_repo=False)
31f77b
+
31f77b
+        q = self.test_base.sack.query()
31f77b
+        packages = q.run()
31f77b
+        # We have many more packages because they are not hidden by modules
31f77b
+        self.assertEqual(len(packages), 44)
31f77b
+        self.assertEqual(packages[0].evr, "10.0-7")
31f77b
+
31f77b
+        self.module_base = dnf.module.module_base.ModuleBase(self.test_base)
31f77b
+        modules, _ = self.module_base._get_modules("base-runtime*")
31f77b
+        self.assertEqual(len(modules), 0)
31f77b
31f77b
From de6177dba3dc20191e275eec14672570a0c4f4a8 Mon Sep 17 00:00:00 2001
31f77b
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
31f77b
Date: Thu, 14 Jan 2021 12:29:06 +0100
31f77b
Subject: [PATCH 3/3] Add docs and examples for fill_sack_from_repos_in_cache
31f77b
 (RhBug:1865803)
31f77b
31f77b
https://bugzilla.redhat.com/show_bug.cgi?id=1865803
31f77b
---
31f77b
 doc/api_base.rst | 41 +++++++++++++++++++++++++++++++++++++++++
31f77b
 1 file changed, 41 insertions(+)
31f77b
31f77b
diff --git a/doc/api_base.rst b/doc/api_base.rst
31f77b
index 24ecb50e43..f0b1992e88 100644
31f77b
--- a/doc/api_base.rst
31f77b
+++ b/doc/api_base.rst
31f77b
@@ -111,6 +111,47 @@
31f77b
             print("id: {}".format(repo.id))
31f77b
             print("baseurl: {}".format(repo.baseurl))
31f77b
 
31f77b
+  .. method:: fill_sack_from_repos_in_cache(load_system_repo=True)
31f77b
+
31f77b
+    Prepare Sack and Goal objects and load all enabled repositories from cache only, it doesn't download anything and it doesn't check if metadata are expired.
31f77b
+    To successfully load a repository cache it requires repond.xml plus metadata (xml, yaml) or repond.xml plus generated cache files (solv, solvx).
31f77b
+    If there is not enough metadata given repo is either skipped or it throws a :exc:`dnf.exceptions.RepoError` exception depending on :attr:`dnf.conf.Conf.skip_if_unavailable` configuration.
31f77b
+
31f77b
+    All additional metadata are loaded if present but are not generally required. Note that some metadata like updateinfo.xml get processed into a solvx cache file and its sufficient to have either xml or solvx. Module metadata represented by modules.yaml are not processed therefore they are needed when they are defined in repomd.xml.
31f77b
+
31f77b
+    Example of loading all configured repositories from cache and printing available packages' names::
31f77b
+
31f77b
+        #!/usr/bin/python3
31f77b
+        import dnf
31f77b
+
31f77b
+        with dnf.Base() as base:
31f77b
+            base.read_all_repos()
31f77b
+
31f77b
+            base.fill_sack_from_repos_in_cache(load_system_repo=False)
31f77b
+
31f77b
+            query = base.sack.query().available()
31f77b
+            for pkg in query.run():
31f77b
+                print(pkg.name)
31f77b
+
31f77b
+    Example of loading a single repository and printing available packages' names without reading repository configuration::
31f77b
+
31f77b
+        #!/usr/bin/python3
31f77b
+        import dnf
31f77b
+
31f77b
+        with dnf.Base() as base:
31f77b
+            repo = dnf.repo.Repo("rawhide", base.conf)
31f77b
+
31f77b
+            # Repository cache is also identified by its source therefore to find it you need to
31f77b
+            # set metalink, mirrorlist or baseurl to the same value from which it was created.
31f77b
+            repo.metalink = "https://mirrors.fedoraproject.org/metalink?repo=rawhide&arch=x86_64"
31f77b
+
31f77b
+            base.repos.add(repo)
31f77b
+
31f77b
+            base.fill_sack_from_repos_in_cache(load_system_repo=False)
31f77b
+
31f77b
+            query = base.sack.query().available()
31f77b
+            for pkg in query.run():
31f77b
+                print(pkg.name)
31f77b
 
31f77b
   .. method:: do_transaction([display])
31f77b