From 465a6a59279bd7fa2680c626ca0f10c059276668 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ale=C5=A1=20Mat=C4=9Bj?= <amatej@redhat.com>
Date: Wed, 9 Feb 2022 13:18:41 +0100
Subject: [PATCH 29/34] Use dnf solv userdata to check versions and checksum
(RhBug:2027445)
Remove unused functions for checksums
= changelog =
msg: Write and check versions and checksums for solvfile cache by using new dnf solvfile userdata (RhBug:2027445)
It is not possible to use old cache files, therefore cache regeneration is triggered automatically.
type: bugfix
resolves: https://bugzilla.redhat.com/show_bug.cgi?id=2027445
---
libdnf/dnf-sack.cpp | 254 ++++++++++++++++++++++--------------
libdnf/hy-iutil-private.hpp | 2 -
libdnf/hy-iutil.cpp | 20 ---
3 files changed, 156 insertions(+), 120 deletions(-)
diff --git a/libdnf/dnf-sack.cpp b/libdnf/dnf-sack.cpp
index b9baeaef..61f4807c 100644
--- a/libdnf/dnf-sack.cpp
+++ b/libdnf/dnf-sack.cpp
@@ -225,17 +225,39 @@ dnf_sack_new(void)
return DNF_SACK(g_object_new(DNF_TYPE_SACK, NULL));
}
-static int
-can_use_repomd_cache(FILE *fp_solv, unsigned char cs_repomd[CHKSUM_BYTES])
-{
- unsigned char cs_cache[CHKSUM_BYTES];
-
- if (fp_solv &&
- !checksum_read(cs_cache, fp_solv) &&
- !checksum_cmp(cs_cache, cs_repomd))
- return 1;
+// Try to load cached solv file into repo otherwise return FALSE
+static gboolean
+try_to_use_cached_solvfile(const char *path, Repo *repo, int flags, const unsigned char *checksum, GError **err){
+ FILE *fp_cache = fopen(path, "r");
+ if (!fp_cache) {
+ // Missing cache files (ENOENT) are not an error and can even be expected in some cases
+ // (such as when repo doesn't have updateinfo/prestodelta metadata).
+ // Use g_debug in order not to pollute the log by default with such entries.
+ if (errno == ENOENT) {
+ g_debug("Failed to open solvfile cache: %s: %s", path, strerror(errno));
+ } else {
+ g_warning("Failed to open solvfile cache: %s: %s", path, strerror(errno));
+ }
+ return FALSE;
+ }
+ std::unique_ptr<SolvUserdata> solv_userdata = solv_userdata_read(fp_cache);
+ gboolean ret = TRUE;
+ if (solv_userdata && solv_userdata_verify(solv_userdata.get(), checksum)) {
+ // after reading the header rewind to the begining
+ fseek(fp_cache, 0, SEEK_SET);
+ if (repo_add_solv(repo, fp_cache, flags)) {
+ g_set_error (err,
+ DNF_ERROR,
+ DNF_ERROR_INTERNAL_ERROR,
+ _("repo_add_solv() has failed."));
+ ret = FALSE;
+ }
+ } else {
+ ret = FALSE;
+ }
- return 0;
+ fclose(fp_cache);
+ return ret;
}
void
@@ -375,33 +397,27 @@ load_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata,
gboolean done = FALSE;
char *fn_cache = dnf_sack_give_cache_fn(sack, name, suffix);
- fp = fopen(fn_cache, "r");
assert(libdnf::repoGetImpl(hrepo)->checksum);
- if (can_use_repomd_cache(fp, libdnf::repoGetImpl(hrepo)->checksum)) {
- int flags = 0;
- /* the updateinfo is not a real extension */
- if (which_repodata != _HY_REPODATA_UPDATEINFO)
- flags |= REPO_EXTEND_SOLVABLES;
- /* do not pollute the main pool with directory component ids */
- if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)
- flags |= REPO_LOCALPOOL;
- done = TRUE;
+
+ int flags = 0;
+ /* the updateinfo is not a real extension */
+ if (which_repodata != _HY_REPODATA_UPDATEINFO)
+ flags |= REPO_EXTEND_SOLVABLES;
+ /* do not pollute the main pool with directory component ids */
+ if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)
+ flags |= REPO_LOCALPOOL;
+ if (try_to_use_cached_solvfile(fn_cache, repo, flags, libdnf::repoGetImpl(hrepo)->checksum, error)) {
g_debug("%s: using cache file: %s", __func__, fn_cache);
- ret = repo_add_solv(repo, fp, flags);
- if (ret) {
- g_set_error_literal (error,
- DNF_ERROR,
- DNF_ERROR_INTERNAL_ERROR,
- _("failed to add solv"));
- return FALSE;
- } else {
- repo_update_state(hrepo, which_repodata, _HY_LOADED_CACHE);
- repo_set_repodata(hrepo, which_repodata, repo->nrepodata - 1);
- }
+ done = TRUE;
+ repo_update_state(hrepo, which_repodata, _HY_LOADED_CACHE);
+ repo_set_repodata(hrepo, which_repodata, repo->nrepodata - 1);
}
+ if (error && *error) {
+ g_prefix_error(error, _("Loading extension cache %s (%d) failed: "), fn_cache, which_repodata);
+ return FALSE;
+ }
+
g_free(fn_cache);
- if (fp)
- fclose(fp);
if (done)
return TRUE;
@@ -514,35 +530,53 @@ write_main(DnfSack *sack, HyRepo hrepo, int switchtosolv, GError **error)
strerror(errno));
goto done;
}
- rc = repo_write(repo, fp);
- rc |= checksum_write(repoImpl->checksum, fp);
- rc |= fclose(fp);
+
+ SolvUserdata solv_userdata;
+ if (solv_userdata_fill(&solv_userdata, repoImpl->checksum, error)) {
+ ret = FALSE;
+ fclose(fp);
+ goto done;
+ }
+
+ Repowriter *writer = repowriter_create(repo);
+ repowriter_set_userdata(writer, &solv_userdata, solv_userdata_size);
+ rc = repowriter_write(writer, fp);
+ repowriter_free(writer);
if (rc) {
+ ret = FALSE;
+ fclose(fp);
+ g_set_error(error,
+ DNF_ERROR,
+ DNF_ERROR_INTERNAL_ERROR,
+ _("While writing primary cache %s repowriter write failed: %i, error: %s"),
+ tmp_fn_templ, rc, pool_errstr(repo->pool));
+ goto done;
+ }
+
+ if (fclose(fp)) {
ret = FALSE;
g_set_error (error,
DNF_ERROR,
DNF_ERROR_FILE_INVALID,
- _("write_main() failed writing data: %i"), rc);
+ _("Failed closing tmp file %s: %s"),
+ tmp_fn_templ, strerror(errno));
goto done;
}
}
if (switchtosolv && repo_is_one_piece(repo)) {
+ repo_empty(repo, 1);
/* switch over to written solv file activate paging */
- FILE *fp = fopen(tmp_fn_templ, "r");
- if (fp) {
- repo_empty(repo, 1);
- rc = repo_add_solv(repo, fp, 0);
- fclose(fp);
- if (rc) {
- /* this is pretty fatal */
- ret = FALSE;
- g_set_error_literal (error,
- DNF_ERROR,
- DNF_ERROR_FILE_INVALID,
- _("write_main() failed to re-load "
- "written solv file"));
- goto done;
- }
+ gboolean loaded = try_to_use_cached_solvfile(tmp_fn_templ, repo, 0, repoImpl->checksum, error);
+ if (error && *error) {
+ g_prefix_error(error, _("Failed to use newly written primary cache: %s: "), tmp_fn_templ);
+ ret = FALSE;
+ goto done;
+ }
+ if (!loaded) {
+ g_set_error(error, DNF_ERROR, DNF_ERROR_INTERNAL_ERROR,
+ _("Failed to use newly written primary cache: %s"), tmp_fn_templ);
+ ret = FALSE;
+ goto done;
}
}
@@ -569,20 +603,6 @@ write_ext_updateinfo_filter(Repo *repo, Repokey *key, void *kfdata)
return repo_write_stdkeyfilter(repo, key, 0);
}
-static int
-write_ext_updateinfo(HyRepo hrepo, Repodata *data, FILE *fp)
-{
- auto repoImpl = libdnf::repoGetImpl(hrepo);
- Repo *repo = repoImpl->libsolvRepo;
- int oldstart = repo->start;
- repo->start = repoImpl->main_end;
- repo->nsolvables -= repoImpl->main_nsolvables;
- int res = repo_write_filtered(repo, fp, write_ext_updateinfo_filter, data, 0);
- repo->start = oldstart;
- repo->nsolvables += repoImpl->main_nsolvables;
- return res;
-}
-
static gboolean
write_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata,
const char *suffix, GError **error)
@@ -611,37 +631,78 @@ write_ext(DnfSack *sack, HyRepo hrepo, _hy_repo_repodata which_repodata,
FILE *fp = fdopen(tmp_fd, "w+");
g_debug("%s: storing %s to: %s", __func__, repo->name, tmp_fn_templ);
- if (which_repodata != _HY_REPODATA_UPDATEINFO)
- ret |= repodata_write(data, fp);
- else
- ret |= write_ext_updateinfo(hrepo, data, fp);
- ret |= checksum_write(repoImpl->checksum, fp);
- ret |= fclose(fp);
+
+ SolvUserdata solv_userdata;
+ if (solv_userdata_fill(&solv_userdata, repoImpl->checksum, error)) {
+ fclose(fp);
+ success = FALSE;
+ goto done;
+ }
+
+ Repowriter *writer = repowriter_create(repo);
+ repowriter_set_userdata(writer, &solv_userdata, solv_userdata_size);
+ if (which_repodata != _HY_REPODATA_UPDATEINFO) {
+ repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
+ repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE);
+ ret = repowriter_write(writer, fp);
+ } else {
+ // write only updateinfo repodata
+ int oldstart = repo->start;
+ repo->start = repoImpl->main_end;
+ repo->nsolvables -= repoImpl->main_nsolvables;
+ repowriter_set_flags(writer, REPOWRITER_LEGACY);
+ repowriter_set_keyfilter(writer, write_ext_updateinfo_filter, data);
+ repowriter_set_keyqueue(writer, 0);
+ ret = repowriter_write(writer, fp);
+ repo->start = oldstart;
+ repo->nsolvables += repoImpl->main_nsolvables;
+ }
+ repowriter_free(writer);
if (ret) {
+ success = FALSE;
+ fclose(fp);
+ g_set_error (error,
+ DNF_ERROR,
+ DNF_ERROR_INTERNAL_ERROR,
+ _("While writing extension cache %s (%d): repowriter write failed: %i, error: %s"),
+ tmp_fn_templ, which_repodata, ret, pool_errstr(repo->pool));
+ goto done;
+ }
+
+ if (fclose(fp)) {
success = FALSE;
g_set_error (error,
DNF_ERROR,
- DNF_ERROR_FAILED,
- _("write_ext(%1$d) has failed: %2$d"),
- which_repodata, ret);
+ DNF_ERROR_FILE_INVALID,
+ _("While writing extension cache (%d): cannot close temporary file: %s"),
+ which_repodata, tmp_fn_templ);
goto done;
}
}
if (repo_is_one_piece(repo) && which_repodata != _HY_REPODATA_UPDATEINFO) {
/* switch over to written solv file activate paging */
- FILE *fp = fopen(tmp_fn_templ, "r");
- if (fp) {
- int flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES;
- /* do not pollute the main pool with directory component ids */
- if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)
- flags |= REPO_LOCALPOOL;
- repodata_extend_block(data, repo->start, repo->end - repo->start);
- data->state = REPODATA_LOADING;
- repo_add_solv(repo, fp, flags);
- data->state = REPODATA_AVAILABLE;
- fclose(fp);
+ int flags = REPO_USE_LOADING | REPO_EXTEND_SOLVABLES;
+ /* do not pollute the main pool with directory component ids */
+ if (which_repodata == _HY_REPODATA_FILENAMES || which_repodata == _HY_REPODATA_OTHER)
+ flags |= REPO_LOCALPOOL;
+ repodata_extend_block(data, repo->start, repo->end - repo->start);
+ data->state = REPODATA_LOADING;
+ int loaded = try_to_use_cached_solvfile(tmp_fn_templ, repo, flags, repoImpl->checksum, error);
+ if (error && *error) {
+ g_prefix_error(error, _("Failed to use newly written extension cache: %s (%d): "),
+ tmp_fn_templ, which_repodata);
+ success = FALSE;
+ goto done;
+ }
+ if (!loaded) {
+ g_set_error(error, DNF_ERROR, DNF_ERROR_INTERNAL_ERROR,
+ _("Failed to use newly written extension cache: %s (%d)"), tmp_fn_templ, which_repodata);
+ success = FALSE;
+ goto done;
}
+
+ data->state = REPODATA_AVAILABLE;
}
if (!mv(tmp_fn_templ, fn, error)) {
@@ -672,7 +733,7 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error)
FILE *fp_primary = NULL;
FILE *fp_repomd = NULL;
- FILE *fp_cache = fopen(fn_cache, "r");
+
if (!fn_repomd) {
g_set_error (error,
DNF_ERROR,
@@ -693,18 +754,17 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error)
}
checksum_fp(repoImpl->checksum, fp_repomd);
- if (can_use_repomd_cache(fp_cache, repoImpl->checksum)) {
+ if (try_to_use_cached_solvfile(fn_cache, repo, 0, repoImpl->checksum, error)) {
const char *chksum = pool_checksum_str(pool, repoImpl->checksum);
g_debug("using cached %s (0x%s)", name, chksum);
- if (repo_add_solv(repo, fp_cache, 0)) {
- g_set_error (error,
- DNF_ERROR,
- DNF_ERROR_INTERNAL_ERROR,
- _("repo_add_solv() has failed."));
- retval = FALSE;
- goto out;
- }
repoImpl->state_main = _HY_LOADED_CACHE;
+ goto out;
+ }
+
+ if (error && *error) {
+ g_prefix_error(error, _("While loading repository failed to use %s: "), fn_cache);
+ retval = FALSE;
+ goto out;
} else {
auto primary = hrepo->getMetadataPath(MD_TYPE_PRIMARY);
if (primary.empty()) {
@@ -733,8 +793,6 @@ load_yum_repo(DnfSack *sack, HyRepo hrepo, GError **error)
repoImpl->state_main = _HY_LOADED_FETCH;
}
out:
- if (fp_cache)
- fclose(fp_cache);
if (fp_repomd)
fclose(fp_repomd);
if (fp_primary)
diff --git a/libdnf/hy-iutil-private.hpp b/libdnf/hy-iutil-private.hpp
index d498c032..efc91c63 100644
--- a/libdnf/hy-iutil-private.hpp
+++ b/libdnf/hy-iutil-private.hpp
@@ -52,9 +52,7 @@ int solv_userdata_verify(const SolvUserdata *solv_userdata, const unsigned char
/* crypto utils */
int checksum_cmp(const unsigned char *cs1, const unsigned char *cs2);
int checksum_fp(unsigned char *out, FILE *fp);
-int checksum_read(unsigned char *csout, FILE *fp);
int checksum_stat(unsigned char *out, FILE *fp);
-int checksum_write(const unsigned char *cs, FILE *fp);
int checksumt_l2h(int type);
const char *pool_checksum_str(Pool *pool, const unsigned char *chksum);
diff --git a/libdnf/hy-iutil.cpp b/libdnf/hy-iutil.cpp
index f81ca52f..c409a10a 100644
--- a/libdnf/hy-iutil.cpp
+++ b/libdnf/hy-iutil.cpp
@@ -142,17 +142,6 @@ checksum_fp(unsigned char *out, FILE *fp)
return 0;
}
-/* calls rewind(fp) before returning */
-int
-checksum_read(unsigned char *csout, FILE *fp)
-{
- if (fseek(fp, -32, SEEK_END) ||
- fread(csout, CHKSUM_BYTES, 1, fp) != 1)
- return 1;
- rewind(fp);
- return 0;
-}
-
/* does not move the fp position */
int
checksum_stat(unsigned char *out, FILE *fp)
@@ -174,15 +163,6 @@ checksum_stat(unsigned char *out, FILE *fp)
return 0;
}
-/* moves fp to the end of file */
-int checksum_write(const unsigned char *cs, FILE *fp)
-{
- if (fseek(fp, 0, SEEK_END) ||
- fwrite(cs, CHKSUM_BYTES, 1, fp) != 1)
- return 1;
- return 0;
-}
-
static std::array<char, solv_userdata_solv_toolversion_size>
get_padded_solv_toolversion()
{
--
2.31.1