From dfe7218f07ffa70b73c51c71b0f051be926b6d92 Mon Sep 17 00:00:00 2001
From: Aleš Matěj <amatej@redhat.com>
Date: Tue, 14 May 2019 16:48:13 +0200
Subject: [PATCH] Correct pkg count in headers if there were invalid pkgs (RhBug:1596211)
---
src/createrepo_c.c | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------
src/dumper_thread.c | 4 +++-
src/dumper_thread.h | 3 ++-
src/threads.c | 23 +++++++++++++++++++++++
src/threads.h | 5 +++++
src/xml_file.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
src/xml_file.h | 15 +++++++++++++++
7 files changed, 260 insertions(+), 16 deletions(-)
diff --git a/src/createrepo_c.c b/src/createrepo_c.c
index e16ae34..67c2752 100644
--- a/src/createrepo_c.c
+++ b/src/createrepo_c.c
@@ -124,7 +124,7 @@ fill_pool(GThreadPool *pool,
struct CmdOptions *cmd_options,
GSList **current_pkglist,
FILE *output_pkg_list,
- long *package_count,
+ long *task_count,
int media_id)
{
GQueue queue = G_QUEUE_INIT;
@@ -259,13 +259,13 @@ fill_pool(GThreadPool *pool,
// Push sorted tasks into the thread pool
while ((task = g_queue_pop_head(&queue)) != NULL) {
- task->id = *package_count;
+ task->id = *task_count;
task->media_id = media_id;
g_thread_pool_push(pool, task, NULL);
- ++*package_count;
+ ++*task_count;
}
- return *package_count;
+ return *task_count;
}
@@ -321,6 +321,27 @@ prepare_cache_dir(struct CmdOptions *cmd_options,
return TRUE;
}
+/** Check if task finished without error, if yes
+ * use content stats of the new file
+ *
+ * @param task Rewrite pkg count task
+ * @param filename Name of file with wrong package count
+ * @param exit_val If errors occured set createrepo_c exit value
+ * @param content_stat Content stats for filename
+ *
+ */
+static void
+error_check_and_set_content_stat(cr_CompressionTask *task, char *filename, int *exit_val, cr_ContentStat **content_stat){
+ if (task->err) {
+ g_critical("Cannot rewrite pkg count in %s: %s",
+ filename, task->err->message);
+ *exit_val = 2;
+ }else{
+ cr_contentstat_free(*content_stat, NULL);
+ *content_stat = task->stat;
+ task->stat = NULL;
+ }
+}
int
main(int argc, char **argv)
@@ -478,7 +499,7 @@ main(int argc, char **argv)
NULL);
g_debug("Thread pool ready");
- long package_count = 0;
+ long task_count = 0;
GSList *current_pkglist = NULL;
/* ^^^ List with basenames of files which will be processed */
@@ -490,26 +511,26 @@ main(int argc, char **argv)
cmd_options,
¤t_pkglist,
output_pkg_list,
- &package_count,
+ &task_count,
media_id);
g_free(tmp_in_dir);
}
- g_debug("Package count: %ld", package_count);
- g_message("Directory walk done - %ld packages", package_count);
+ g_debug("Package count: %ld", task_count);
+ g_message("Directory walk done - %ld packages", task_count);
if (output_pkg_list)
fclose(output_pkg_list);
// Load old metadata if --update
cr_Metadata *old_metadata = NULL;
struct cr_MetadataLocation *old_metadata_location = NULL;
- if (!package_count)
+ if (!task_count)
g_debug("No packages found - skipping metadata loading");
- if (package_count && cmd_options->update) {
+ if (task_count && cmd_options->update) {
int ret;
old_metadata = cr_metadata_new(CR_HT_KEY_FILENAME, 1, current_pkglist);
cr_metadata_set_dupaction(old_metadata, CR_HT_DUPACT_REMOVEALL);
@@ -741,9 +762,9 @@ main(int argc, char **argv)
// Set number of packages
g_debug("Setting number of packages");
- cr_xmlfile_set_num_of_pkgs(pri_cr_file, package_count, NULL);
- cr_xmlfile_set_num_of_pkgs(fil_cr_file, package_count, NULL);
- cr_xmlfile_set_num_of_pkgs(oth_cr_file, package_count, NULL);
+ cr_xmlfile_set_num_of_pkgs(pri_cr_file, task_count, NULL);
+ cr_xmlfile_set_num_of_pkgs(fil_cr_file, task_count, NULL);
+ cr_xmlfile_set_num_of_pkgs(oth_cr_file, task_count, NULL);
// Open sqlite databases
gchar *pri_db_filename = NULL;
@@ -832,7 +853,8 @@ main(int argc, char **argv)
user_data.checksum_cachedir = cmd_options->checksum_cachedir;
user_data.skip_symlinks = cmd_options->skip_symlinks;
user_data.repodir_name_len = strlen(in_dir);
- user_data.package_count = package_count;
+ user_data.task_count = task_count;
+ user_data.package_count = 0;
user_data.skip_stat = cmd_options->skip_stat;
user_data.old_metadata = old_metadata;
user_data.mutex_pri = g_mutex_new();
@@ -876,6 +898,59 @@ main(int argc, char **argv)
cr_xmlfile_close(fil_cr_file, NULL);
cr_xmlfile_close(oth_cr_file, NULL);
+
+ /* At the time of writing xml metadata headers we haven't yet parsed all
+ * the packages and we don't know whether there were some invalid ones,
+ * therefore we write the task count into the headers instead of the actual package count.
+ * If there actually were some invalid packages we have to correct this value
+ * that unfortunately means we have to decompress metadata files change package
+ * count value and compress them again.
+ */
+ if (user_data.package_count != user_data.task_count){
+ g_message("Warning: There were some invalid packages: we have to recompress other, filelists and primary xml metadata files in order to have correct package counts");
+
+ GThreadPool *rewrite_pkg_count_pool = g_thread_pool_new(cr_rewrite_pkg_count_thread,
+ &user_data, 3, FALSE, NULL);
+
+ cr_CompressionTask *pri_rewrite_pkg_count_task;
+ cr_CompressionTask *fil_rewrite_pkg_count_task;
+ cr_CompressionTask *oth_rewrite_pkg_count_task;
+
+ pri_rewrite_pkg_count_task = cr_compressiontask_new(pri_xml_filename,
+ NULL,
+ xml_compression,
+ cmd_options->repomd_checksum_type,
+ 1,
+ &tmp_err);
+ g_thread_pool_push(rewrite_pkg_count_pool, pri_rewrite_pkg_count_task, NULL);
+
+ fil_rewrite_pkg_count_task = cr_compressiontask_new(fil_xml_filename,
+ NULL,
+ xml_compression,
+ cmd_options->repomd_checksum_type,
+ 1,
+ &tmp_err);
+ g_thread_pool_push(rewrite_pkg_count_pool, fil_rewrite_pkg_count_task, NULL);
+
+ oth_rewrite_pkg_count_task = cr_compressiontask_new(oth_xml_filename,
+ NULL,
+ xml_compression,
+ cmd_options->repomd_checksum_type,
+ 1,
+ &tmp_err);
+ g_thread_pool_push(rewrite_pkg_count_pool, oth_rewrite_pkg_count_task, NULL);
+
+ g_thread_pool_free(rewrite_pkg_count_pool, FALSE, TRUE);
+
+ error_check_and_set_content_stat(pri_rewrite_pkg_count_task, pri_xml_filename, &exit_val, &pri_stat);
+ error_check_and_set_content_stat(fil_rewrite_pkg_count_task, fil_xml_filename, &exit_val, &fil_stat);
+ error_check_and_set_content_stat(oth_rewrite_pkg_count_task, oth_xml_filename, &exit_val, &oth_stat);
+
+ cr_compressiontask_free(pri_rewrite_pkg_count_task, NULL);
+ cr_compressiontask_free(fil_rewrite_pkg_count_task, NULL);
+ cr_compressiontask_free(oth_rewrite_pkg_count_task, NULL);
+ }
+
g_queue_free(user_data.buffer);
g_mutex_free(user_data.mutex_buffer);
g_cond_free(user_data.cond_pri);
diff --git a/src/dumper_thread.c b/src/dumper_thread.c
index fbaa5be..e282f96 100644
--- a/src/dumper_thread.c
+++ b/src/dumper_thread.c
@@ -74,6 +74,8 @@ write_pkg(long id,
g_mutex_lock(udata->mutex_pri);
while (udata->id_pri != id)
g_cond_wait (udata->cond_pri, udata->mutex_pri);
+
+ udata->package_count++;
++udata->id_pri;
cr_xmlfile_add_chunk(udata->pri_f, (const char *) res.primary, &tmp_err);
if (tmp_err) {
@@ -476,7 +478,7 @@ cr_dumper_thread(gpointer data, gpointer user_data)
if (g_queue_get_length(udata->buffer) < MAX_TASK_BUFFER_LEN
&& udata->id_pri != task->id
- && udata->package_count > (task->id + 1))
+ && udata->task_count > (task->id + 1))
{
// If:
// * this isn't our turn
diff --git a/src/dumper_thread.h b/src/dumper_thread.h
index ed21053..4e18869 100644
--- a/src/dumper_thread.h
+++ b/src/dumper_thread.h
@@ -61,7 +61,8 @@ struct UserData {
cr_ChecksumType checksum_type; // Constant representing selected checksum
const char *checksum_cachedir; // Dir with cached checksums
gboolean skip_symlinks; // Skip symlinks
- long package_count; // Total number of packages to process
+ long task_count; // Total number of task to process
+ long package_count; // Total number of packages processed
// Update stuff
gboolean skip_stat; // Skip stat() while updating
diff --git a/src/threads.c b/src/threads.c
index aee07d1..844e900 100644
--- a/src/threads.c
+++ b/src/threads.c
@@ -21,6 +21,7 @@
#include "threads.h"
#include "error.h"
#include "misc.h"
+#include "dumper_thread.h"
#define ERR_DOMAIN CREATEREPO_C_ERROR
@@ -108,6 +109,28 @@ cr_compressing_thread(gpointer data, G_GNUC_UNUSED gpointer user_data)
}
}
+void
+cr_rewrite_pkg_count_thread(gpointer data, gpointer user_data)
+{
+ cr_CompressionTask *task = data;
+ struct UserData *ud = user_data;
+ GError *tmp_err = NULL;
+
+ assert(task);
+
+ cr_rewrite_header_package_count(task->src,
+ task->type,
+ ud->package_count,
+ ud->task_count,
+ task->stat,
+ &tmp_err);
+
+ if (tmp_err) {
+ // Error encountered
+ g_propagate_error(&task->err, tmp_err);
+ }
+}
+
/** Parallel Repomd Record Fill */
cr_RepomdRecordFillTask *
diff --git a/src/threads.h b/src/threads.h
index 2d554cd..19ba917 100644
--- a/src/threads.h
+++ b/src/threads.h
@@ -150,6 +150,11 @@ cr_repomdrecordfilltask_free(cr_RepomdRecordFillTask *task, GError **err);
void
cr_repomd_record_fill_thread(gpointer data, gpointer user_data);
+/** Function for GThread Pool.
+ */
+void
+cr_rewrite_pkg_count_thread(gpointer data, gpointer user_data);
+
/** @} */
#ifdef __cplusplus
diff --git a/src/xml_file.c b/src/xml_file.c
index 65fb945..1d670ae 100644
--- a/src/xml_file.c
+++ b/src/xml_file.c
@@ -18,8 +18,10 @@
*/
#include <glib.h>
+#include <glib/gstdio.h>
#include <assert.h>
#include "xml_file.h"
+#include <errno.h>
#include "error.h"
#include "xml_dump.h"
#include "compression_wrapper.h"
@@ -40,6 +42,9 @@
#define XML_PRESTODELTA_HEADER XML_HEADER"<prestodelta>\n"
#define XML_UPDATEINFO_HEADER XML_HEADER"<updates>\n"
+#define XML_MAX_HEADER_SIZE 300
+#define XML_RECOMPRESS_BUFFER_SIZE 8192
+
#define XML_PRIMARY_FOOTER "</metadata>"
#define XML_FILELISTS_FOOTER "</filelists>"
#define XML_OTHER_FOOTER "</otherdata>"
@@ -317,3 +322,121 @@ cr_xmlfile_close(cr_XmlFile *f, GError **err)
return CRE_OK;
}
+
+static int
+write_modified_header(int task_count,
+ int package_count,
+ cr_XmlFile *cr_file,
+ gchar *header_buf,
+ int header_len,
+ GError **err)
+{
+ GError *tmp_err = NULL;
+ gchar *package_count_string;
+ gchar *task_count_string;
+ int bytes_written = 0;
+ int package_count_string_len = rasprintf(&package_count_string, "packages=\"%i\"", package_count);
+ int task_count_string_len = rasprintf(&task_count_string, "packages=\"%i\"", task_count);
+
+ gchar *pointer_to_pkgs = strstr(header_buf, task_count_string);
+ if (!pointer_to_pkgs){
+ g_free(package_count_string);
+ g_free(task_count_string);
+ return 0;
+ }
+ gchar *pointer_to_pkgs_end = pointer_to_pkgs + task_count_string_len;
+
+ bytes_written += cr_write(cr_file->f, header_buf, pointer_to_pkgs - header_buf, &tmp_err);
+ if (!tmp_err)
+ bytes_written += cr_write(cr_file->f, package_count_string, package_count_string_len, &tmp_err);
+ if (!tmp_err)
+ bytes_written += cr_write(cr_file->f, pointer_to_pkgs_end, header_len - (pointer_to_pkgs_end - header_buf), &tmp_err);
+ if (tmp_err) {
+ g_propagate_prefixed_error(err, tmp_err, "Error encountered while writing header part:");
+ g_free(package_count_string);
+ g_free(task_count_string);
+ return 0;
+ }
+ g_free(package_count_string);
+ g_free(task_count_string);
+ return bytes_written;
+}
+
+void
+cr_rewrite_header_package_count(gchar *original_filename,
+ cr_CompressionType xml_compression,
+ int package_count,
+ int task_count,
+ cr_ContentStat *file_stat,
+ GError **err)
+{
+ GError *tmp_err = NULL;
+ CR_FILE *original_file = cr_open(original_filename, CR_CW_MODE_READ, CR_CW_AUTO_DETECT_COMPRESSION, &tmp_err);
+ if (tmp_err) {
+ g_propagate_prefixed_error(err, tmp_err, "Error encountered while reopening for reading:");
+ return;
+ }
+
+ gchar *tmp_xml_filename = g_strconcat(original_filename, ".tmp", NULL);
+ cr_XmlFile *new_file = cr_xmlfile_sopen_primary(tmp_xml_filename,
+ xml_compression,
+ file_stat,
+ &tmp_err);
+ if (tmp_err) {
+ g_propagate_prefixed_error(err, tmp_err, "Error encountered while opening for writing:");
+ cr_close(original_file, NULL);
+ g_free(tmp_xml_filename);
+ return;
+ }
+
+ gchar header_buf[XML_MAX_HEADER_SIZE];
+ int len_read = cr_read(original_file, header_buf, XML_MAX_HEADER_SIZE, &tmp_err);
+ if (!tmp_err)
+ write_modified_header(task_count, package_count, new_file, header_buf, len_read, &tmp_err);
+ if (tmp_err) {
+ g_propagate_prefixed_error(err, tmp_err, "Error encountered while recompressing:");
+ cr_xmlfile_close(new_file, NULL);
+ cr_close(original_file, NULL);
+ g_free(tmp_xml_filename);
+ return;
+ }
+ //Copy the rest of the file
+ gchar copy_buf[XML_RECOMPRESS_BUFFER_SIZE];
+ while(len_read)
+ {
+ len_read = cr_read(original_file, copy_buf, XML_RECOMPRESS_BUFFER_SIZE, &tmp_err);
+ if (!tmp_err)
+ cr_write(new_file->f, copy_buf, len_read, &tmp_err);
+ if (tmp_err) {
+ g_propagate_prefixed_error(err, tmp_err, "Error encountered while recompressing:");
+ cr_xmlfile_close(new_file, NULL);
+ cr_close(original_file, NULL);
+ g_free(tmp_xml_filename);
+ return;
+ }
+ }
+
+ new_file->header = 1;
+ new_file->footer = 1;
+
+ cr_xmlfile_close(new_file, &tmp_err);
+ if (tmp_err) {
+ g_propagate_prefixed_error(err, tmp_err, "Error encountered while writing:");
+ cr_close(original_file, NULL);
+ g_free(tmp_xml_filename);
+ return;
+ }
+ cr_close(original_file, &tmp_err);
+ if (tmp_err) {
+ g_propagate_prefixed_error(err, tmp_err, "Error encountered while writing:");
+ g_free(tmp_xml_filename);
+ return;
+ }
+
+ if (g_rename(tmp_xml_filename, original_filename) == -1) {
+ g_propagate_prefixed_error(err, tmp_err, "Error encountered while renaming:");
+ g_free(tmp_xml_filename);
+ return;
+ }
+ g_free(tmp_xml_filename);
+}
diff --git a/src/xml_file.h b/src/xml_file.h
index 96ef5e3..6ac4c97 100644
--- a/src/xml_file.h
+++ b/src/xml_file.h
@@ -221,6 +221,21 @@ int cr_xmlfile_add_chunk(cr_XmlFile *f, const char *chunk, GError **err);
*/
int cr_xmlfile_close(cr_XmlFile *f, GError **err);
+/** Rewrite package count field in repodata header in xml file.
+ * In order to do this we have to decompress and after the change
+ * compress the whole file again, so entirely new file is created.
+ * @param original_filename Current file with wrong value in header
+ * @param package_count Actual package count (desired value in header)
+ * @param task_count Task count (current value in header)
+ * @param file_stat cr_ContentStat for stats of the new file, it will be modified
+ * @param err **GError
+ */
+void cr_rewrite_header_package_count(gchar *original_filename,
+ cr_CompressionType xml_compression,
+ int package_count,
+ int task_count,
+ cr_ContentStat *file_stat,
+ GError **err);
/** @} */
--
libgit2 0.27.8