From af2e7fb6a5790aa6c35a2d283c3d9388565a3702 Mon Sep 17 00:00:00 2001 From: Matteo Croce Date: Dec 17 2024 11:05:23 +0000 Subject: add CoW Port RPM CoW on top of latest C10s tree. Port measure plugin as well. --- diff --git a/0001-Measure-plugin.patch b/0001-Measure-plugin.patch new file mode 100644 index 0000000..aa804c8 --- /dev/null +++ b/0001-Measure-plugin.patch @@ -0,0 +1,277 @@ +From cf758158efa2832b38a49770f3489f408cb93b41 Mon Sep 17 00:00:00 2001 +From: Matthew Almond +Date: Fri, 6 Dec 2024 08:00:31 +0100 +Subject: [PATCH] Measure plugin + +--- + macros.in | 1 + + plugins/CMakeLists.txt | 1 + + plugins/measure.c | 232 +++++++++++++++++++++++++++++++++++++++++ + 3 files changed, 234 insertions(+) + create mode 100644 plugins/measure.c + +diff --git a/macros.in b/macros.in +index df7defe..1af187a 100644 +--- a/macros.in ++++ b/macros.in +@@ -1189,6 +1189,7 @@ Supplements: (%{name} = %{version}-%{release} and langpacks-%{1})\ + # Transaction plugin macros + %__plugindir %{_libdir}/rpm-plugins + %__transaction_reflink %{__plugindir}/reflink.so ++%__transaction_measure %{__plugindir}/measure.so + %__transaction_systemd_inhibit %{__plugindir}/systemd_inhibit.so + %__transaction_selinux %{__plugindir}/selinux.so + %__transaction_syslog %{__plugindir}/syslog.so +diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt +index ed63a43..cda8f96 100644 +--- a/plugins/CMakeLists.txt ++++ b/plugins/CMakeLists.txt +@@ -1,6 +1,7 @@ + add_library(prioreset MODULE prioreset.c) + add_library(syslog MODULE syslog.c) + add_library(reflink MODULE reflink.c) ++add_library(measure MODULE measure.c) + + if(WITH_SELINUX) + add_library(selinux MODULE selinux.c) +diff --git a/plugins/measure.c b/plugins/measure.c +new file mode 100644 +index 0000000..593e463 +--- /dev/null ++++ b/plugins/measure.c +@@ -0,0 +1,232 @@ ++#include "system.h" ++#include "time.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "rpmplugin.h" ++ ++struct measurestat { ++ /* We're counting psm not packages because packages often run psm_pre/post ++ more than once and we want to accumulate the time ++ */ ++ unsigned int psm_count; ++ unsigned int scriptlet_count; ++ struct timespec plugin_start; ++ struct timespec psm_start; ++ struct timespec scriptlet_start; ++}; ++ ++static rpmRC push(const char *format, const char *value, const char *formatted) ++{ ++ char *key = NULL; ++ rpmRC rc = RPMRC_FAIL; ++ if (formatted == NULL) { ++ /* yes we're okay with discarding const here */ ++ key = (char *)format; ++ } else { ++ if (rasprintf(&key, format, formatted) == -1) { ++ rpmlog( ++ RPMLOG_ERR, ++ _("measure: Failed to allocate formatted key %s, %s\n"), ++ format, ++ formatted ++ ); ++ goto exit; ++ } ++ } ++ if (rpmPushMacro(NULL, key, NULL, value, RMIL_GLOBAL)) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to set %s\n"), key); ++ goto exit; ++ } ++ rc = RPMRC_OK; ++exit: ++ if (formatted != NULL) { ++ free(key); ++ } ++ return rc; ++} ++ ++static rpmRC diff_ms(char **ms, struct timespec *start, struct timespec *end) ++{ ++ if (rasprintf(ms, "%ld", ( ++ (end->tv_sec - start->tv_sec) * 1000 + ++ (end->tv_nsec - start->tv_nsec) / 1000000 ++ )) == -1) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to allocate formatted ms\n")); ++ return RPMRC_FAIL; ++ } ++ return RPMRC_OK; ++} ++ ++static rpmRC measure_init(rpmPlugin plugin, rpmts ts) ++{ ++ rpmRC rc = RPMRC_FAIL; ++ struct measurestat *state = rcalloc(1, sizeof(*state)); ++ if (clock_gettime(CLOCK_MONOTONIC, &state->plugin_start)) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to get plugin_start time\n")); ++ goto exit; ++ } ++ state->psm_count = 0; ++ state->scriptlet_count = 0; ++ rpmPluginSetData(plugin, state); ++ rc = RPMRC_OK; ++exit: ++ return rc; ++} ++ ++static void measure_cleanup(rpmPlugin plugin) ++{ ++ struct measurestat *state = rpmPluginGetData(plugin); ++ free(state); ++} ++ ++static rpmRC measure_tsm_post(rpmPlugin plugin, rpmts ts, int res) ++{ ++ struct measurestat *state = rpmPluginGetData(plugin); ++ char *psm_count = NULL, *scriptlet_count = NULL; ++ rpmRC rc = RPMRC_FAIL; ++ ++ if (rasprintf(&psm_count, "%d", state->psm_count) == -1) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to allocate formatted psm_count\n")); ++ goto exit; ++ } ++ if (rasprintf(&scriptlet_count, "%d", state->scriptlet_count) == -1) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to allocate formatted scriptlet_count\n")); ++ goto exit; ++ } ++ if (push("_measure_plugin_psm_count", psm_count, NULL) != RPMRC_OK) { ++ goto exit; ++ } ++ if (push("_measure_plugin_scriptlet_count", scriptlet_count, NULL) != RPMRC_OK) { ++ goto exit; ++ } ++ rc = RPMRC_OK; ++exit: ++ free(psm_count); ++ free(scriptlet_count); ++ return rc; ++} ++ ++static rpmRC measure_psm_pre(rpmPlugin plugin, rpmte te) ++{ ++ struct measurestat *state = rpmPluginGetData(plugin); ++ rpmRC rc = RPMRC_FAIL; ++ ++ if (clock_gettime(CLOCK_MONOTONIC, &state->psm_start)) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to get psm_start time\n")); ++ goto exit; ++ } ++ rc = RPMRC_OK; ++exit: ++ return rc; ++} ++ ++static rpmRC measure_psm_post(rpmPlugin plugin, rpmte te, int res) ++{ ++ struct measurestat *state = rpmPluginGetData(plugin); ++ struct timespec end; ++ char *offset = NULL, *duration = NULL, *prefix = NULL; ++ Header h = rpmteHeader(te); ++ rpmRC rc = RPMRC_FAIL; ++ ++ if (clock_gettime(CLOCK_MONOTONIC, &end)) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to get psm end time\n")); ++ goto exit; ++ } ++ if (rasprintf(&prefix, "_measure_plugin_package_%u", state->psm_count) == -1) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to allocate prefix\n")); ++ goto exit; ++ } ++ if (diff_ms(&offset, &state->plugin_start, &state->psm_start) != RPMRC_OK) { ++ goto exit; ++ } ++ if (diff_ms(&duration, &state->psm_start, &end) != RPMRC_OK) { ++ goto exit; ++ } ++ if (push("%s_nevra", rpmteNEVRA(te), prefix) != RPMRC_OK) { ++ goto exit; ++ } ++ if (push("%s_compressor", headerGetString(h, RPMTAG_PAYLOADCOMPRESSOR), prefix) != RPMRC_OK) { ++ goto exit; ++ } ++ if (push("%s_offset", offset, prefix) != RPMRC_OK) { ++ goto exit; ++ } ++ if (push("%s_ms", duration, prefix) != RPMRC_OK) { ++ goto exit; ++ } ++ state->psm_count += 1; ++ rc = RPMRC_OK; ++exit: ++ headerFree(h); ++ free(prefix); ++ free(duration); ++ free(offset); ++ return rc; ++} ++ ++static rpmRC measure_scriptlet_pre(rpmPlugin plugin, ++ const char *s_name, int type) ++{ ++ struct measurestat *state = rpmPluginGetData(plugin); ++ if (clock_gettime(CLOCK_MONOTONIC, &state->scriptlet_start)) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to get scriptlet_start time\n")); ++ return RPMRC_FAIL; ++ } ++ return RPMRC_OK; ++} ++ ++static rpmRC measure_scriptlet_post(rpmPlugin plugin, ++ const char *s_name, int type, int res) ++{ ++ struct measurestat *state = rpmPluginGetData(plugin); ++ struct timespec end; ++ char *offset = NULL, *duration = NULL, *prefix = NULL; ++ rpmRC rc = RPMRC_FAIL; ++ ++ if (clock_gettime(CLOCK_MONOTONIC, &end)) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to get end time\n")); ++ goto exit; ++ } ++ ++ if (rasprintf(&prefix, "_measure_plugin_scriptlet_%d", state->scriptlet_count) == -1) { ++ rpmlog(RPMLOG_ERR, _("measure: Failed to allocate formatted prefix\n")); ++ goto exit; ++ } ++ if (diff_ms(&offset, &state->plugin_start, &state->scriptlet_start) != RPMRC_OK) { ++ goto exit; ++ } ++ if (diff_ms(&duration, &state->scriptlet_start, &end) != RPMRC_OK) { ++ goto exit; ++ } ++ if (push("%s_name", s_name, prefix) != RPMRC_OK) { ++ goto exit; ++ } ++ if (push("%s_offset", offset, prefix) != RPMRC_OK) { ++ goto exit; ++ } ++ if (push("%s_ms", duration, prefix) != RPMRC_OK) { ++ goto exit; ++ } ++ state->scriptlet_count += 1; ++ rc = RPMRC_OK; ++exit: ++ free(prefix); ++ free(duration); ++ free(offset); ++ return rc; ++} ++ ++struct rpmPluginHooks_s measure_hooks = { ++ .init = measure_init, ++ .cleanup = measure_cleanup, ++ .tsm_post = measure_tsm_post, ++ .psm_pre = measure_psm_pre, ++ .psm_post = measure_psm_post, ++ .scriptlet_pre = measure_scriptlet_pre, ++ .scriptlet_post = measure_scriptlet_post, ++}; +-- +2.47.1 + diff --git a/0001-RPM-with-Copy-on-Write.patch b/0001-RPM-with-Copy-on-Write.patch new file mode 100644 index 0000000..cfa7b11 --- /dev/null +++ b/0001-RPM-with-Copy-on-Write.patch @@ -0,0 +1,2698 @@ +From e38a26f5d6050a5c72f7503c8664ca11623ed319 Mon Sep 17 00:00:00 2001 +From: Matteo Croce +Date: Fri, 6 Dec 2024 07:45:17 +0100 +Subject: [PATCH] RPM with Copy on Write + +This is part of https://fedoraproject.org/wiki/Changes/RPMCoW + +The majority of changes are in two new programs: + += rpm2extents + +Modeled as a 'stream processor'. It reads a regular .rpm file on stdin, +and produces a modified .rpm file on stdout. The lead, signature and +headers are preserved 1:1 to allow all the normal metadata inspection, +signature verification to work as expected. Only the 'payload' is +modified. + +The primary motivation for this tool is to re-organize the payload as a +sequence of raw file extents (hence the name). The files are organized +by their digest identity instead of path/filename. If any digest is +repeated, then the file is skipped/de-duped. Only regular files are +represented. All other entries like directories, symlinks, devices are +fully described in the headers and are omitted. + +The files are padded so they start on `sysconf(_SC_PAGESIZE)` boundries +to permit 'reflink' syscalls to work in the `reflink` plugin. + +At the end of the file is a footer with 3 sections: + +1. List of calculated digests of the input stream. This is used in + `librepo` because the file *written* is a derivative, and not the + same as the repo metadata describes. `rpm2extents` takes one or more + positional arguments that described which digest algorithms are + desired. This is often just `SHA256`. This program is only measuring + and recording the digest - it does not express an opinion on whether + the file is correct. Due to the API on most compression libraries + directly reading the source file, the whole file digest is measured + using a subprocess and pipes. I don't love it, but it works. +2. Sorted List of file content digests + offset pairs. This is used in + the plugin with a trivial binary search to locate the start of file + content. The size is not needed because it's part of normal headers. +3. (offset of 1., offset of 2., 8 byte MAGIC value) triple + += reflink plugin + +Looks for the 8 byte magic value at the end of the rpm file. If present +it alters the `RPMTAG_PAYLOADFORMAT` in memory to `clon`, and reads in +the digest-> offset table. + +`rpmPackageFilesInstall()` in `fsm.c` is +modified to alter the enumeration strategy from +`rpmfiNewArchiveReader()` to `rpmfilesIter()` if not `cpio`. This is +needed because there is no cpio to enumerate. In the same function, if +`rpmpluginsCallFsmFilePre()` returns `RPMRC_PLUGIN_CONTENTS` then +`fsmMkfile()` is skipped as it is assumed the plugin did the work. + +The majority of the work is in `reflink_fsm_file_pre()` - the per file +hook for RPM plugins. If the file enumerated in +`rpmPackageFilesInstall()` is a regular file, this function will look up +the offset in the digest->offset table and will try to reflink it, then +fall back to a regular copy. If reflinking does work: we will have +reflinked a whole number of pages, so we truncate the file to the +expected size. Therefore installing most files does involve two writes: +the reflink of the full size, then a fork/copy on write for the last +page worth. + +If the file passed to `reflink_fsm_file_pre()` is anything other than a +regular file, it return `RPMRC_OK` so the normal mechanics of +`rpmPackageFilesInstall()` are used. That handles directories, symlinks +and other non file types. + += New API for internal use + +1. `rpmReadPackageRaw()` is used within `rpm2extents` to read all the + headers without trying to validate signatures. This eliminates the + runtime dependency on rpmdb. +2. `rpmteFd()` exposes the Fd behind the rpmte, so plugins can interact + with the rpm itself. +3. `RPMRC_PLUGIN_CONTENTS` in `rpmRC_e` for use in + `rpmpluginsCallFsmFilePre()` specifically. +4. `pgpStringVal()` is used to help parse the command line in + `rpm2extents` - the positional arguments are strings, and this + converts the values back to the values in the table. + +Nothing has been removed, and none of the changes are intended to be +used externally, so I don't think a soname bump is warranted here. + +Co-authored-by: Matthew Almond +--- + build/pack.c | 2 +- + include/rpm/rpmcli.h | 10 + + include/rpm/rpmextents_internal.h | 58 +++ + include/rpm/rpmlib.h | 9 + + include/rpm/rpmpgp.h | 9 + + include/rpm/rpmte.h | 2 + + include/rpm/rpmtypes.h | 3 +- + lib/CMakeLists.txt | 1 + + lib/fsm.c | 45 +- + lib/package.c | 36 ++ + lib/rpmchecksig.c | 116 ++++- + lib/rpmextents.c | 109 +++++ + lib/rpmlead.c | 43 +- + lib/rpmlead.h | 37 +- + lib/rpmplugin.h | 9 + + lib/rpmplugins.c | 94 +++- + lib/rpmplugins.h | 17 + + lib/rpmte.c | 5 + + lib/transaction.c | 29 +- + macros.in | 1 + + plugins/CMakeLists.txt | 4 +- + plugins/reflink.c | 401 +++++++++++++++++ + rpmio/rpmpgp.c | 28 ++ + rpmio/rpmpgp_internal.c | 18 - + scripts/CMakeLists.txt | 2 +- + scripts/rpm2extents_dump | 94 ++++ + tests/CMakeLists.txt | 1 + + tests/atlocal.in | 21 + + tests/rpm2extents.at | 151 +++++++ + tests/rpmtests.at | 1 + + tools/CMakeLists.txt | 5 +- + tools/rpm2extents.c | 707 ++++++++++++++++++++++++++++++ + 32 files changed, 1978 insertions(+), 90 deletions(-) + create mode 100644 include/rpm/rpmextents_internal.h + create mode 100644 lib/rpmextents.c + create mode 100644 plugins/reflink.c + create mode 100755 scripts/rpm2extents_dump + create mode 100644 tests/rpm2extents.at + create mode 100644 tools/rpm2extents.c + +diff --git a/build/pack.c b/build/pack.c +index f7dac6d..c38b1a9 100644 +--- a/build/pack.c ++++ b/build/pack.c +@@ -495,7 +495,7 @@ static rpmRC writeRPM(Package pkg, unsigned char ** pkgidp, + } + + /* Write the lead section into the package. */ +- if (rpmLeadWrite(fd, pkg->header)) { ++ if (rpmLeadWriteFromHeader(fd, pkg->header)) { + rpmlog(RPMLOG_ERR, _("Unable to write package: %s\n"), Fstrerror(fd)); + goto exit; + } +diff --git a/include/rpm/rpmcli.h b/include/rpm/rpmcli.h +index 3e5900d..09274e3 100644 +--- a/include/rpm/rpmcli.h ++++ b/include/rpm/rpmcli.h +@@ -421,6 +421,16 @@ int rpmcliImportPubkeys(rpmts ts, ARGV_const_t argv); + */ + int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv); + ++ ++/** \ingroup rpmcli ++ * Verify package signatures. ++ * @param ts transaction set ++ * @param fd a file descriptor to verify ++ * @param msg a string containing textual information about the verification, similar to rpmcliVerifySignatures output. ++ * @return 0 on success ++ */ ++int rpmcliVerifySignaturesFD(rpmts ts, FD_t fd, char **msg); ++ + #ifdef __cplusplus + } + #endif +diff --git a/include/rpm/rpmextents_internal.h b/include/rpm/rpmextents_internal.h +new file mode 100644 +index 0000000..0a3318c +--- /dev/null ++++ b/include/rpm/rpmextents_internal.h +@@ -0,0 +1,58 @@ ++#ifndef _RPMEXTENTS_INTERNAL_H ++#define _RPMEXTENTS_INTERNAL_H ++ ++#ifdef __cplusplus ++extern "C" { ++#endif ++ ++#include ++ ++/** \ingroup rpmextents ++ * RPM extents library ++ */ ++ ++/* magic value at end of file (64 bits) that indicates this is a transcoded ++ * rpm. ++ */ ++#define EXTENTS_MAGIC 3472329499408095051 ++ ++typedef uint64_t extents_magic_t; ++ ++struct __attribute__ ((__packed__)) extents_footer_offsets_t { ++ off64_t checksig_offset; ++ off64_t table_offset; ++ off64_t csum_offset; ++}; ++ ++struct __attribute__ ((__packed__)) extents_footer_t { ++ struct extents_footer_offsets_t offsets; ++ extents_magic_t magic; ++}; ++ ++/** \ingroup rpmextents ++ * Checks the results of the signature verification ran during transcoding. ++ * @param fd The FD_t of the transcoded RPM ++ * @param print_content Whether or not to print the result from rpmsig ++ * @return The number of checks that `rpmvsVerify` failed during transcoding. ++ */ ++int extentsVerifySigs(FD_t fd, int print_content); ++ ++/** \ingroup rpmextents ++ * Read the RPM Extents footer from a file descriptor. ++ * @param fd The FD_t of the transcoded RPM ++ * @param[out] footer A pointer to an allocated extents_footer_t with a copy of the footer. ++ * @return RPMRC_OK on success, RPMRC_NOTFOUND if not a transcoded file, RPMRC_FAIL on any failure. ++ */ ++rpmRC extentsFooterFromFD(FD_t fd, struct extents_footer_t *footer); ++ ++/** \ingroup rpmextents ++ * Check if a RPM is a transcoded RPM ++ * @param fd The FD_t of the transcoded RPM ++ * return RPMRC_OK on success, RPMRC_NOTFOUND if not a transcoded file, RPMRC_FAIL on any failure. ++ */ ++rpmRC isTranscodedRpm(FD_t fd); ++ ++#ifdef __cplusplus ++} ++#endif ++#endif /* _RPMEXTENTS_INTERNAL_H */ +diff --git a/include/rpm/rpmlib.h b/include/rpm/rpmlib.h +index db38397..b11f87f 100644 +--- a/include/rpm/rpmlib.h ++++ b/include/rpm/rpmlib.h +@@ -155,6 +155,15 @@ rpmRC rpmReadHeader(rpmts ts, FD_t fd, Header *hdrp, char ** msg); + rpmRC rpmReadPackageFile(rpmts ts, FD_t fd, + const char * fn, Header * hdrp); + ++/** \ingroup header ++ * Return package signature, header from file handle, no verification. ++ * @param fd file handle ++ * @param[out] sigp address of header (or NULL) ++ * @param[out] hdrp address of header (or NULL) ++ * @return RPMRC_OK on success ++ */ ++rpmRC rpmReadPackageRaw(FD_t fd, Header * sigp, Header * hdrp); ++ + /** \ingroup rpmtrans + * Install source package. + * @param ts transaction set +diff --git a/include/rpm/rpmpgp.h b/include/rpm/rpmpgp.h +index a7eecbe..24e9617 100644 +--- a/include/rpm/rpmpgp.h ++++ b/include/rpm/rpmpgp.h +@@ -962,6 +962,15 @@ typedef enum pgpValType_e { + */ + const char * pgpValString(pgpValType type, uint8_t val); + ++/** \ingroup rpmpgp ++ * Return OpenPGP value for a string. ++ * @param type type of value ++ * @param str string to lookup ++ * @param[out] val byte value associated with string ++ * @return 0 on success else -1 ++ */ ++int pgpStringVal(pgpValType type, const char *str, uint8_t *val); ++ + /** \ingroup rpmpgp + * Return (native-endian) integer from big-endian representation. + * @param s pointer to big-endian integer +diff --git a/include/rpm/rpmte.h b/include/rpm/rpmte.h +index effcdab..499fd4f 100644 +--- a/include/rpm/rpmte.h ++++ b/include/rpm/rpmte.h +@@ -209,6 +209,8 @@ const char * rpmteNEVR(rpmte te); + */ + const char * rpmteNEVRA(rpmte te); + ++FD_t rpmteFd(rpmte te); ++ + /** \ingroup rpmte + * Retrieve key from transaction element. + * @param te transaction element +diff --git a/include/rpm/rpmtypes.h b/include/rpm/rpmtypes.h +index 7c14553..552ec52 100644 +--- a/include/rpm/rpmtypes.h ++++ b/include/rpm/rpmtypes.h +@@ -106,7 +106,8 @@ typedef enum rpmRC_e { + RPMRC_NOTFOUND = 1, /*!< Generic not found code. */ + RPMRC_FAIL = 2, /*!< Generic failure code. */ + RPMRC_NOTTRUSTED = 3, /*!< Signature is OK, but key is not trusted. */ +- RPMRC_NOKEY = 4 /*!< Public key is unavailable. */ ++ RPMRC_NOKEY = 4, /*!< Public key is unavailable. */ ++ RPMRC_PLUGIN_CONTENTS = 5 /*!< fsm_file_pre plugin is handling content */ + } rpmRC; + + #ifdef __cplusplus +diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt +index 2528a64..754a198 100644 +--- a/lib/CMakeLists.txt ++++ b/lib/CMakeLists.txt +@@ -41,6 +41,7 @@ target_sources(librpm PRIVATE + rpmchroot.c rpmchroot.h + rpmplugins.c rpmplugins.h rpmplugin.h rpmug.c rpmug.h + rpmtriggers.h rpmtriggers.c rpmvs.c rpmvs.h ++ rpmextents.c + ) + + if(ENABLE_SQLITE) +diff --git a/lib/fsm.c b/lib/fsm.c +index 36708ac..2df7db7 100644 +--- a/lib/fsm.c ++++ b/lib/fsm.c +@@ -869,6 +869,24 @@ static rpmfi fsmIterFini(rpmfi fi, struct diriter_s *di) + return rpmfiFree(fi); + } + ++static int fiIterator(rpmPlugins plugins, FD_t payload, rpmfiles files, rpmfi *fi) ++{ ++ rpmRC plugin_rc = rpmpluginsCallFsmFileArchiveReader(plugins, payload, files, fi); ++ switch (plugin_rc) { ++ case RPMRC_PLUGIN_CONTENTS: ++ if (*fi == NULL) ++ return RPMERR_BAD_MAGIC; ++ return RPMRC_OK; ++ case RPMRC_OK: ++ *fi = rpmfiNewArchiveReader(payload, files, RPMFI_ITER_READ_ARCHIVE); ++ if (*fi == NULL) ++ return RPMERR_BAD_MAGIC; ++ return RPMRC_OK; ++ default: ++ return RPMRC_FAIL; ++ } ++} ++ + int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files, + rpmpsm psm, char ** failedFile) + { +@@ -920,8 +938,9 @@ int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files, + if (rc) + goto exit; + +- fi = fsmIter(payload, files, +- payload ? RPMFI_ITER_READ_ARCHIVE : RPMFI_ITER_FWD, &di); ++ rc = fiIterator(plugins, payload, files, &fi); ++ if (rc) ++ goto exit; + + if (fi == NULL) { + rc = RPMERR_BAD_MAGIC; +@@ -944,6 +963,9 @@ int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files, + if (!fp->skip) { + int mayopen = 0; + int fd = -1; ++ ++ if (di.dirfd >= 0) ++ fsmClose(&di.dirfd); + rc = ensureDir(plugins, rpmfiDN(fi), 0, + (fp->action == FA_CREATE), 0, &di.dirfd); + +@@ -953,9 +975,10 @@ int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files, + } + + /* Run fsm file pre hook for all plugins */ +- if (!rc) ++ if (!rc) { + rc = rpmpluginsCallFsmFilePre(plugins, fi, fp->fpath, + fp->sb.st_mode, fp->action); ++ } + if (rc) + goto setmeta; /* for error notification */ + +@@ -983,11 +1006,18 @@ int rpmPackageFilesInstall(rpmts ts, rpmte te, rpmfiles files, + if (fp->action == FA_TOUCH) + goto setmeta; + +- if (S_ISREG(fp->sb.st_mode)) { ++ rpmRC plugin_rc = rpmpluginsCallFsmFileInstall(plugins, fi, fp->fpath, fp->sb.st_mode, fp->action); ++ if (!(plugin_rc == RPMRC_PLUGIN_CONTENTS || plugin_rc == RPMRC_OK)){ ++ rc = plugin_rc; ++ } else if(plugin_rc == RPMRC_PLUGIN_CONTENTS){ ++ rc = RPMRC_OK; ++ /* The reflink plugins handles hardlink differently, metadata has to be set. */ ++ fp->setmeta = 1; ++ } else if (S_ISREG(fp->sb.st_mode)) { + if (rc == RPMERR_ENOENT) { + rc = fsmMkfile(di.dirfd, fi, fp, files, psm, nodigest, +- &firstlink, &firstlinkfile, &di.firstdir, +- &fd); ++ &firstlink, &firstlinkfile, ++ &di.firstdir, &fd); + } + } else if (S_ISDIR(fp->sb.st_mode)) { + if (rc == RPMERR_ENOENT) { +@@ -1056,10 +1086,13 @@ setmeta: + + /* If all went well, commit files to final destination */ + fi = fsmIter(NULL, files, RPMFI_ITER_FWD, &di); ++ + while (!rc && (fx = rpmfiNext(fi)) >= 0) { + struct filedata_s *fp = &fdata[fx]; + + if (!fp->skip) { ++ if (di.dirfd >= 0) ++ fsmClose(&di.dirfd); + if (!rc) + rc = ensureDir(NULL, rpmfiDN(fi), 0, 0, 0, &di.dirfd); + +diff --git a/lib/package.c b/lib/package.c +index 8eee368..d4256a7 100644 +--- a/lib/package.c ++++ b/lib/package.c +@@ -394,5 +394,41 @@ exit: + return rc; + } + ++rpmRC rpmReadPackageRaw(FD_t fd, Header * sigp, Header * hdrp) ++{ ++ char *msg = NULL; ++ hdrblob sigblob = hdrblobCreate(); ++ hdrblob blob = hdrblobCreate(); ++ Header h = NULL; ++ Header sigh = NULL; ++ ++ rpmRC rc = hdrblobRead(fd, 1, 0, RPMTAG_HEADERSIGNATURES, sigblob, &msg); ++ if (rc != RPMRC_OK) ++ goto exit; ++ ++ rc = hdrblobRead(fd, 1, 1, RPMTAG_HEADERIMMUTABLE, blob, &msg); ++ if (rc != RPMRC_OK) ++ goto exit; ++ ++ rc = hdrblobImport(sigblob, 0, &sigh, &msg); ++ if (rc) ++ goto exit; ++ ++ rc = hdrblobImport(blob, 0, &h, &msg); ++ if (rc) ++ goto exit; ++ ++ *sigp = headerLink(sigh); ++ *hdrp = headerLink(h); + ++exit: ++ if (rc != RPMRC_OK && msg) ++ rpmlog(RPMLOG_ERR, "%s: %s\n", Fdescr(fd), msg); ++ hdrblobFree(sigblob); ++ hdrblobFree(blob); ++ headerFree(sigh); ++ headerFree(h); ++ free(msg); + ++ return rc; ++} +diff --git a/lib/rpmchecksig.c b/lib/rpmchecksig.c +index 3a3a4bd..8257dd9 100644 +--- a/lib/rpmchecksig.c ++++ b/lib/rpmchecksig.c +@@ -16,6 +16,7 @@ + #include + #include + #include ++#include + + #include "rpmio_internal.h" /* fdSetBundle() */ + #include "rpmlead.h" +@@ -208,36 +209,24 @@ exit: + } + + static int rpmpkgVerifySigs(rpmKeyring keyring, int vfylevel, rpmVSFlags flags, +- FD_t fd, const char *fn) ++ FD_t fd, rpmsinfoCb cb, void *cbdata) + { + char *msg = NULL; +- struct vfydata_s vd = { .seen = 0, +- .bad = 0, +- .verbose = rpmIsVerbose(), +- }; + int rc; +- struct rpmvs_s *vs = rpmvsCreate(vfylevel, flags, keyring); + +- rpmlog(RPMLOG_NOTICE, "%s:%s", fn, vd.verbose ? "\n" : ""); ++ ++ if(isTranscodedRpm(fd) == RPMRC_OK){ ++ return extentsVerifySigs(fd, 1); ++ } ++ ++ struct rpmvs_s *vs = rpmvsCreate(vfylevel, flags, keyring); + + rc = rpmpkgRead(vs, fd, NULL, NULL, &msg); + + if (rc) + goto exit; + +- rc = rpmvsVerify(vs, RPMSIG_VERIFIABLE_TYPE, vfyCb, &vd); +- +- if (!vd.verbose) { +- if (vd.seen & RPMSIG_DIGEST_TYPE) { +- rpmlog(RPMLOG_NOTICE, " %s", (vd.bad & RPMSIG_DIGEST_TYPE) ? +- _("DIGESTS") : _("digests")); +- } +- if (vd.seen & RPMSIG_SIGNATURE_TYPE) { +- rpmlog(RPMLOG_NOTICE, " %s", (vd.bad & RPMSIG_SIGNATURE_TYPE) ? +- _("SIGNATURES") : _("signatures")); +- } +- rpmlog(RPMLOG_NOTICE, " %s\n", rc ? _("NOT OK") : _("OK")); +- } ++ rc = rpmvsVerify(vs, RPMSIG_VERIFIABLE_TYPE, cb, cbdata); + + exit: + if (rc && msg) +@@ -247,15 +236,39 @@ exit: + return rc; + } + ++static void rpmkgVerifySigsPreLogging(struct vfydata_s *vd, const char *fn){ ++ rpmlog(RPMLOG_NOTICE, "%s:%s", fn, vd->verbose ? "\n" : ""); ++} ++ ++static void rpmkgVerifySigsPostLogging(struct vfydata_s *vd, int rc){ ++ if (!vd->verbose) { ++ if (vd->seen & RPMSIG_DIGEST_TYPE) { ++ rpmlog(RPMLOG_NOTICE, " %s", (vd->bad & RPMSIG_DIGEST_TYPE) ? ++ _("DIGESTS") : _("digests")); ++ } ++ if (vd->seen & RPMSIG_SIGNATURE_TYPE) { ++ rpmlog(RPMLOG_NOTICE, " %s", (vd->bad & RPMSIG_SIGNATURE_TYPE) ? ++ _("SIGNATURES") : _("signatures")); ++ } ++ rpmlog(RPMLOG_NOTICE, " %s\n", rc ? _("NOT OK") : _("OK")); ++ } ++} ++ + /* Wrapper around rpmkVerifySigs to preserve API */ + int rpmVerifySignatures(QVA_t qva, rpmts ts, FD_t fd, const char * fn) + { + int rc = 1; /* assume failure */ ++ struct vfydata_s vd = { .seen = 0, ++ .bad = 0, ++ .verbose = rpmIsVerbose(), ++ }; + if (ts && qva && fd && fn) { + rpmKeyring keyring = rpmtsGetKeyring(ts, 1); + rpmVSFlags vsflags = rpmtsVfyFlags(ts); + int vfylevel = rpmtsVfyLevel(ts); +- rc = rpmpkgVerifySigs(keyring, vfylevel, vsflags, fd, fn); ++ rpmkgVerifySigsPreLogging(&vd, fn); ++ rc = rpmpkgVerifySigs(keyring, vfylevel, vsflags, fd, vfyCb, &vd); ++ rpmkgVerifySigsPostLogging(&vd, rc); + rpmKeyringFree(keyring); + } + return rc; +@@ -277,12 +290,22 @@ int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv) + + while ((arg = *argv++) != NULL) { + FD_t fd = Fopen(arg, "r.ufdio"); ++ struct vfydata_s vd = { .seen = 0, ++ .bad = 0, ++ .verbose = rpmIsVerbose(), ++ }; + if (fd == NULL || Ferror(fd)) { + rpmlog(RPMLOG_ERR, _("%s: open failed: %s\n"), + arg, Fstrerror(fd)); + res++; +- } else if (rpmpkgVerifySigs(keyring, vfylevel, vsflags, fd, arg)) { ++ } else { ++ rpmkgVerifySigsPreLogging(&vd, arg); ++ int rc = rpmpkgVerifySigs(keyring, vfylevel, vsflags, fd, ++ vfyCb, &vd); ++ rpmkgVerifySigsPostLogging(&vd, rc); ++ if (rc) { + res++; ++ } + } + + Fclose(fd); +@@ -290,3 +313,52 @@ int rpmcliVerifySignatures(rpmts ts, ARGV_const_t argv) + rpmKeyringFree(keyring); + return res; + } ++ ++struct vfydatafd_s { ++ size_t len; ++ char msg[BUFSIZ]; ++}; ++ ++ ++static int vfyFDCb(struct rpmsinfo_s *sinfo, void *cbdata) ++{ ++ struct vfydatafd_s *vd = cbdata; ++ char *vmsg, *msg; ++ size_t n; ++ size_t remainder = BUFSIZ - vd->len >= 0 ? BUFSIZ - vd->len : 0; ++ ++ vmsg = rpmsinfoMsg(sinfo); ++ rasprintf(&msg, " %s\n", vmsg); ++ n = rstrlcpy(vd->msg + vd->len, msg, remainder); ++ free(vmsg); ++ free(msg); ++ if(n <= remainder){ ++ vd->len += n; ++ } ++ return 1; ++} ++ ++ ++int rpmcliVerifySignaturesFD(rpmts ts, FD_t fdi, char **msg) ++{ ++ rpmRC rc = RPMRC_FAIL; ++ rpmKeyring keyring = rpmtsGetKeyring(ts, 1); ++ rpmVSFlags vsflags = rpmtsVfyFlags(ts); ++ int vfylevel = rpmtsVfyLevel(ts); ++ struct vfydatafd_s vd = {.len = 0}; ++ ++ vsflags |= rpmcliVSFlags; ++ if (rpmcliVfyLevelMask) { ++ vfylevel &= ~rpmcliVfyLevelMask; ++ rpmtsSetVfyLevel(ts, vfylevel); ++ } ++ ++ if (!rpmpkgVerifySigs(keyring, vfylevel, vsflags, fdi, vfyFDCb, &vd)) { ++ rc = RPMRC_OK; ++ } ++ *msg = strdup(vd.msg); ++ ++ rpmKeyringFree(keyring); ++ return rc; ++} ++ +diff --git a/lib/rpmextents.c b/lib/rpmextents.c +new file mode 100644 +index 0000000..ef687d6 +--- /dev/null ++++ b/lib/rpmextents.c +@@ -0,0 +1,109 @@ ++ ++#include "system.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++ ++ ++int extentsVerifySigs(FD_t fd, int print_content){ ++ rpm_loff_t current; ++ int32_t rc; ++ size_t len; ++ uint64_t content_len; ++ char *content = NULL; ++ struct extents_footer_t footer; ++ ++ current = Ftell(fd); ++ ++ if(extentsFooterFromFD(fd, &footer) != RPMRC_OK) { ++ rc = -1; ++ goto exit; ++ } ++ if(Fseek(fd, footer.offsets.checksig_offset, SEEK_SET) < 0) { ++ rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to seek signature verification offset\n")); ++ rc = -1; ++ goto exit; ++ } ++ len = sizeof(rc); ++ if (Fread(&rc, len, 1, fd) != len) { ++ rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to read Signature Verification RC\n")); ++ rc = -1; ++ goto exit; ++ } ++ ++ if(print_content) { ++ len = sizeof(content_len); ++ if (Fread(&content_len, len, 1, fd) != len) { ++ rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to read signature content length\n")); ++ goto exit; ++ } ++ ++ content = rmalloc(content_len + 1); ++ if(content == NULL) { ++ rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to allocate memory to read signature content\n")); ++ goto exit; ++ } ++ content[content_len] = 0; ++ if (Fread(content, content_len, 1, fd) != content_len) { ++ rpmlog(RPMLOG_ERR, _("extentsVerifySigs: Failed to read signature content\n")); ++ goto exit; ++ } ++ ++ rpmlog(RPMLOG_NOTICE, "%s", content); ++ } ++exit: ++ if(content){ ++ rfree(content); ++ } ++ if (Fseek(fd, current, SEEK_SET) < 0) { ++ rpmlog(RPMLOG_ERR, _("extentsVerifySigs: unable to seek back to original location\n")); ++ } ++ return rc; ++ ++} ++ ++rpmRC extentsFooterFromFD(FD_t fd, struct extents_footer_t *footer) { ++ ++ rpmRC rc = RPMRC_NOTFOUND; ++ rpm_loff_t current; ++ size_t len; ++ ++ // If the file is not seekable, we cannot detect whether or not it is transcoded. ++ if(Fseek(fd, 0, SEEK_CUR) < 0) { ++ return RPMRC_FAIL; ++ } ++ current = Ftell(fd); ++ ++ len = sizeof(struct extents_footer_t); ++ if(Fseek(fd, -len, SEEK_END) < 0) { ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ if (Fread(footer, len, 1, fd) != len) { ++ rpmlog(RPMLOG_ERR, _("isTranscodedRpm: unable to read footer\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ if (footer->magic != EXTENTS_MAGIC) { ++ rc = RPMRC_NOTFOUND; ++ goto exit; ++ } ++ rc = RPMRC_OK; ++exit: ++ if (Fseek(fd, current, SEEK_SET) < 0) { ++ rpmlog(RPMLOG_ERR, _("isTranscodedRpm: unable to seek back to original location\n")); ++ rc = RPMRC_FAIL; ++ } ++ return rc; ++} ++ ++rpmRC isTranscodedRpm(FD_t fd) { ++ struct extents_footer_t footer; ++ return extentsFooterFromFD(fd, &footer); ++} ++ ++ +diff --git a/lib/rpmlead.c b/lib/rpmlead.c +index d0db60a..e4504d2 100644 +--- a/lib/rpmlead.c ++++ b/lib/rpmlead.c +@@ -24,24 +24,6 @@ static unsigned char const lead_magic[] = { + RPMLEAD_MAGIC0, RPMLEAD_MAGIC1, RPMLEAD_MAGIC2, RPMLEAD_MAGIC3 + }; + +-/** \ingroup lead +- * The lead data structure. +- * The lead needs to be 8 byte aligned. +- * @deprecated The lead (except for signature_type) is legacy. +- * @todo Don't use any information from lead. +- */ +-struct rpmlead_s { +- unsigned char magic[4]; +- unsigned char major; +- unsigned char minor; +- short type; +- short archnum; +- char name[66]; +- short osnum; +- short signature_type; /*!< Signature header type (RPMSIG_HEADERSIG) */ +- char reserved[16]; /*!< Pad to 96 bytes -- 8 byte aligned! */ +-}; +- + static int rpmLeadFromHeader(Header h, struct rpmlead_s *l) + { + if (h != NULL) { +@@ -70,13 +52,23 @@ static int rpmLeadFromHeader(Header h, struct rpmlead_s *l) + } + + /* The lead needs to be 8 byte aligned */ +-rpmRC rpmLeadWrite(FD_t fd, Header h) ++rpmRC rpmLeadWriteFromHeader(FD_t fd, Header h) + { + rpmRC rc = RPMRC_FAIL; + struct rpmlead_s l; + +- if (rpmLeadFromHeader(h, &l)) { +- ++ if (rpmLeadFromHeader(h, &l)) { ++ rc = rpmLeadWrite(fd, l); ++ } ++ ++ return rc; ++} ++ ++/* The lead needs to be 8 byte aligned */ ++rpmRC rpmLeadWrite(FD_t fd, struct rpmlead_s l) ++{ ++ rpmRC rc = RPMRC_FAIL; ++ + l.type = htons(l.type); + l.archnum = htons(l.archnum); + l.osnum = htons(l.osnum); +@@ -84,7 +76,6 @@ rpmRC rpmLeadWrite(FD_t fd, Header h) + + if (Fwrite(&l, 1, sizeof(l), fd) == sizeof(l)) + rc = RPMRC_OK; +- } + + return rc; + } +@@ -99,6 +90,11 @@ static rpmRC rpmLeadCheck(struct rpmlead_s *lead, char **msg) + } + + rpmRC rpmLeadRead(FD_t fd, char **emsg) ++{ ++ return rpmLeadReadAndReturn(fd, emsg, NULL); ++} ++ ++rpmRC rpmLeadReadAndReturn(FD_t fd, char **emsg, struct rpmlead_s * ret) + { + rpmRC rc = RPMRC_OK; + struct rpmlead_s l; +@@ -128,5 +124,8 @@ rpmRC rpmLeadRead(FD_t fd, char **emsg) + free(err); + } + ++ if (ret) ++ *ret = l; ++ + return rc; + } +diff --git a/lib/rpmlead.h b/lib/rpmlead.h +index 80db44e..4ba2ba9 100644 +--- a/lib/rpmlead.h ++++ b/lib/rpmlead.h +@@ -19,13 +19,39 @@ extern "C" { + + #define RPMLEAD_SIZE 96 /*!< Don't rely on sizeof(struct) */ + ++/** \ingroup lead ++ * The lead data structure. ++ * The lead needs to be 8 byte aligned. ++ * @deprecated The lead (except for signature_type) is legacy. ++ * @todo Don't use any information from lead. ++ */ ++struct rpmlead_s { ++ unsigned char magic[4]; ++ unsigned char major; ++ unsigned char minor; ++ short type; ++ short archnum; ++ char name[66]; ++ short osnum; ++ short signature_type; /*!< Signature header type (RPMSIG_HEADERSIG) */ ++ char reserved[16]; /*!< Pad to 96 bytes -- 8 byte aligned! */ ++}; ++ + /** \ingroup lead + * Write lead to file handle. + * @param fd file handle + * @param h package header + * @return RPMRC_OK on success, RPMRC_FAIL on error + */ +-rpmRC rpmLeadWrite(FD_t fd, Header h); ++rpmRC rpmLeadWriteFromHeader(FD_t fd, Header h); ++ ++/** \ingroup lead ++ * Write lead to file handle. ++ * @param fd file handle ++ * @param l lead ++ * @return RPMRC_OK on success, RPMRC_FAIL on error ++ */ ++rpmRC rpmLeadWrite(FD_t fd, struct rpmlead_s l); + + /** \ingroup lead + * Read lead from file handle. +@@ -35,6 +61,15 @@ rpmRC rpmLeadWrite(FD_t fd, Header h); + */ + rpmRC rpmLeadRead(FD_t fd, char **emsg); + ++/** \ingroup lead ++ * Read lead from file handle and return it. ++ * @param fd file handle ++ * @param[out] emsg failure message on error (malloced) ++ * @param[out] ret address of lead ++ * @return RPMRC_OK on success, RPMRC_FAIL/RPMRC_NOTFOUND on error ++ */ ++rpmRC rpmLeadReadAndReturn(FD_t fd, char **emsg, struct rpmlead_s * ret); ++ + #ifdef __cplusplus + } + #endif +diff --git a/lib/rpmplugin.h b/lib/rpmplugin.h +index fab4b3e..c82d6be 100644 +--- a/lib/rpmplugin.h ++++ b/lib/rpmplugin.h +@@ -60,6 +60,13 @@ typedef rpmRC (*plugin_fsm_file_prepare_func)(rpmPlugin plugin, rpmfi fi, + int fd, const char* path, + const char *dest, + mode_t file_mode, rpmFsmOp op); ++typedef rpmRC (*plugin_fsm_file_install_func)(rpmPlugin plugin, rpmfi fi, ++ const char* path, ++ mode_t file_mode, rpmFsmOp op); ++typedef rpmRC (*plugin_fsm_file_archive_reader_func)(rpmPlugin plugin, ++ FD_t payload, ++ rpmfiles files, rpmfi *fi); ++ + + typedef struct rpmPluginHooks_s * rpmPluginHooks; + struct rpmPluginHooks_s { +@@ -80,6 +87,8 @@ struct rpmPluginHooks_s { + plugin_fsm_file_pre_func fsm_file_pre; + plugin_fsm_file_post_func fsm_file_post; + plugin_fsm_file_prepare_func fsm_file_prepare; ++ plugin_fsm_file_install_func fsm_file_install; ++ plugin_fsm_file_archive_reader_func fsm_file_archive_reader; + }; + + #ifdef __cplusplus +diff --git a/lib/rpmplugins.c b/lib/rpmplugins.c +index 62e806b..24abe96 100644 +--- a/lib/rpmplugins.c ++++ b/lib/rpmplugins.c +@@ -364,14 +364,29 @@ rpmRC rpmpluginsCallFsmFilePre(rpmPlugins plugins, rpmfi fi, const char *path, + plugin_fsm_file_pre_func hookFunc; + int i; + rpmRC rc = RPMRC_OK; ++ rpmRC hook_rc; + char *apath = abspath(fi, path); + + for (i = 0; i < plugins->count; i++) { + rpmPlugin plugin = plugins->plugins[i]; + RPMPLUGINS_SET_HOOK_FUNC(fsm_file_pre); +- if (hookFunc && hookFunc(plugin, fi, apath, file_mode, op) == RPMRC_FAIL) { +- rpmlog(RPMLOG_ERR, "Plugin %s: hook fsm_file_pre failed\n", plugin->name); +- rc = RPMRC_FAIL; ++ if (hookFunc) { ++ hook_rc = hookFunc(plugin, fi, apath, file_mode, op); ++ if (hook_rc == RPMRC_FAIL) { ++ rpmlog(RPMLOG_ERR, "Plugin %s: hook fsm_file_pre failed\n", plugin->name); ++ rc = RPMRC_FAIL; ++ } else if (hook_rc == RPMRC_PLUGIN_CONTENTS && rc != RPMRC_FAIL) { ++ if (rc == RPMRC_PLUGIN_CONTENTS) { ++ /* Another plugin already said it'd handle contents. It's ++ * undefined how these would combine, so treat this as a ++ * failure condition. ++ */ ++ rc = RPMRC_FAIL; ++ } else { ++ /* Plugin will handle content */ ++ rc = RPMRC_PLUGIN_CONTENTS; ++ } ++ } + } + } + free(apath); +@@ -420,3 +435,76 @@ rpmRC rpmpluginsCallFsmFilePrepare(rpmPlugins plugins, rpmfi fi, + + return rc; + } ++ ++rpmRC rpmpluginsCallFsmFileInstall(rpmPlugins plugins, rpmfi fi, ++ const char *path, mode_t file_mode, ++ rpmFsmOp op) ++{ ++ plugin_fsm_file_install_func hookFunc; ++ int i; ++ rpmRC rc = RPMRC_OK; ++ rpmRC hook_rc; ++ char *apath = abspath(fi, path); ++ ++ for (i = 0; i < plugins->count; i++) { ++ rpmPlugin plugin = plugins->plugins[i]; ++ RPMPLUGINS_SET_HOOK_FUNC(fsm_file_install); ++ if (hookFunc) { ++ hook_rc = hookFunc(plugin, fi, apath, file_mode, op); ++ if (hook_rc == RPMRC_FAIL) { ++ rpmlog(RPMLOG_ERR, "Plugin %s: hook fsm_file_install failed\n", plugin->name); ++ rc = RPMRC_FAIL; ++ } else if (hook_rc == RPMRC_PLUGIN_CONTENTS && rc != RPMRC_FAIL) { ++ if (rc == RPMRC_PLUGIN_CONTENTS) { ++ /* Another plugin already said it'd handle contents. It's ++ * undefined how these would combine, so treat this as a ++ * failure condition. ++ */ ++ rc = RPMRC_FAIL; ++ } else { ++ /* Plugin will handle content */ ++ rc = RPMRC_PLUGIN_CONTENTS; ++ } ++ } ++ } ++ } ++ free(apath); ++ ++ return rc; ++} ++ ++rpmRC rpmpluginsCallFsmFileArchiveReader(rpmPlugins plugins, FD_t payload, ++ rpmfiles files, rpmfi *fi) ++{ ++ plugin_fsm_file_archive_reader_func hookFunc; ++ int i; ++ rpmRC rc = RPMRC_OK; ++ rpmRC hook_rc; ++ ++ for (i = 0; i < plugins->count; i++) { ++ rpmPlugin plugin = plugins->plugins[i]; ++ RPMPLUGINS_SET_HOOK_FUNC(fsm_file_archive_reader); ++ if (hookFunc) { ++ hook_rc = hookFunc(plugin, payload, files, fi); ++ if (hook_rc == RPMRC_FAIL) { ++ rpmlog(RPMLOG_ERR, "Plugin %s: hook fsm_file_archive_reader failed\n", plugin->name); ++ rc = RPMRC_FAIL; ++ } else if (hook_rc == RPMRC_PLUGIN_CONTENTS && rc != RPMRC_FAIL) { ++ if (rc == RPMRC_PLUGIN_CONTENTS) { ++ /* Another plugin already said it'd handle contents. It's ++ * undefined how these would combine, so treat this as a ++ * failure condition. ++ */ ++ rc = RPMRC_FAIL; ++ } else { ++ /* Plugin will handle content */ ++ rc = RPMRC_PLUGIN_CONTENTS; ++ } ++ } ++ } ++ } ++ ++ return rc; ++} ++ ++ +diff --git a/lib/rpmplugins.h b/lib/rpmplugins.h +index 287a302..a00f156 100644 +--- a/lib/rpmplugins.h ++++ b/lib/rpmplugins.h +@@ -168,6 +168,23 @@ rpmRC rpmpluginsCallFsmFilePrepare(rpmPlugins plugins, rpmfi fi, + int fd, const char *path, const char *dest, + mode_t mode, rpmFsmOp op); + ++/** \ingroup rpmplugins ++ * Call the fsm file install plugin hook ++ * @param plugins plugins structure ++ * @param fi file info iterator (or NULL) ++ * @param path file object path ++ * @param file_mode file object mode ++ * @param op file operation + associated flags ++ * @return RPMRC_OK on success, RPMRC_FAIL otherwise ++ */ ++RPM_GNUC_INTERNAL ++rpmRC rpmpluginsCallFsmFileInstall(rpmPlugins plugins, rpmfi fi, ++ const char* path, mode_t file_mode, ++ rpmFsmOp op); ++ ++RPM_GNUC_INTERNAL ++rpmRC rpmpluginsCallFsmFileArchiveReader(rpmPlugins plugins, FD_t payload, ++ rpmfiles files, rpmfi *fi); + #ifdef __cplusplus + } + #endif +diff --git a/lib/rpmte.c b/lib/rpmte.c +index d31152a..642e809 100644 +--- a/lib/rpmte.c ++++ b/lib/rpmte.c +@@ -437,6 +437,11 @@ FD_t rpmteSetFd(rpmte te, FD_t fd) + return NULL; + } + ++FD_t rpmteFd(rpmte te) ++{ ++ return (te != NULL ? te->fd : NULL); ++} ++ + fnpyKey rpmteKey(rpmte te) + { + return (te != NULL ? te->key : NULL); +diff --git a/lib/transaction.c b/lib/transaction.c +index 70d2587..1e98f11 100644 +--- a/lib/transaction.c ++++ b/lib/transaction.c +@@ -28,6 +28,7 @@ + #include + #include + #include ++#include + + #include "fprint.h" + #include "misc.h" +@@ -1292,19 +1293,25 @@ static int verifyPackageFiles(rpmts ts, rpm_loff_t total) + + rpmtsNotify(ts, p, RPMCALLBACK_VERIFY_PROGRESS, oc++, total); + FD_t fd = rpmtsNotify(ts, p, RPMCALLBACK_INST_OPEN_FILE, 0, 0); +- if (fd != NULL) { +- prc = rpmpkgRead(vs, fd, NULL, NULL, &vd.msg); +- rpmtsNotify(ts, p, RPMCALLBACK_INST_CLOSE_FILE, 0, 0); ++ if(fd != NULL && isTranscodedRpm(fd) == RPMRC_OK) { ++ /* Transcoded RPMs are validated at transcoding time */ ++ prc = RPMRC_OK; ++ verified = 1; ++ } else { ++ if (fd != NULL) { ++ prc = rpmpkgRead(vs, fd, NULL, NULL, &vd.msg); ++ rpmtsNotify(ts, p, RPMCALLBACK_INST_CLOSE_FILE, 0, 0); ++ } ++ if (prc == RPMRC_OK) ++ prc = rpmvsVerify(vs, RPMSIG_VERIFIABLE_TYPE, vfyCb, &vd); ++ ++ /* Record verify result */ ++ if (vd.type[RPMSIG_SIGNATURE_TYPE] == RPMRC_OK) ++ verified |= RPMSIG_SIGNATURE_TYPE; ++ if (vd.type[RPMSIG_DIGEST_TYPE] == RPMRC_OK) ++ verified |= RPMSIG_DIGEST_TYPE; + } + +- if (prc == RPMRC_OK) +- prc = rpmvsVerify(vs, RPMSIG_VERIFIABLE_TYPE, vfyCb, &vd); +- +- /* Record verify result */ +- if (vd.type[RPMSIG_SIGNATURE_TYPE] == RPMRC_OK) +- verified |= RPMSIG_SIGNATURE_TYPE; +- if (vd.type[RPMSIG_DIGEST_TYPE] == RPMRC_OK) +- verified |= RPMSIG_DIGEST_TYPE; + rpmteSetVerified(p, verified); + + if (prc) +diff --git a/macros.in b/macros.in +index 7eb3d2b..df7defe 100644 +--- a/macros.in ++++ b/macros.in +@@ -1188,6 +1188,7 @@ Supplements: (%{name} = %{version}-%{release} and langpacks-%{1})\ + + # Transaction plugin macros + %__plugindir %{_libdir}/rpm-plugins ++%__transaction_reflink %{__plugindir}/reflink.so + %__transaction_systemd_inhibit %{__plugindir}/systemd_inhibit.so + %__transaction_selinux %{__plugindir}/selinux.so + %__transaction_syslog %{__plugindir}/syslog.so +diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt +index c2c112b..ed63a43 100644 +--- a/plugins/CMakeLists.txt ++++ b/plugins/CMakeLists.txt +@@ -1,5 +1,6 @@ + add_library(prioreset MODULE prioreset.c) + add_library(syslog MODULE syslog.c) ++add_library(reflink MODULE reflink.c) + + if(WITH_SELINUX) + add_library(selinux MODULE selinux.c) +@@ -44,7 +45,6 @@ get_property(plugins DIRECTORY PROPERTY BUILDSYSTEM_TARGETS) + foreach(plugin ${plugins}) + target_link_libraries(${plugin} PRIVATE librpmio librpm ${Intl_LIBRARIES}) + # XX this is wrong, but required while rpmplugin.h is there +- target_include_directories(${plugin} PRIVATE ${CMAKE_SOURCE_DIR}/lib ${Intl_INCLUDE_DIRS}) ++ target_include_directories(${plugin} PRIVATE ${CMAKE_SOURCE_DIR}/lib ${CMAKE_SOURCE_DIR}/rpmio ${Intl_INCLUDE_DIRS}) + install(TARGETS ${plugin} DESTINATION ${plugindir}) + endforeach() +- +diff --git a/plugins/reflink.c b/plugins/reflink.c +new file mode 100644 +index 0000000..7ce3a9f +--- /dev/null ++++ b/plugins/reflink.c +@@ -0,0 +1,401 @@ ++#include "system.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#if defined(__linux__) ++#include /* For FICLONE */ ++#endif ++ ++#include ++#include ++#include ++#include ++#include "rpmplugin.h" ++#include "rpmte_internal.h" ++#include "rpmio_internal.h" ++ ++ ++#include "debug.h" ++ ++#include ++ ++/* use hash table to remember inode -> ix (for rpmfilesFN(ix)) lookups */ ++#undef HASHTYPE ++#undef HTKEYTYPE ++#undef HTDATATYPE ++#define HASHTYPE inodeIndexHash ++#define HTKEYTYPE rpm_ino_t ++#define HTDATATYPE const char * ++#include "rpmhash.H" ++#include "rpmhash.C" ++ ++/* We use this in find to indicate a key wasn't found. This is an ++ * unrecoverable error, but we can at least show a decent error. 0 is never a ++ * valid offset because it's the offset of the start of the file. ++ */ ++#define NOT_FOUND 0 ++ ++#define BUFFER_SIZE (1024 * 128) ++ ++struct reflink_state_s { ++ /* Stuff that's used across rpms */ ++ long fundamental_block_size; ++ char *buffer; ++ ++ /* stuff that's used/updated per psm */ ++ uint32_t keys, keysize; ++ ++ /* table for current rpm, keys * (keysize + sizeof(rpm_loff_t)) */ ++ unsigned char *table; ++ FD_t fd; ++ rpmfiles files; ++ inodeIndexHash inodeIndexes; ++ int transcoded; ++}; ++ ++typedef struct reflink_state_s * reflink_state; ++ ++/* ++ * bsearch_r: implements a re-entrant version of stdlib's bsearch. ++ * code taken and adapted from /usr/include/bits/stdlib-bsearch.h ++ */ ++static inline void * ++bsearch_r (const void *__key, const void *__base, size_t __nmemb, size_t __size, ++ __compar_d_fn_t __compar, void *__arg) ++{ ++ size_t __l, __u, __idx; ++ const void *__p; ++ int __comparison; ++ ++ __l = 0; ++ __u = __nmemb; ++ while (__l < __u) ++ { ++ __idx = (__l + __u) / 2; ++ __p = (const void *) (((const char *) __base) + (__idx * __size)); ++ __comparison = (*__compar) (__key, __p, __arg); ++ if (__comparison < 0) ++ __u = __idx; ++ else if (__comparison > 0) ++ __l = __idx + 1; ++ else ++ { ++#if __GNUC_PREREQ(4, 6) ++# pragma GCC diagnostic push ++# pragma GCC diagnostic ignored "-Wcast-qual" ++#endif ++ return (void *) __p; ++#if __GNUC_PREREQ(4, 6) ++# pragma GCC diagnostic pop ++#endif ++ } ++ } ++ ++ return NULL; ++} ++ ++static int cmpdigest(const void *k1, const void *k2, void *data) { ++ rpmlog(RPMLOG_DEBUG, _("reflink: cmpdigest k1=%p k2=%p\n"), k1, k2); ++ return memcmp(k1, k2, *(int *)data); ++} ++ ++static int inodeCmp(rpm_ino_t a, rpm_ino_t b) ++{ ++ return (a != b); ++} ++ ++static unsigned int inodeId(rpm_ino_t a) ++{ ++ /* rpm_ino_t is uint32_t so maps safely to unsigned int */ ++ return (unsigned int)a; ++} ++ ++static rpmRC reflink_init(rpmPlugin plugin, rpmts ts) { ++ reflink_state state = rcalloc(1, sizeof(struct reflink_state_s)); ++ ++ /* IOCTL-FICLONERANGE(2): ...Disk filesystems generally require the offset ++ * and length arguments to be aligned to the fundamental block size. ++ * ++ * The value of "fundamental block size" is directly related to the ++ * system's page size, so we should use that. ++ */ ++ state->fundamental_block_size = sysconf(_SC_PAGESIZE); ++ state->buffer = rcalloc(1, BUFFER_SIZE); ++ rpmPluginSetData(plugin, state); ++ ++ return RPMRC_OK; ++} ++ ++static void reflink_cleanup(rpmPlugin plugin) { ++ reflink_state state = rpmPluginGetData(plugin); ++ free(state->buffer); ++ free(state); ++} ++ ++static rpmRC reflink_psm_pre(rpmPlugin plugin, rpmte te) { ++ rpmRC rc; ++ size_t len; ++ ++ reflink_state state = rpmPluginGetData(plugin); ++ state->fd = rpmteFd(te); ++ if (state->fd == 0) { ++ rpmlog(RPMLOG_DEBUG, _("reflink: fd = 0, no install\n")); ++ return RPMRC_OK; ++ } ++ ++ rpm_loff_t current = Ftell(state->fd); ++ rc = isTranscodedRpm(state->fd); ++ ++ switch(rc){ ++ // Fail to parse the file, fail the plugin. ++ case RPMRC_FAIL: ++ return RPMRC_FAIL; ++ // This is not a transcoded file, do nothing. ++ case RPMRC_NOTFOUND: ++ return RPMRC_OK; ++ default: ++ break; ++ } ++ rpmlog(RPMLOG_DEBUG, _("reflink: *is* transcoded\n")); ++ state->transcoded = 1; ++ ++ state->files = rpmteFiles(te); ++ /* tail of file contains offset_table, offset_checksums then magic */ ++ if (Fseek(state->fd, -(sizeof(rpm_loff_t) * 2 + sizeof(extents_magic_t)), SEEK_END) < 0) { ++ rpmlog(RPMLOG_ERR, _("reflink: failed to seek for tail %p\n"), ++ state->fd); ++ return RPMRC_FAIL; ++ } ++ rpm_loff_t table_start; ++ len = sizeof(table_start); ++ if (Fread(&table_start, len, 1, state->fd) != len) { ++ rpmlog(RPMLOG_ERR, _("reflink: unable to read table_start\n")); ++ return RPMRC_FAIL; ++ } ++ if (Fseek(state->fd, table_start, SEEK_SET) < 0) { ++ rpmlog(RPMLOG_ERR, _("reflink: unable to seek to table_start\n")); ++ return RPMRC_FAIL; ++ } ++ len = sizeof(state->keys); ++ if (Fread(&state->keys, len, 1, state->fd) != len) { ++ rpmlog(RPMLOG_ERR, _("reflink: unable to read number of keys\n")); ++ return RPMRC_FAIL; ++ } ++ len = sizeof(state->keysize); ++ if (Fread(&state->keysize, len, 1, state->fd) != len) { ++ rpmlog(RPMLOG_ERR, _("reflink: unable to read keysize\n")); ++ return RPMRC_FAIL; ++ } ++ rpmlog( ++ RPMLOG_DEBUG, ++ _("reflink: table_start=0x%lx, keys=%d, keysize=%d\n"), ++ table_start, state->keys, state->keysize ++ ); ++ /* now get digest table if there is a reason to have one. */ ++ if (state->keys == 0 || state->keysize == 0) { ++ /* no files (or no digests(!)) */ ++ state->table = NULL; ++ } else { ++ int table_size = state->keys * (state->keysize + sizeof(rpm_loff_t)); ++ state->table = rcalloc(1, table_size); ++ if (Fread(state->table, table_size, 1, state->fd) != table_size) { ++ rpmlog(RPMLOG_ERR, _("reflink: unable to read table\n")); ++ return RPMRC_FAIL; ++ } ++ state->inodeIndexes = inodeIndexHashCreate( ++ state->keys, inodeId, inodeCmp, NULL, (inodeIndexHashFreeData)free ++ ); ++ } ++ ++ /* Seek back to original location. ++ * Might not be needed if we seek to offset immediately ++ */ ++ if (Fseek(state->fd, current, SEEK_SET) < 0) { ++ rpmlog(RPMLOG_ERR, ++ _("reflink: unable to seek back to original location\n")); ++ return RPMRC_FAIL; ++ } ++ return RPMRC_OK; ++} ++ ++static rpmRC reflink_psm_post(rpmPlugin plugin, rpmte te, int res) ++{ ++ reflink_state state = rpmPluginGetData(plugin); ++ state->files = rpmfilesFree(state->files); ++ if (state->table) { ++ free(state->table); ++ state->table = NULL; ++ } ++ if (state->inodeIndexes) { ++ inodeIndexHashFree(state->inodeIndexes); ++ state->inodeIndexes = NULL; ++ } ++ state->transcoded = 0; ++ return RPMRC_OK; ++} ++ ++ ++/* have a prototype, warnings system */ ++rpm_loff_t find(const unsigned char *digest, reflink_state state); ++ ++rpm_loff_t find(const unsigned char *digest, reflink_state state) { ++ rpmlog(RPMLOG_DEBUG, ++ _("reflink: bsearch_r(key=%p, base=%p, nmemb=%d, size=%lu)\n"), ++ digest, state->table, state->keys, ++ state->keysize + sizeof(rpm_loff_t)); ++ char *entry = bsearch_r(digest, state->table, state->keys, ++ state->keysize + sizeof(rpm_loff_t), cmpdigest, ++ &state->keysize); ++ if (entry == NULL) { ++ return NOT_FOUND; ++ } ++ rpm_loff_t offset = *(rpm_loff_t *)(entry + state->keysize); ++ return offset; ++} ++ ++static rpmRC reflink_fsm_file_install(rpmPlugin plugin, rpmfi fi, const char* path, ++ mode_t file_mode, rpmFsmOp op) ++{ ++ struct file_clone_range fcr; ++ rpm_loff_t size; ++ int dst, rc; ++ const char **hl_target = NULL; ++ ++ reflink_state state = rpmPluginGetData(plugin); ++ if (state->table == NULL) { ++ /* no table means rpm is not in reflink format, so leave. Now. */ ++ return RPMRC_OK; ++ } ++ if (op == FA_TOUCH) { ++ /* we're not overwriting an existing file. */ ++ return RPMRC_OK; ++ } ++ fcr.dest_offset = 0; ++ if (S_ISREG(file_mode) && !(rpmfiFFlags(fi) & RPMFILE_GHOST)) { ++ rpm_ino_t inode = rpmfiFInode(fi); ++ /* check for hard link entry in table. GetEntry overwrites hlix with ++ * the address of the first match. ++ */ ++ if (inodeIndexHashGetEntry(state->inodeIndexes, inode, &hl_target, ++ NULL, NULL)) { ++ /* entry is in table, use hard link */ ++ if (link(hl_target[0], path) != 0) { ++ rpmlog(RPMLOG_ERR, ++ _("reflink: Unable to hard link %s -> %s due to %s\n"), ++ hl_target[0], path, strerror(errno)); ++ return RPMRC_FAIL; ++ } ++ return RPMRC_PLUGIN_CONTENTS; ++ } ++ /* if we didn't hard link, then we'll track this inode as being ++ * created soon ++ */ ++ if (rpmfiFNlink(fi) > 1) { ++ /* minor optimization: only store files with more than one link */ ++ inodeIndexHashAddEntry(state->inodeIndexes, inode, rstrdup(path)); ++ } ++ /* derived from wfd_open in fsm.c */ ++ mode_t old_umask = umask(0577); ++ dst = open(path, O_WRONLY | O_CREAT | O_EXCL, S_IRUSR); ++ umask(old_umask); ++ if (dst == -1) { ++ rpmlog(RPMLOG_ERR, ++ _("reflink: Unable to open %s for writing due to %s, flags = %x\n"), ++ path, strerror(errno), rpmfiFFlags(fi)); ++ return RPMRC_FAIL; ++ } ++ size = rpmfiFSize(fi); ++ if (size > 0) { ++ /* round src_length down to fundamental_block_size multiple */ ++ fcr.src_length = size / state->fundamental_block_size * state->fundamental_block_size; ++ if ((size % state->fundamental_block_size) > 0) { ++ /* round up to next fundamental_block_size. We expect the data ++ * in the rpm to be similarly padded. ++ */ ++ fcr.src_length += state->fundamental_block_size; ++ } ++ fcr.src_fd = Fileno(state->fd); ++ if (fcr.src_fd == -1) { ++ close(dst); ++ rpmlog(RPMLOG_ERR, _("reflink: src fd lookup failed\n")); ++ return RPMRC_FAIL; ++ } ++ fcr.src_offset = find(rpmfiFDigest(fi, NULL, NULL), state); ++ if (fcr.src_offset == NOT_FOUND) { ++ close(dst); ++ rpmlog(RPMLOG_ERR, _("reflink: digest not found\n")); ++ return RPMRC_FAIL; ++ } ++ rpmlog(RPMLOG_DEBUG, ++ _("reflink: Reflinking %llu bytes at %llu to %s orig size=%ld, file=%lld\n"), ++ fcr.src_length, fcr.src_offset, path, size, fcr.src_fd); ++ rc = ioctl(dst, FICLONERANGE, &fcr); ++ if (rc) { ++ rpmlog(RPMLOG_WARNING, ++ _("reflink: falling back to copying bits for %s due to %d, %d = %s\n"), ++ path, rc, errno, strerror(errno)); ++ if (Fseek(state->fd, fcr.src_offset, SEEK_SET) < 0) { ++ close(dst); ++ rpmlog(RPMLOG_ERR, ++ _("reflink: unable to seek on copying bits\n")); ++ return RPMRC_FAIL; ++ } ++ rpm_loff_t left = size; ++ size_t len, read, written; ++ while (left) { ++ len = (left > BUFFER_SIZE ? BUFFER_SIZE : left); ++ read = Fread(state->buffer, len, 1, state->fd); ++ if (read != len) { ++ close(dst); ++ rpmlog(RPMLOG_ERR, ++ _("reflink: short read on copying bits\n")); ++ return RPMRC_FAIL; ++ } ++ written = write(dst, state->buffer, len); ++ if (read != written) { ++ close(dst); ++ rpmlog(RPMLOG_ERR, ++ _("reflink: short write on copying bits\n")); ++ return RPMRC_FAIL; ++ } ++ left -= len; ++ } ++ } else { ++ /* reflink worked, so truncate */ ++ rc = ftruncate(dst, size); ++ if (rc) { ++ rpmlog(RPMLOG_ERR, ++ _("reflink: Unable to truncate %s to %ld due to %s\n"), ++ path, size, strerror(errno)); ++ return RPMRC_FAIL; ++ } ++ } ++ } ++ close(dst); ++ return RPMRC_PLUGIN_CONTENTS; ++ } ++ return RPMRC_OK; ++} ++ ++static rpmRC reflink_fsm_file_archive_reader(rpmPlugin plugin, FD_t payload, ++ rpmfiles files, rpmfi *fi) { ++ reflink_state state = rpmPluginGetData(plugin); ++ if(state->transcoded) { ++ *fi = rpmfilesIter(files, RPMFI_ITER_FWD); ++ return RPMRC_PLUGIN_CONTENTS; ++ } ++ return RPMRC_OK; ++} ++ ++struct rpmPluginHooks_s reflink_hooks = { ++ .init = reflink_init, ++ .cleanup = reflink_cleanup, ++ .psm_pre = reflink_psm_pre, ++ .psm_post = reflink_psm_post, ++ .fsm_file_install = reflink_fsm_file_install, ++ .fsm_file_archive_reader = reflink_fsm_file_archive_reader, ++}; +diff --git a/rpmio/rpmpgp.c b/rpmio/rpmpgp.c +index f545213..5b2ad78 100644 +--- a/rpmio/rpmpgp.c ++++ b/rpmio/rpmpgp.c +@@ -81,3 +81,31 @@ pgpArmor pgpReadPkts(const char * fn, uint8_t ** pkt, size_t * pktlen) + free(b); + return ec; + } ++ ++/** \ingroup rpmpgp ++ * Return value of an OpenPGP string. ++ * @param vs table of (string,value) pairs ++ * @param s string token to lookup ++ * @param se end-of-string address ++ * @return byte value ++ */ ++static inline ++int pgpValTok(pgpValTbl vs, const char * s, const char * se) ++{ ++ do { ++ size_t vlen = strlen(vs->str); ++ if (vlen <= (se-s) && rstreqn(s, vs->str, vlen)) ++ break; ++ } while ((++vs)->val != -1); ++ return vs->val; ++} ++ ++int pgpStringVal(pgpValType type, const char *str, uint8_t *val) ++{ ++ pgpValTbl tbl = pgpValTable(type); ++ if (tbl == NULL) return -1; ++ int v = pgpValTok(tbl, str, str + strlen(str)); ++ if (v == -1) return -1; ++ *val = (uint8_t)v; ++ return 0; ++} +diff --git a/rpmio/rpmpgp_internal.c b/rpmio/rpmpgp_internal.c +index 0e38946..0cfe9b9 100644 +--- a/rpmio/rpmpgp_internal.c ++++ b/rpmio/rpmpgp_internal.c +@@ -88,24 +88,6 @@ static void pgpPrtTime(const char * pre, const uint8_t *p, size_t plen) + } + } + +-/** \ingroup rpmpgp +- * Return value of an OpenPGP string. +- * @param vs table of (string,value) pairs +- * @param s string token to lookup +- * @param se end-of-string address +- * @return byte value +- */ +-static inline +-int pgpValTok(pgpValTbl vs, const char * s, const char * se) +-{ +- do { +- size_t vlen = strlen(vs->str); +- if (vlen <= (se-s) && rstreqn(s, vs->str, vlen)) +- break; +- } while ((++vs)->val != -1); +- return vs->val; +-} +- + /** \ingroup rpmpgp + * Decode length from 1, 2, or 5 octet body length encoding, used in + * new format packet headers and V4 signature subpackets. +diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt +index 07749bc..8f8ae64 100644 +--- a/scripts/CMakeLists.txt ++++ b/scripts/CMakeLists.txt +@@ -12,7 +12,7 @@ install(PROGRAMS + rpm_macros_provides.sh + rpmdb_dump rpmdb_load + rpm2cpio.sh tgpg +- sysusers.sh ++ sysusers.sh rpm2extents_dump + DESTINATION ${RPM_CONFIGDIR} + ) + install(FILES +diff --git a/scripts/rpm2extents_dump b/scripts/rpm2extents_dump +new file mode 100755 +index 0000000..596a59a +--- /dev/null ++++ b/scripts/rpm2extents_dump +@@ -0,0 +1,94 @@ ++#!/usr/bin/env python3 ++ ++import argparse ++import binascii ++import os ++import struct ++import sys ++ ++MAGIC_SIZE = 8 ++MAGIC_STR = b'KWTSH100' ++ ++POS_SIZE = 8 ++ ++def keep_position(func): ++ def wrapper(*args, **kwargs): ++ curr = args[0].tell() ++ res = func(*args, **kwargs) ++ f.seek(curr, os.SEEK_SET) ++ return res ++ return wrapper ++ ++def read_validation_digest(f, validation_offset): ++ digests = [] ++ # validation ++ f.seek(validation_offset, os.SEEK_SET) ++ val_content_len, val_digests_num = struct.unpack('=QI', f.read(8+4)) ++ for i in range(val_digests_num): ++ algo_name_len, digest_len = struct.unpack('=II', f.read(8)) ++ algo_name, digest = struct.unpack(f'{algo_name_len}s{digest_len}s', f.read(algo_name_len+digest_len)) ++ digests.append((algo_name, binascii.hexlify(digest))) ++ return digests ++ ++ ++def read_digests_table(f, digest_offset): ++ digests = [] ++ # validation ++ f.seek(digest_offset, os.SEEK_SET) ++ table_len, digest_len = struct.unpack('=II', f.read(8)) ++ ++ for i in range(table_len): ++ digest, pos = struct.unpack(f'{digest_len}sQ', f.read(digest_len + 8)) ++ digests.append((pos, binascii.hexlify(digest))) ++ return digests ++ ++def read_signature_output(f, signature_offset): ++ f.seek(signature_offset, os.SEEK_SET) ++ signature_rc, signature_output_len = struct.unpack('=IQ', f.read(12)) ++ return signature_rc, f.read(signature_output_len) ++ ++@keep_position ++def parse_file(f): ++ digests = [] ++ pos_table_offset = f.seek(-8 - 3*POS_SIZE, os.SEEK_END) ++ signature_offset, digest_offset, validation_offset = struct.unpack('=QQQ', f.read(3*POS_SIZE)) ++ ++ validation_digests = read_validation_digest(f, validation_offset) ++ digests_table = read_digests_table(f, digest_offset) ++ signature_ouput = read_signature_output(f, signature_offset) ++ ++ return validation_digests, digests_table, signature_ouput ++ ++@keep_position ++def is_transcoded(f): ++ f.seek(-MAGIC_SIZE, os.SEEK_END) ++ magic = f.read(MAGIC_SIZE) ++ return magic == MAGIC_STR ++ ++def arg_parse(): ++ parser = argparse.ArgumentParser() ++ parser.add_argument('--dump-signature', action='store_true') ++ parser.add_argument('--dump-file-digest-table', action='store_true') ++ parser.add_argument('--dump-digests', action='store_true') ++ parser.add_argument('file') ++ ++ return parser.parse_args() ++ ++if __name__ == '__main__': ++ args = arg_parse() ++ f = open(args.file, 'rb') ++ if not is_transcoded(f): ++ sys.exit(1) ++ ++ validation_digests, digests_table, signature_output = parse_file(f) ++ if(args.dump_file_digest_table): ++ for digest in digests_table: ++ print(f"FileDigest {hex(digest[0])}: {digest[1]}") ++ ++ if(args.dump_digests): ++ for validation_digest in validation_digests: ++ print(f"HeaderDigest {validation_digest[0]} {validation_digest[1]}") ++ ++ if(args.dump_signature): ++ print(f"RPMSignOutput RC {signature_output[0]}\nRPMSignOutput Content {signature_output[1].decode()}") ++ +diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt +index a6afbc0..bee3b64 100644 +--- a/tests/CMakeLists.txt ++++ b/tests/CMakeLists.txt +@@ -51,6 +51,7 @@ set(TESTSUITE_AT + rpmvfylevel.at + rpmpgp.at + rpmdevel.at ++ rpm2extents.at + ) + + set(TESTPROGS rpmpgpcheck rpmpgppubkeyfingerprint) +diff --git a/tests/atlocal.in b/tests/atlocal.in +index a5faeed..9930cb4 100644 +--- a/tests/atlocal.in ++++ b/tests/atlocal.in +@@ -100,6 +100,19 @@ snapshot() + esac + } + ++FSTYPE=$(stat -f -c %T /) ++REFLINKABLE_FS=("xfs" "brtfs") ++ ++REFLINK_DISABLED=true; ++for item in "${REFLINKABLE_FS[@]}" ++do ++ if test "${FSTYPE}" = "${item}" ++ then ++ REFLINK_DISABLED=false; ++ break ++ fi ++done ++ + setup_env() + { + if [ -d tree ]; then +@@ -138,6 +151,14 @@ runroot() + --nouserns + } + ++function runroot_plugins() ++{ ++ setup_env ++ (unset RPM_CONFIGDIR RPM_POPTEXEC_PATH; cd ${RPMTEST} && \ ++ MAGIC="/magic/magic" FAKECHROOT_BASE="${RPMTEST}" fakechroot "$@" --define "_buildhost testhost" --define "_topdir /build" --nouserns ++ ) ++} ++ + runroot_other() + { + setup_env +diff --git a/tests/rpm2extents.at b/tests/rpm2extents.at +new file mode 100644 +index 0000000..c9c79c5 +--- /dev/null ++++ b/tests/rpm2extents.at +@@ -0,0 +1,151 @@ ++# rpm2extents.at: Some very basic checks ++# ++# Copyright (C) 2022 Manu Bretelle ++# ++# This program is free software; you can redistribute it and/or modify ++# it under the terms of the GNU General Public License as published by ++# the Free Software Foundation; either version 2 of the License, or ++# (at your option) any later version. ++# ++# This program is distributed in the hope that it will be useful, ++# but WITHOUT ANY WARRANTY; without even the implied warranty of ++# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ++# GNU General Public License for more details. ++# ++# You should have received a copy of the GNU General Public License ++# along with this program; if not, write to the Free Software ++# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ++ ++AT_BANNER([rpm2extents tests]) ++ ++# ------------------------------ ++ ++# check that transcoder write magic at the end ++AT_SETUP([rpm2extents magic]) ++AT_KEYWORDS([rpm2extents]) ++AT_CHECK([runroot_other cat /data/RPMS/hello-2.0-1.x86_64.rpm | runroot_other rpm2extents SHA256 | tail -c8], ++[0], ++[KWTSH100], ++[ignore]) ++AT_CLEANUP ++ ++# Check that transcoder writes checksig return code and content. ++# ++AT_SETUP([rpm2extents signature]) ++AT_KEYWORDS([rpm2extents]) ++AT_CHECK([ ++RPMDB_INIT ++ ++runroot_other cat /data/RPMS/hello-2.0-1.x86_64-signed.rpm | runroot_other rpm2extents SHA256 > /tmp/hello-2.0-1.x86_64-signed.rpm 2> /dev/null ++rpm2extents_dump --dump-signature /tmp/hello-2.0-1.x86_64-signed.rpm ++runroot rpmkeys --import /data/keys/rpm.org-rsa-2048-test.pub ++runroot_other cat /data/RPMS/hello-2.0-1.x86_64-signed.rpm | runroot_other rpm2extents SHA256 > /tmp/hello-2.0-1.x86_64-signed.rpm ++rpm2extents_dump --dump-signature /tmp/hello-2.0-1.x86_64-signed.rpm ++], ++[0], ++[RPMSignOutput RC 2 ++RPMSignOutput Content Header V4 RSA/SHA256 Signature, key ID 1964c5fc: NOKEY ++ Header SHA256 digest: OK ++ Header SHA1 digest: OK ++ Payload SHA256 digest: OK ++ V4 RSA/SHA256 Signature, key ID 1964c5fc: NOKEY ++ MD5 digest: OK ++ ++RPMSignOutput RC 0 ++RPMSignOutput Content Header V4 RSA/SHA256 Signature, key ID 1964c5fc: OK ++ Header SHA256 digest: OK ++ Header SHA1 digest: OK ++ Payload SHA256 digest: OK ++ V4 RSA/SHA256 Signature, key ID 1964c5fc: OK ++ MD5 digest: OK ++ ++], ++[]) ++AT_CLEANUP ++ ++AT_SETUP([rpm2extents signature verification]) ++AT_KEYWORDS([rpm2extents]) ++AT_CHECK([ ++RPMDB_INIT ++ ++runroot_other cat /data/RPMS/hello-2.0-1.x86_64-signed.rpm | runroot_other rpm2extents SHA256 > ${RPMTEST}/tmp/hello-2.0-1.x86_64-signed.rpm 2> /dev/null ++runroot rpmkeys -Kv /tmp/hello-2.0-1.x86_64-signed.rpm; echo $? ++runroot rpmkeys --import /data/keys/rpm.org-rsa-2048-test.pub ++runroot_other cat /data/RPMS/hello-2.0-1.x86_64-signed.rpm | runroot_other rpm2extents SHA256 > ${RPMTEST}/tmp/hello-2.0-1.x86_64-signed.rpm ++runroot rpmkeys -Kv /tmp/hello-2.0-1.x86_64-signed.rpm; echo $? ++], ++[0], ++[/tmp/hello-2.0-1.x86_64-signed.rpm: ++ Header V4 RSA/SHA256 Signature, key ID 1964c5fc: NOKEY ++ Header SHA256 digest: OK ++ Header SHA1 digest: OK ++ Payload SHA256 digest: OK ++ V4 RSA/SHA256 Signature, key ID 1964c5fc: NOKEY ++ MD5 digest: OK ++1 ++/tmp/hello-2.0-1.x86_64-signed.rpm: ++ Header V4 RSA/SHA256 Signature, key ID 1964c5fc: OK ++ Header SHA256 digest: OK ++ Header SHA1 digest: OK ++ Payload SHA256 digest: OK ++ V4 RSA/SHA256 Signature, key ID 1964c5fc: OK ++ MD5 digest: OK ++0 ++], ++[]) ++AT_CLEANUP ++ ++# check that package in denylist is not transcoded ++AT_SETUP([rpm2extents denylist]) ++AT_KEYWORDS([rpm2extents]) ++AT_CHECK([ ++export LIBREPO_TRANSCODE_RPMS_DENYLIST="vim,hello,cowsay" ++runroot_other cat /data/RPMS/hello-2.0-1.x86_64.rpm | runroot_other rpm2extents SHA256 | runroot_other cmp /data/RPMS/hello-2.0-1.x86_64.rpm -], ++[0], ++[], ++[ignore]) ++AT_CLEANUP ++ ++AT_SETUP([rpm2extents install package]) ++AT_KEYWORDS([rpm2extents reflink]) ++AT_SKIP_IF([$REFLINK_DISABLED]) ++AT_CHECK([ ++RPMDB_INIT ++ ++runroot_other cat /data/RPMS/hello-2.0-1.x86_64.rpm | runroot_other rpm2extents SHA256 > ${RPMTEST}/tmp/hello-2.0-1.x86_64.rpm 2> /dev/null ++runroot_plugins rpm -i --nodeps --undefine=%__transaction_dbus_announce /tmp/hello-2.0-1.x86_64.rpm ++test -f ${RPMTEST}/usr/bin/hello ++], ++[0], ++[], ++[]) ++AT_CLEANUP ++ ++AT_SETUP([reflink ignores non-transcoded package]) ++AT_KEYWORDS([reflink]) ++AT_CHECK([ ++RPMDB_INIT ++ ++runroot_plugins rpm -i --nodeps --undefine=%__transaction_dbus_announce /data/RPMS/hello-2.0-1.x86_64.rpm && exit $? ++# Check that the file is properly installed in chroot ++test -f ${RPMTEST}/usr/bin/hello ++], ++[0], ++[], ++[]) ++AT_CLEANUP ++ ++AT_SETUP([reflink hardlink package]) ++AT_KEYWORDS([reflink hardlink]) ++AT_SKIP_IF([$REFLINK_DISABLED]) ++AT_CHECK([ ++RPMDB_INIT ++ ++PKG=hlinktest-1.0-1.noarch.rpm ++runroot_other cat /data/RPMS/${PKG} | runroot_other rpm2extents SHA256 > ${RPMTEST}/tmp/${PKG} 2> /dev/null ++runroot_plugins rpm -i --nodeps --undefine=%__transaction_dbus_announce /tmp/${PKG} ++], ++[0], ++[], ++[]) ++AT_CLEANUP +diff --git a/tests/rpmtests.at b/tests/rpmtests.at +index d675452..3e6d528 100644 +--- a/tests/rpmtests.at ++++ b/tests/rpmtests.at +@@ -24,3 +24,4 @@ m4_include([rpmreplace.at]) + m4_include([rpmconfig.at]) + m4_include([rpmconfig2.at]) + m4_include([rpmconfig3.at]) ++m4_include([rpm2extents.at]) +diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt +index f4fbefe..d369d20 100644 +--- a/tools/CMakeLists.txt ++++ b/tools/CMakeLists.txt +@@ -12,6 +12,7 @@ add_executable(rpmdeps rpmdeps.c) + add_executable(rpmgraph rpmgraph.c) + add_executable(rpmlua rpmlua.c) + add_executable(rpmuncompress rpmuncompress.c) ++add_executable(rpm2extents rpm2extents.c) + + target_link_libraries(rpmsign PRIVATE librpmsign) + target_link_libraries(rpmlua PRIVATE LUA::LUA) +@@ -32,6 +33,7 @@ endif() + + target_include_directories(rpmlua PRIVATE ${CMAKE_SOURCE_DIR}/rpmio) + target_include_directories(rpmgraph PRIVATE ${CMAKE_SOURCE_DIR}/lib) ++target_include_directories(rpm2extents PRIVATE ${CMAKE_SOURCE_DIR}/lib ${CMAKE_SOURCE_DIR}/rpmio) + + if (READLINE_FOUND) + target_link_libraries(rpmspec PRIVATE PkgConfig::READLINE) +@@ -62,5 +64,4 @@ install(TARGETS + rpm rpmdb rpmkeys rpm2cpio rpmsign rpmbuild rpmspec + rpmlua rpmgraph + ) +-install(TARGETS rpmdeps rpmuncompress DESTINATION ${RPM_CONFIGDIR}) +- ++install(TARGETS rpmdeps rpmuncompress rpm2extents DESTINATION ${RPM_CONFIGDIR}) +diff --git a/tools/rpm2extents.c b/tools/rpm2extents.c +new file mode 100644 +index 0000000..98124c2 +--- /dev/null ++++ b/tools/rpm2extents.c +@@ -0,0 +1,707 @@ ++/* rpm2extents: convert payload to inline extents */ ++ ++#include "system.h" ++ ++#include ++#include /* rpmReadPackageFile .. */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "rpmlead.h" ++#include "signature.h" ++#include "header_internal.h" ++#include "rpmio_internal.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "debug.h" ++ ++/* hash of void * (pointers) to file digests to offsets within output. ++ * The length of the key depends on what the FILEDIGESTALGO is. ++ */ ++#undef HASHTYPE ++#undef HTKEYTYPE ++#undef HTDATATYPE ++#define HASHTYPE digestSet ++#define HTKEYTYPE const unsigned char * ++#include "rpmhash.H" ++#include "rpmhash.C" ++ ++struct digestoffset { ++ const unsigned char * digest; ++ rpm_loff_t pos; ++}; ++ ++rpm_loff_t pad_to(rpm_loff_t pos, rpm_loff_t unit); ++ ++rpm_loff_t pad_to(rpm_loff_t pos, rpm_loff_t unit) ++{ ++ return (unit - (pos % unit)) % unit; ++} ++ ++static struct poptOption optionsTable[] = { ++ { NULL, '\0', POPT_ARG_INCLUDE_TABLE, rpmcliAllPoptTable, 0, ++ N_("Common options for all rpm modes and executables:"), NULL }, ++ ++ POPT_AUTOALIAS ++ POPT_AUTOHELP ++ POPT_TABLEEND ++}; ++ ++ ++static void FDDigestInit(FD_t fdi, uint8_t algos[], uint32_t algos_len){ ++ int algo; ++ ++ for (algo = 0; algo < algos_len; algo++) { ++ fdInitDigest(fdi, algos[algo], 0); ++ } ++} ++ ++static int FDWriteDigests( ++ FD_t fdi, ++ FD_t fdo, ++ uint8_t algos[], ++ uint32_t algos_len) ++{ ++ const char *filedigest, *algo_name; ++ size_t filedigest_len, len; ++ uint32_t algo_name_len, algo_digest_len; ++ int algo; ++ rpmRC rc = RPMRC_FAIL; ++ ++ ssize_t fdilength = fdOp(fdi, FDSTAT_READ)->bytes; ++ ++ len = sizeof(fdilength); ++ if (Fwrite(&fdilength, len, 1, fdo) != len) { ++ rpmlog(RPMLOG_ERR, _("Unable to write input length %zd: %d, %s\n"), ++ fdilength, errno, strerror(errno)); ++ goto exit; ++ } ++ len = sizeof(algos_len); ++ if (Fwrite(&algos_len, len, 1, fdo) != len) { ++ algo_digest_len = (uint32_t)filedigest_len; ++ rpmlog(RPMLOG_ERR, _("Unable to write number of digests: %d, %s\n"), ++ errno, strerror(errno)); ++ goto exit; ++ } ++ for (algo = 0; algo < algos_len; algo++) { ++ fdFiniDigest(fdi, algos[algo], (void **)&filedigest, &filedigest_len, 0); ++ ++ algo_name = pgpValString(PGPVAL_HASHALGO, algos[algo]); ++ algo_name_len = (uint32_t)strlen(algo_name); ++ algo_digest_len = (uint32_t)filedigest_len; ++ ++ len = sizeof(algo_name_len); ++ if (Fwrite(&algo_name_len, len, 1, fdo) != len) { ++ rpmlog(RPMLOG_ERR, ++ _("Unable to write digest algo name length: %d, %s\n"), ++ errno, strerror(errno)); ++ goto exit; ++ } ++ len = sizeof(algo_digest_len); ++ if (Fwrite(&algo_digest_len, len, 1, fdo) != len) { ++ rpmlog(RPMLOG_ERR, ++ _("Unable to write number of bytes for digest: %d, %s\n"), ++ errno, strerror(errno)); ++ goto exit; ++ } ++ if (Fwrite(algo_name, algo_name_len, 1, fdo) != algo_name_len) { ++ rpmlog(RPMLOG_ERR, _("Unable to write digest algo name: %d, %s\n"), ++ errno, strerror(errno)); ++ goto exit; ++ } ++ if (Fwrite(filedigest, algo_digest_len, 1, fdo ) != algo_digest_len) { ++ rpmlog(RPMLOG_ERR, ++ _("Unable to write digest value %u, %zu: %d, %s\n"), ++ algo_digest_len, filedigest_len, ++ errno, strerror(errno)); ++ goto exit; ++ } ++ } ++ rc = RPMRC_OK; ++exit: ++ return rc; ++} ++ ++/** ++ * Check if package is in deny list. ++ * @param package_name package name ++ * @return true if package is in deny list ++ */ ++static inline int isInDenyList(char *package_name) ++{ ++ int is_in_deny_list = 0; ++ if (package_name) { ++ char *e_denylist = getenv("LIBREPO_TRANSCODE_RPMS_DENYLIST"); ++ char *denytlist_item = strtok(e_denylist, ","); ++ while (denytlist_item) { ++ if (strstr(package_name, denytlist_item)) { ++ is_in_deny_list = 1; ++ break; ++ } ++ denytlist_item = strtok(NULL, ","); ++ } ++ } ++ return is_in_deny_list; ++} ++ ++static rpmRC FDWriteSignaturesValidation(FD_t fdo, int rpmvsrc, char *msg) { ++ size_t len; ++ rpmRC rc = RPMRC_FAIL; ++ ++ if(rpmvsrc){ ++ rpmlog(RPMLOG_WARNING, ++ _("Error verifying package signatures:\n%s\n"), msg); ++ } ++ ++ len = sizeof(rpmvsrc); ++ if (Fwrite(&rpmvsrc, len, 1, fdo) != len) { ++ rpmlog(RPMLOG_ERR, ++ _("Unable to write signature verification RC code %d: %d, %s\n"), ++ rpmvsrc, errno, strerror(errno)); ++ goto exit; ++ } ++ size_t content_len = msg ? strlen(msg) : 0; ++ len = sizeof(content_len); ++ if (Fwrite(&content_len, len, 1, fdo) != len) { ++ rpmlog(RPMLOG_ERR, ++ _("Unable to write signature verification output length %zd: %d, %s\n"), ++ content_len, errno, strerror(errno)); ++ goto exit; ++ } ++ if (Fwrite(msg, content_len, 1, fdo) != content_len) { ++ rpmlog(RPMLOG_ERR, ++ _("Unable to write signature verification output %s: %d, %s\n"), ++ msg, errno, strerror(errno)); ++ goto exit; ++ } ++ ++ rc = RPMRC_OK; ++exit: ++ ++ return rc; ++} ++ ++static rpmRC validator(FD_t fdi, FD_t digesto, FD_t sigo, ++ uint8_t algos[], ++ uint32_t algos_len){ ++ int rpmvsrc; ++ rpmRC rc = RPMRC_FAIL; ++ char *msg = NULL; ++ rpmts ts = rpmtsCreate(); ++ ++ rpmtsSetRootDir(ts, rpmcliRootDir); ++ ++ FDDigestInit(fdi, algos, algos_len); ++ ++ rpmvsrc = rpmcliVerifySignaturesFD(ts, fdi, &msg); ++ ++ // Write result of digest computation ++ if(FDWriteDigests(fdi, digesto, algos, algos_len) != RPMRC_OK) { ++ rpmlog(RPMLOG_ERR, _("Failed to write digests: %d, %s\n"), ++ errno, strerror(errno)); ++ goto exit; ++ } ++ ++ // Write result of signature validation. ++ if(FDWriteSignaturesValidation(sigo, rpmvsrc, msg)) { ++ rpmlog(RPMLOG_ERR, ++ _("Failed to write signature verification result: %d, %s\n"), ++ errno, strerror(errno)); ++ goto exit; ++ } ++ rc = RPMRC_OK; ++exit: ++ if(msg) { ++ free(msg); ++ } ++ rpmtsFree(ts); ++ return rc; ++} ++ ++static void sanitizeSignatureHeader(Header * sigh) ++{ ++ struct rpmtd_s td; ++ ++ /* This is inspired by the code in unloadImmutableRegion. See https://github.com/rpm-software-management/rpm/pull/1330 */ ++ if (!headerGet(*sigh, RPMTAG_HEADERSIGNATURES, &td, HEADERGET_DEFAULT)) { ++ /* Signature header corrupt/missing */ ++ rpmlog(RPMLOG_WARNING, _("Error verifying signature header\n")); ++ rpmtdFreeData(&td); ++ Header nh = headerCopy(*sigh); ++ headerFree(*sigh); ++ *sigh = headerLink(nh); ++ headerFree(nh); ++ } ++ rpmtdFreeData(&td); ++} ++ ++static rpmRC process_package(FD_t fdi, FD_t digestori, FD_t validationi) ++{ ++ uint32_t diglen; ++ /* GNU C extension: can use diglen from outer context */ ++ int digestSetCmp(const unsigned char * a, const unsigned char * b) { ++ return memcmp(a, b, diglen); ++ } ++ ++ unsigned int digestSetHash(const unsigned char * digest) { ++ /* assumes sizeof(unsigned int) < diglen */ ++ return *(unsigned int *)digest; ++ } ++ ++ int digestoffsetCmp(const void * a, const void * b) { ++ return digestSetCmp( ++ ((struct digestoffset *)a)->digest, ++ ((struct digestoffset *)b)->digest ++ ); ++ } ++ ++ FD_t fdo; ++ FD_t gzdi; ++ Header h, sigh; ++ long fundamental_block_size = sysconf(_SC_PAGESIZE); ++ rpmRC rc = RPMRC_OK; ++ rpm_mode_t mode; ++ char *rpmio_flags = NULL, *zeros; ++ const unsigned char *digest; ++ rpm_loff_t pos, size, pad, digest_pos, validation_pos, digest_table_pos; ++ uint32_t offset_ix = 0; ++ size_t len; ++ int next = 0; ++ struct rpmlead_s l; ++ rpmfiles files = NULL; ++ rpmfi fi = NULL; ++ char *msg = NULL; ++ struct digestoffset *offsets = NULL; ++ digestSet ds = NULL; ++ ++ fdo = fdDup(STDOUT_FILENO); ++ ++ rc = rpmLeadReadAndReturn(fdi, &msg, &l); ++ if (rc != RPMRC_OK) ++ goto exit; ++ ++ /* Skip conversion if package is in deny list */ ++ if (isInDenyList(l.name)) { ++ rpmlog(RPMLOG_WARNING, _("package %s is in deny list: conversion skipped\n"), l.name); ++ if (rpmLeadWrite(fdo, l)) { ++ rpmlog(RPMLOG_ERR, _("Unable to write package lead: %s\n"), ++ Fstrerror(fdo)); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ ++ ssize_t fdilength = ufdCopy(fdi, fdo); ++ if (fdilength == -1) { ++ rpmlog(RPMLOG_ERR, _("process_package cat failed\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ ++ goto exit; ++ } else { ++ if (rpmReadPackageRaw(fdi, &sigh, &h)) { ++ rpmlog(RPMLOG_ERR, _("Error reading package\n")); ++ exit(EXIT_FAILURE); ++ } ++ ++ sanitizeSignatureHeader(&sigh); ++ ++ if (rpmLeadWriteFromHeader(fdo, h)) { ++ rpmlog(RPMLOG_ERR, _("Unable to write package lead: %s\n"), ++ Fstrerror(fdo)); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (rpmWriteSignature(fdo, sigh)) { ++ rpmlog(RPMLOG_ERR, _("Unable to write signature: %s\n"), ++ Fstrerror(fdo)); ++ exit(EXIT_FAILURE); ++ } ++ ++ if (headerWrite(fdo, h, HEADER_MAGIC_YES)) { ++ rpmlog(RPMLOG_ERR, _("Unable to write headers: %s\n"), ++ Fstrerror(fdo)); ++ exit(EXIT_FAILURE); ++ } ++ ++ /* Retrieve payload size and compression type. */ ++ { ++ const char *compr = ++ headerGetString(h, RPMTAG_PAYLOADCOMPRESSOR); ++ rpmio_flags = ++ rstrscat(NULL, "r.", compr ? compr : "gzip", NULL); ++ } ++ ++ gzdi = Fdopen(fdi, rpmio_flags); /* XXX gzdi == fdi */ ++ free(rpmio_flags); ++ ++ if (gzdi == NULL) { ++ rpmlog(RPMLOG_ERR, _("cannot re-open payload: %s\n"), ++ Fstrerror(gzdi)); ++ exit(EXIT_FAILURE); ++ } ++ ++ files = rpmfilesNew(NULL, h, 0, RPMFI_KEEPHEADER); ++ fi = rpmfiNewArchiveReader(gzdi, files, ++ RPMFI_ITER_READ_ARCHIVE_CONTENT_FIRST); ++ ++ /* this is encoded in the file format, so needs to be fixed size (for ++ * now?) ++ */ ++ diglen = (uint32_t) rpmDigestLength(rpmfiDigestAlgo(fi)); ++ ds = digestSetCreate(rpmfiFC(fi), digestSetHash, digestSetCmp, NULL); ++ offsets = xcalloc(rpmfiFC(fi), sizeof(*offsets)); ++ pos = RPMLEAD_SIZE + headerSizeof(sigh, HEADER_MAGIC_YES); ++ ++ /* main headers are aligned to 8 byte boundry */ ++ pos += pad_to(pos, 8); ++ pos += headerSizeof(h, HEADER_MAGIC_YES); ++ ++ zeros = xcalloc(fundamental_block_size, 1); ++ ++ while (next >= 0) { ++ next = rpmfiNext(fi); ++ if (next == RPMERR_ITER_END) { ++ rc = RPMRC_OK; ++ break; ++ } ++ mode = rpmfiFMode(fi); ++ if (!S_ISREG(mode) || !rpmfiArchiveHasContent(fi)) { ++ /* not a regular file, or the archive doesn't contain any content ++ * for this entry. ++ */ ++ continue; ++ } ++ digest = rpmfiFDigest(fi, NULL, NULL); ++ if (digestSetGetEntry(ds, digest, NULL)) { ++ /* This specific digest has already been included, so skip it. */ ++ continue; ++ } ++ pad = pad_to(pos, fundamental_block_size); ++ if (Fwrite(zeros, sizeof(char), pad, fdo) != pad) { ++ rpmlog(RPMLOG_ERR, _("Unable to write padding\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ /* round up to next fundamental_block_size */ ++ pos += pad; ++ digestSetAddEntry(ds, digest); ++ offsets[offset_ix].digest = digest; ++ offsets[offset_ix].pos = pos; ++ offset_ix++; ++ size = rpmfiFSize(fi); ++ rc = rpmfiArchiveReadToFile(fi, fdo, 0); ++ if (rc != RPMRC_OK) { ++ char *errstr = rpmfileStrerror(rc); ++ rpmlog(RPMLOG_ERR, ++ _("rpmfiArchiveReadToFile failed while extracting " ++ "\"%s\" with RC %d: %s\n"), ++ rpmfiFN(fi), rc, errstr); ++ free(errstr); ++ goto exit; ++ } ++ pos += size; ++ } ++ Fclose(gzdi); /* XXX gzdi == fdi */ ++ ++ qsort(offsets, (size_t) offset_ix, sizeof(struct digestoffset), ++ digestoffsetCmp); ++ ++ validation_pos = pos; ++ ssize_t validation_len = ufdCopy(validationi, fdo); ++ if (validation_len == -1) { ++ rpmlog(RPMLOG_ERR, _("validation output ufdCopy failed\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ ++ digest_table_pos = validation_pos + validation_len; ++ ++ len = sizeof(offset_ix); ++ if (Fwrite(&offset_ix, len, 1, fdo) != len) { ++ rpmlog(RPMLOG_ERR, _("Unable to write length of table\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ len = sizeof(diglen); ++ if (Fwrite(&diglen, len, 1, fdo) != len) { ++ rpmlog(RPMLOG_ERR, _("Unable to write length of digest\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ len = sizeof(rpm_loff_t); ++ for (int x = 0; x < offset_ix; x++) { ++ if (Fwrite(offsets[x].digest, diglen, 1, fdo) != diglen) { ++ rpmlog(RPMLOG_ERR, _("Unable to write digest\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ if (Fwrite(&offsets[x].pos, len, 1, fdo) != len) { ++ rpmlog(RPMLOG_ERR, _("Unable to write offset\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ } ++ digest_pos = ++ (digest_table_pos + sizeof(offset_ix) + sizeof(diglen) + ++ offset_ix * (diglen + sizeof(rpm_loff_t)) ++ ); ++ ++ ssize_t digest_len = ufdCopy(digestori, fdo); ++ if (digest_len == -1) { ++ rpmlog(RPMLOG_ERR, _("digest table ufdCopy failed\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ ++ /* add more padding so the last file can be cloned. It doesn't matter that ++ * the table and validation etc are in this space. In fact, it's pretty ++ * efficient if it is. ++ */ ++ ++ pad = ++ pad_to((validation_pos + validation_len + ++ 2 * sizeof(rpm_loff_t) + sizeof(uint64_t)), ++ fundamental_block_size); ++ if (Fwrite(zeros, sizeof(char), pad, fdo) != pad) { ++ rpmlog(RPMLOG_ERR, _("Unable to write final padding\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ zeros = _free(zeros); ++ struct extents_footer_t footer = {.offsets = ++ { validation_pos, digest_table_pos, digest_pos },.magic = ++ EXTENTS_MAGIC }; ++ len = sizeof(footer); ++ if (Fwrite(&footer, len, 1, fdo) != len) { ++ rpmlog(RPMLOG_ERR, _("Unable to write footer\n")); ++ rc = RPMRC_FAIL; ++ goto exit; ++ } ++ } ++ ++ exit: ++ rpmfilesFree(files); ++ rpmfiFree(fi); ++ headerFree(h); ++ headerFree(sigh); ++ free(offsets); ++ Fclose(fdo); ++ digestSetFree(ds); ++ return rc; ++} ++ ++static off_t ufdTee(FD_t sfd, FD_t *fds, int len) ++{ ++ char buf[BUFSIZ]; ++ ssize_t rdbytes, wrbytes; ++ off_t total = 0; ++ ++ while (1) { ++ rdbytes = Fread(buf, sizeof(buf[0]), sizeof(buf), sfd); ++ ++ if (rdbytes > 0) { ++ for(int i=0; i < len; i++) { ++ wrbytes = Fwrite(buf, sizeof(buf[0]), rdbytes, fds[i]); ++ if (wrbytes != rdbytes) { ++ rpmlog(RPMLOG_ERR, ++ _("Error wriing to FD %d: %s\n"), ++ i, Fstrerror(fds[i])); ++ total = -1; ++ break; ++ } ++ } ++ if(total == -1){ ++ break; ++ } ++ total += wrbytes; ++ } else { ++ if (rdbytes < 0) ++ total = -1; ++ break; ++ } ++ } ++ ++ return total; ++} ++ ++static rpmRC teeRpm(FD_t fdi, uint8_t algos[], uint32_t algos_len) { ++ rpmRC rc = RPMRC_FAIL; ++ off_t offt = -1; ++ // tee-ed stdin ++ int processorpipefd[2]; ++ int validatorpipefd[2]; ++ // metadata ++ int meta_digestpipefd[2]; ++ int meta_rpmsignpipefd[2]; ++ ++ pid_t cpids[2], w; ++ int wstatus; ++ FD_t fds[2]; ++ ++ if (pipe(processorpipefd) == -1) { ++ rpmlog(RPMLOG_ERR, _("Processor pipe failure\n")); ++ return RPMRC_FAIL; ++ } ++ ++ if (pipe(validatorpipefd) == -1) { ++ rpmlog(RPMLOG_ERR, _("Validator pipe failure\n")); ++ return RPMRC_FAIL; ++ } ++ ++ if (pipe(meta_digestpipefd) == -1) { ++ rpmlog(RPMLOG_ERR, _("Meta digest pipe failure\n")); ++ return RPMRC_FAIL; ++ } ++ ++ if (pipe(meta_rpmsignpipefd) == -1) { ++ rpmlog(RPMLOG_ERR, _("Meta rpm signature pipe failure\n")); ++ return RPMRC_FAIL; ++ } ++ ++ cpids[0] = fork(); ++ if (cpids[0] == 0) { ++ /* child: validator */ ++ close(processorpipefd[0]); ++ close(processorpipefd[1]); ++ close(validatorpipefd[1]); ++ close(meta_digestpipefd[0]); ++ close(meta_rpmsignpipefd[0]); ++ FD_t fdi = fdDup(validatorpipefd[0]); ++ FD_t digesto = fdDup(meta_digestpipefd[1]); ++ FD_t sigo = fdDup(meta_rpmsignpipefd[1]); ++ close(meta_digestpipefd[1]); ++ close(meta_rpmsignpipefd[1]); ++ rc = validator(fdi, digesto, sigo, algos, algos_len); ++ if(rc != RPMRC_OK) { ++ rpmlog(RPMLOG_ERR, _("Validator failed with RC %d\n"), rc); ++ } ++ Fclose(fdi); ++ Fclose(digesto); ++ Fclose(sigo); ++ if (rc != RPMRC_OK) { ++ exit(EXIT_FAILURE); ++ } ++ exit(EXIT_SUCCESS); ++ } else { ++ /* parent: main program */ ++ cpids[1] = fork(); ++ if (cpids[1] == 0) { ++ /* child: process_package */ ++ close(validatorpipefd[0]); ++ close(validatorpipefd[1]); ++ close(processorpipefd[1]); ++ close(meta_digestpipefd[1]); ++ close(meta_rpmsignpipefd[1]); ++ FD_t fdi = fdDup(processorpipefd[0]); ++ close(processorpipefd[0]); ++ FD_t sigi = fdDup(meta_rpmsignpipefd[0]); ++ close(meta_rpmsignpipefd[0]); ++ FD_t digestori = fdDup(meta_digestpipefd[0]); ++ close(meta_digestpipefd[0]); ++ ++ rc = process_package(fdi, digestori, sigi); ++ if(rc != RPMRC_OK) { ++ rpmlog(RPMLOG_ERR, _("Package processor failed: %d\n"), rc); ++ } ++ Fclose(digestori); ++ Fclose(sigi); ++ /* fdi is normally closed through the stacked file gzdi in the ++ * function ++ */ ++ ++ if (rc != RPMRC_OK) { ++ exit(EXIT_FAILURE); ++ } ++ exit(EXIT_SUCCESS); ++ ++ ++ } else { ++ /* Actual parent. Read from fdi and write to both processes */ ++ close(processorpipefd[0]); ++ close(validatorpipefd[0]); ++ fds[0] = fdDup(processorpipefd[1]); ++ fds[1] = fdDup(validatorpipefd[1]); ++ close(validatorpipefd[1]); ++ close(processorpipefd[1]); ++ close(meta_digestpipefd[0]); ++ close(meta_digestpipefd[1]); ++ close(meta_rpmsignpipefd[0]); ++ close(meta_rpmsignpipefd[1]); ++ ++ rc = RPMRC_OK; ++ offt = ufdTee(fdi, fds, 2); ++ if(offt == -1){ ++ rpmlog(RPMLOG_ERR, _("Failed to tee RPM\n")); ++ rc = RPMRC_FAIL; ++ } ++ Fclose(fds[0]); ++ Fclose(fds[1]); ++ w = waitpid(cpids[0], &wstatus, 0); ++ if (w == -1) { ++ rpmlog(RPMLOG_ERR, _("waitpid cpids[0] failed\n")); ++ rc = RPMRC_FAIL; ++ } ++ w = waitpid(cpids[1], &wstatus, 0); ++ if (w == -1) { ++ rpmlog(RPMLOG_ERR, _("waitpid cpids[1] failed\n")); ++ rc = RPMRC_FAIL; ++ } ++ } ++ } ++ ++ return rc; ++} ++ ++int main(int argc, char *argv[]) { ++ rpmRC rc; ++ poptContext optCon = NULL; ++ const char **args = NULL; ++ int nb_algos = 0; ++ ++ xsetprogname(argv[0]); /* Portability call -- see system.h */ ++ rpmReadConfigFiles(NULL, NULL); ++ optCon = rpmcliInit(argc, argv, optionsTable); ++ poptSetOtherOptionHelp(optCon, "[OPTIONS]* "); ++ ++ if (poptPeekArg(optCon) == NULL) { ++ rpmlog(RPMLOG_ERR, ++ _("Need at least one DIGESTALGO parameter, e.g. 'SHA256'\n")); ++ poptPrintUsage(optCon, stderr, 0); ++ exit(EXIT_FAILURE); ++ } ++ ++ args = poptGetArgs(optCon); ++ ++ for (nb_algos=0; args[nb_algos]; nb_algos++); ++ uint8_t algos[nb_algos]; ++ for (int x = 0; x < nb_algos; x++) { ++ if (pgpStringVal(PGPVAL_HASHALGO, args[x], &algos[x]) != 0) ++ { ++ rpmlog(RPMLOG_ERR, ++ _("Unable to resolve '%s' as a digest algorithm, exiting\n"), ++ args[x]); ++ exit(EXIT_FAILURE); ++ } ++ } ++ ++ FD_t fdi = fdDup(STDIN_FILENO); ++ rc = teeRpm(fdi, algos, nb_algos); ++ Fclose(fdi); ++ if (rc != RPMRC_OK) { ++ /* translate rpmRC into generic failure return code. */ ++ return EXIT_FAILURE; ++ } ++ return EXIT_SUCCESS; ++} +-- +2.47.1 + diff --git a/rpm.spec b/rpm.spec index 29a3e59..8bf409a 100644 --- a/rpm.spec +++ b/rpm.spec @@ -27,7 +27,7 @@ %global rpmver 4.19.1.1 #global snapver rc1 -%global baserelease 9 +%global baserelease 9.1 %global sover 10 %global srcver %{rpmver}%{?snapver:-%{snapver}} @@ -149,6 +149,10 @@ rpm-4.18.90-weak-user-group.patch # These are not yet upstream rpm-4.7.1-geode-i686.patch +# CoW +0001-RPM-with-Copy-on-Write.patch +0001-Measure-plugin.patch + %description The RPM Package Manager (RPM) is a powerful command line driven package management system capable of installing, uninstalling, @@ -353,6 +357,20 @@ system DBUS - like packages installed or removed. Other programs can subscribe to the signals to get notified when packages on the system change. +%package plugin-reflink +Summary: Rpm plugin for reflink functionality +Requires: rpm-libs%{_isa} = %{version}-%{release} + +%description plugin-reflink +%{summary}. + +%package plugin-measure +Summary: Rpm plugin for measure +Requires: rpm-libs%{_isa} = %{version}-%{release} + +%description plugin-measure +Adds measure support + # with plugins %endif @@ -563,6 +581,14 @@ fi %{_datadir}/dbus-1/system.d/org.rpm.conf %endif +%files plugin-reflink +%{rpmhome}/rpm2extents +%{rpmhome}/rpm2extents_dump +%{_libdir}/rpm-plugins/reflink.so + +%files plugin-measure +%{_libdir}/rpm-plugins/measure.so + %files build-libs %{_libdir}/librpmbuild.so.%{sover} %{_libdir}/librpmbuild.so.%{sover}.* @@ -628,6 +654,9 @@ fi %doc %{_defaultdocdir}/rpm/API/ %changelog +* Thu Dec 05 2024 Matteo Croce - 4.19.1.1-9.1 +- Port CoW and measure plugins to 4.19 + * Wed Dec 04 2024 Panu Matilainen - 4.19.1.1-9 - Revert the gnupg/sequoia sub-packages, too much headache - Ship sequoia-signing enablement macros as documentation instead