rpms / rpm

Clone
Source Code
GIT

#9 Backport multi-threaded zstd compression to 4.16.x. Similar to https://git.centos.org/rpms/rpm/pull-request/6
Merged 2 years ago by dcavalca. Opened 2 years ago by alexk.
rpms/ alexk/rpm c9s-sig-hyperscale  into  c9s-sig-hyperscale

file added
+193 
@@ -0,0 +1,193 @@ 
 

+ From 8c0d382a8c62ec6f4f5d69920b76cd4f6bb24e9b Mon Sep 17 00:00:00 2001
 

+ From: Aleksandr Kazakov <alexkazakov@meta.com>
 

+ Date: Mon, 6 Feb 2023 19:15:45 +0000
 

+ Subject: [PATCH] Backport multi-threaded zstd to 4.16.x
 

+ 

+ Signed-off-by: Aleksandr Kazakov <alexkazakov@meta.com>
 

+ ---
 

+  configure.ac  |  2 +-
 

+  macros.in     | 15 ++++-----
 

+  rpmio/rpmio.c | 84 ++++++++++++++++++++++++++++++++++-----------------
 

+  3 files changed, 66 insertions(+), 35 deletions(-)
 

+ 

+ diff --git a/configure.ac b/configure.ac
 

+ index 71e9009..28979af 100644
 

+ --- a/configure.ac
 

+ +++ b/configure.ac
 

+ @@ -215,7 +215,7 @@ AC_ARG_ENABLE([zstd],
 

+                [enable_zstd=auto])
 

+  

+  AS_IF([test "x$enable_zstd" != "xno"], [
 

+ -  PKG_CHECK_MODULES([ZSTD], [libzstd], [have_zstd=yes], [have_zstd=no])
 

+ +  PKG_CHECK_MODULES([ZSTD], [libzstd >= 1.3.8], [have_zstd=yes], [have_zstd=no])
 

+    AS_IF([test "$enable_zstd" = "yes"], [
 

+      if test "$have_zstd" = "no"; then
 

+        AC_MSG_ERROR([--enable-zstd specified, but not available])
 

+ diff --git a/macros.in b/macros.in
 

+ index 99eb034..fa85fdb 100644
 

+ --- a/macros.in
 

+ +++ b/macros.in
 

+ @@ -365,13 +365,14 @@ package or when debugging this package.\
 

+  #%packager
 

+  

+  #	Compression type and level for source/binary package payloads.
 

+ -#		"w9.gzdio"	gzip level 9 (default).
 

+ -#		"w9.bzdio"	bzip2 level 9.
 

+ -#		"w6.xzdio"	xz level 6, xz's default.
 

+ -#		"w7T16.xzdio"	xz level 7 using 16 thread (xz only)
 

+ -#		"w6.lzdio"	lzma-alone level 6, lzma's default
 

+ -#		"w3.zstdio"	zstd level 3, zstd's default
 

+ -#		"w.ufdio"	uncompressed
 

+ +#		"w9.gzdio"		gzip level 9 (default).
 

+ +#		"w9.bzdio"		bzip2 level 9.
 

+ +#		"w6.xzdio"		xz level 6, xz's default.
 

+ +#		"w7T16.xzdio"		xz level 7 using 16 threads
 

+ +#		"w19T8.zstdio"		zstd level 19 using 8 threads
 

+ +#		"w6.lzdio"		lzma-alone level 6, lzma's default
 

+ +#		"w3.zstdio"		zstd level 3, zstd's default
 

+ +#		"w.ufdio"		uncompressed
 

+  #
 

+  #%_source_payload	w9.gzdio
 

+  #%_binary_payload	w9.gzdio
 

+ diff --git a/rpmio/rpmio.c b/rpmio/rpmio.c
 

+ index 52be6a3..b5c76a8 100644
 

+ --- a/rpmio/rpmio.c
 

+ +++ b/rpmio/rpmio.c
 

+ @@ -1073,6 +1073,7 @@ static rpmzstd rpmzstdNew(int fdno, const char *fmode)
 

+      char *t = stdio;
 

+      char *te = t + sizeof(stdio) - 2;
 

+      int c;
 

+ +    int threads = 0;
 

+  

+      switch ((c = *s++)) {
 

+      case 'a':
 

+ @@ -1101,7 +1102,14 @@ static rpmzstd rpmzstdNew(int fdno, const char *fmode)
 

+  	    flags &= ~O_ACCMODE;
 

+  	    flags |= O_RDWR;
 

+  	    continue;
 

+ -	    break;
 

+ +	case 'T':
 

+ +	    if (*s >= '0' && *s <= '9') {
 

+ +		threads = strtol(s, (char **)&s, 10);
 

+ +		/* T0 means automatic detection */
 

+ +		if (threads == 0)
 

+ +		    threads = sysconf(_SC_NPROCESSORS_ONLN);
 

+ +	    }
 

+ +	    continue;
 

+  	default:
 

+  	    if (c >= (int)'0' && c <= (int)'9') {
 

+  		level = strtol(s-1, (char **)&s, 10);
 

+ @@ -1135,11 +1143,17 @@ static rpmzstd rpmzstdNew(int fdno, const char *fmode)
 

+  	}
 

+  	nb = ZSTD_DStreamInSize();
 

+      } else {					/* compressing */
 

+ -	if ((_stream = (void *) ZSTD_createCStream()) == NULL
 

+ -	 || ZSTD_isError(ZSTD_initCStream(_stream, level))) {
 

+ +	if ((_stream = (void *) ZSTD_createCCtx()) == NULL
 

+ +	 || ZSTD_isError(ZSTD_CCtx_setParameter(_stream, ZSTD_c_compressionLevel, level))) {
 

+  	    goto err;
 

+  	}
 

+ -	nb = ZSTD_CStreamOutSize();
 

+ +
 

+ +        rpmlog(RPMLOG_DEBUG, "using %i threads in zstd compression\n", threads);
 

+ +        if (threads > 0) {
 

+ +            if (ZSTD_isError (ZSTD_CCtx_setParameter(_stream, ZSTD_c_nbWorkers, threads)))
 

+ +            rpmlog(RPMLOG_WARNING, "zstd library does not support multi-threading\n");
 

+ +	}
 

+ + 	nb = ZSTD_CStreamOutSize();
 

+      }
 

+  

+      rpmzstd zstd = (rpmzstd) xcalloc(1, sizeof(*zstd));
 

+ @@ -1158,7 +1172,7 @@ err:
 

+      if ((flags & O_ACCMODE) == O_RDONLY)
 

+  	ZSTD_freeDStream(_stream);
 

+      else
 

+ -	ZSTD_freeCStream(_stream);
 

+ +	ZSTD_freeCCtx(_stream);
 

+      return NULL;
 

+  }
 

+  

+ @@ -1184,16 +1198,24 @@ assert(zstd);
 

+  	rc = 0;
 

+      } else {					/* compressing */
 

+  	/* close frame */
 

+ -	zstd->zob.dst  = zstd->b;
 

+ -	zstd->zob.size = zstd->nb;
 

+ -	zstd->zob.pos  = 0;
 

+ -	int xx = ZSTD_flushStream(zstd->_stream, &zstd->zob);
 

+ -	if (ZSTD_isError(xx))
 

+ -	    fps->errcookie = ZSTD_getErrorName(xx);
 

+ -	else if (zstd->zob.pos != fwrite(zstd->b, 1, zstd->zob.pos, zstd->fp))
 

+ -	    fps->errcookie = "zstdFlush fwrite failed.";
 

+ -	else
 

+ -	    rc = 0;
 

+ +	int xx;
 

+ +	do {
 

+ +	  ZSTD_inBuffer zib = { NULL, 0, 0 };
 

+ +	  zstd->zob.dst  = zstd->b;
 

+ +	  zstd->zob.size = zstd->nb;
 

+ +	  zstd->zob.pos  = 0;
 

+ +	  xx = ZSTD_compressStream2(zstd->_stream, &zstd->zob, &zib, ZSTD_e_flush);
 

+ +	  if (ZSTD_isError(xx)) {
 

+ +	      fps->errcookie = ZSTD_getErrorName(xx);
 

+ +	      break;
 

+ +	  }
 

+ +	  else if (zstd->zob.pos != fwrite(zstd->b, 1, zstd->zob.pos, zstd->fp)) {
 

+ +	      fps->errcookie = "zstdClose fwrite failed.";
 

+ +	      break;
 

+ +	  }
 

+ +	  else
 

+ +	      rc = 0;
 

+ +	} while (xx != 0);
 

+      }
 

+      return rc;
 

+  }
 

+ @@ -1238,7 +1260,7 @@ assert(zstd);
 

+  	zstd->zob.pos  = 0;
 

+  

+  	/* Compress next chunk. */
 

+ -        int xx = ZSTD_compressStream(zstd->_stream, &zstd->zob, &zib);
 

+ +        int xx = ZSTD_compressStream2(zstd->_stream, &zstd->zob, &zib, ZSTD_e_continue);
 

+          if (ZSTD_isError(xx)) {
 

+  	    fps->errcookie = ZSTD_getErrorName(xx);
 

+  	    return -1;
 

+ @@ -1267,17 +1289,25 @@ assert(zstd);
 

+  	ZSTD_freeDStream(zstd->_stream);
 

+      } else {					/* compressing */
 

+  	/* close frame */
 

+ -	zstd->zob.dst  = zstd->b;
 

+ -	zstd->zob.size = zstd->nb;
 

+ -	zstd->zob.pos  = 0;
 

+ -	int xx = ZSTD_endStream(zstd->_stream, &zstd->zob);
 

+ -	if (ZSTD_isError(xx))
 

+ -	    fps->errcookie = ZSTD_getErrorName(xx);
 

+ -	else if (zstd->zob.pos != fwrite(zstd->b, 1, zstd->zob.pos, zstd->fp))
 

+ -	    fps->errcookie = "zstdClose fwrite failed.";
 

+ -	else
 

+ -	    rc = 0;
 

+ -	ZSTD_freeCStream(zstd->_stream);
 

+ +	int xx;
 

+ +	do {
 

+ +	  ZSTD_inBuffer zib = { NULL, 0, 0 };
 

+ +	  zstd->zob.dst  = zstd->b;
 

+ +	  zstd->zob.size = zstd->nb;
 

+ +	  zstd->zob.pos  = 0;
 

+ +	  xx = ZSTD_compressStream2(zstd->_stream, &zstd->zob, &zib, ZSTD_e_end);
 

+ +	  if (ZSTD_isError(xx)) {
 

+ +	      fps->errcookie = ZSTD_getErrorName(xx);
 

+ +	      break;
 

+ +	  }
 

+ +	  else if (zstd->zob.pos != fwrite(zstd->b, 1, zstd->zob.pos, zstd->fp)) {
 

+ +	      fps->errcookie = "zstdClose fwrite failed.";
 

+ +	      break;
 

+ +	  }
 

+ +	  else
 

+ +	      rc = 0;
 

+ +	} while (xx != 0);
 

+ +	ZSTD_freeCCtx(zstd->_stream);
 

+      }
 

+  

+      if (zstd->fp && fileno(zstd->fp) > 2)
 

+ -- 

+ 2.38.1
 

+

file modified
+8 -2 
@@ -42,7 +42,7 @@ 
 

  

  %global rpmver 4.16.1.3
 

  #global snapver rc1
 

- %global rel 22.1
 

+ %global rel 22.2
 

  %global sover 9
 

  

  %global srcver %{rpmver}%{?snapver:-%{snapver}}
 
@@ -132,6 +132,9 @@ 
 

  # Downstream-only patches
 

  Patch1000: rpm-4.16.1.3-hashtab-use-after-free-fix.patch
 

  Patch1001: rpm-4.16.1.3-find_debuginfo_vendor_opts.patch
 

+ %if %{with zstd}
 

+ Patch1002: rpm-4.16.1.3-backport-multithreaded-zstd.patch
 

+ %endif
 

  

  # fsverity support
 

  %if %{with libfsverity}
 
@@ -329,7 +332,7 @@ 
 

  Requires: findutils sed grep gawk diffutils file patch >= 2.5
 

  Requires: tar unzip gzip bzip2 cpio xz
 

  %if %{with zstd}
 

- Requires: zstd
 

+ Requires: zstd >= 1.3.8
 

  %endif
 

  Requires: pkgconfig >= 1:0.24
 

  Requires: /usr/bin/gdb-add-index
 
@@ -825,6 +828,9 @@ 
 

  %{_datadir}/selinux/packages/rpm_hs.pp.bz2
 

  

  %changelog
 

+ * Mon Feb 06 2022 Aleksandr Kazakov <alexkazakov@meta.com> - 4.16.1.3-22.2
 

+ - Backport multi-threaded zstd for Hyperscale
 

+ 

  * Tue Dec 20 2022 Davide Cavalca <dcavalca@centosproject.org> - 4.16.1.3-22.1
 

  - Merge upstream changes for Hyperscale

Similar to https://git.centos.org/rpms/rpm/pull-request/6 but for centos 9.

Testing

  1. Built with mock -r centos-stream-hyperscale-9-x86_64 --sources . --spec rpm.spec --postinstall
  2. Connected to the chroot with mock -r centos-stream-hyperscale-9-x86_64 --sources ./SOURCES --spec rpm.spec --shell
  3. Set up a test rpm build project with two files, 24M and 43M of size.
  4. Ran rpmbuild with different payload settings
1. `time rpmbuild -ba --define '_binary_payload w19.zstdio' --define '_source_payload w19.zstdio' ./SPECS/hello.spec `
** ... redacted **
real    0m52.310s
user    0m49.147s
sys 0m0.904s

<mock-chroot> sh-5.1# du -sh /builddir/build/RPMS/hello-1-1.noarch.rpm 
45M /builddir/build/RPMS/hello-1-1.noarch.rpm
<mock-chroot> sh-5.1# rpm -q --queryformat '%{PAYLOADCOMPRESSOR}\n' -p /builddir/build/RPMS/hello-1-1.noarch.rpm 
zstd


2. `time rpmbuild -ba --define '_binary_payload w19T0.zstdio' --define '_source_payload w19T0.zstdio' ./SPECS/hello.spec `
** ... redacted **
real    0m36.152s
user    1m0.773s
sys 0m1.199s

<mock-chroot> sh-5.1# du -sh /builddir/build/RPMS/hello-1-1.noarch.rpm 
45M /builddir/build/RPMS/hello-1-1.noarch.rpm

<mock-chroot> sh-5.1# rpm -q --queryformat '%{PAYLOADCOMPRESSOR}\n' -p /builddir/build/RPMS/hello-1-1.noarch.rpm 
zstd

3. `time rpmbuild -ba --define '_binary_payload w2.xzdio' --define '_source_payload w2.xzdio' ./SPECS/hello.spec`
** ... redacted **
real    0m55.148s
user    0m51.783s
sys 0m1.051s

time rpmbuild -ba --define '_binary_payload w2.xzdio' --define '_source_payload w2.xzdio' ./SPECS/hello.spec

<mock-chroot> sh-5.1# rpm -q --queryformat '%{PAYLOADCOMPRESSOR}\n' -p /builddir/build/RPMS/hello-1-1.noarch.rpm 
xz

Pull-Request has been merged by dcavalca
2 years ago

Can we get this submitted upstream? https://gitlab.com/redhat/centos-stream/rpms/rpm

Yeah we should do that. @alexk can you put up a PR there too please? It'll make things easier to maintain down the road.

Don't forget you need a BZ for it too.
Metadata
None
No Tags
Changes Summary 2
+193
file added
rpm-4.16.1.3-backport-multithreaded-zstd.patch
+8 -2
file changed
rpm.spec

Powered by Pagure 5.14.1

SSH Hostkey/Fingerprint | Documentation