From 6113361316d5ce5bfdc118d188e5617a1fcd747c Mon Sep 17 00:00:00 2001 From: Sean Purcell Date: Mon, 14 Aug 2017 22:46:04 -0700 Subject: [PATCH 1/4] squashfs-tools: Add zstd support This patch adds zstd support to squashfs-tools. It works with zstd versions >= 1.0.0. It was originally written by Sean Purcell. Signed-off-by: Sean Purcell Signed-off-by: Nick Terrell --- squashfs-tools/Makefile | 20 +++ squashfs-tools/compressor.c | 8 ++ squashfs-tools/squashfs_fs.h | 1 + squashfs-tools/zstd_wrapper.c | 254 ++++++++++++++++++++++++++++++++++ squashfs-tools/zstd_wrapper.h | 48 +++++++ 5 files changed, 331 insertions(+) create mode 100644 squashfs-tools/zstd_wrapper.c create mode 100644 squashfs-tools/zstd_wrapper.h diff --git a/squashfs-tools/Makefile b/squashfs-tools/Makefile index 52d2582..22fc559 100644 --- a/squashfs-tools/Makefile +++ b/squashfs-tools/Makefile @@ -75,6 +75,18 @@ GZIP_SUPPORT = 1 #LZMA_SUPPORT = 1 #LZMA_DIR = ../../../../LZMA/lzma465 + +########### Building ZSTD support ############ +# +# The ZSTD library is supported +# ZSTD homepage: http://zstd.net +# ZSTD source repository: https://github.com/facebook/zstd +# +# To build using the ZSTD library - install the library and uncomment the +# ZSTD_SUPPORT line below. +# +#ZSTD_SUPPORT = 1 + ######## Specifying default compression ######## # # The next line specifies which compression algorithm is used by default @@ -177,6 +189,14 @@ LIBS += -llz4 COMPRESSORS += lz4 endif +ifeq ($(ZSTD_SUPPORT),1) +CFLAGS += -DZSTD_SUPPORT +MKSQUASHFS_OBJS += zstd_wrapper.o +UNSQUASHFS_OBJS += zstd_wrapper.o +LIBS += -lzstd +COMPRESSORS += zstd +endif + ifeq ($(XATTR_SUPPORT),1) ifeq ($(XATTR_DEFAULT),1) CFLAGS += -DXATTR_SUPPORT -DXATTR_DEFAULT diff --git a/squashfs-tools/compressor.c b/squashfs-tools/compressor.c index 525e316..02b5e90 100644 --- a/squashfs-tools/compressor.c +++ b/squashfs-tools/compressor.c @@ -65,6 +65,13 @@ static struct compressor xz_comp_ops = { extern struct compressor xz_comp_ops; #endif +#ifndef ZSTD_SUPPORT +static struct compressor zstd_comp_ops = { + ZSTD_COMPRESSION, "zstd" +}; +#else +extern struct compressor zstd_comp_ops; +#endif static struct compressor unknown_comp_ops = { 0, "unknown" @@ -77,6 +84,7 @@ struct compressor *compressor[] = { &lzo_comp_ops, &lz4_comp_ops, &xz_comp_ops, + &zstd_comp_ops, &unknown_comp_ops }; diff --git a/squashfs-tools/squashfs_fs.h b/squashfs-tools/squashfs_fs.h index 791fe12..afca918 100644 --- a/squashfs-tools/squashfs_fs.h +++ b/squashfs-tools/squashfs_fs.h @@ -277,6 +277,7 @@ typedef long long squashfs_inode; #define LZO_COMPRESSION 3 #define XZ_COMPRESSION 4 #define LZ4_COMPRESSION 5 +#define ZSTD_COMPRESSION 6 struct squashfs_super_block { unsigned int s_magic; diff --git a/squashfs-tools/zstd_wrapper.c b/squashfs-tools/zstd_wrapper.c new file mode 100644 index 0000000..dcab75a --- /dev/null +++ b/squashfs-tools/zstd_wrapper.c @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2017 + * Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * zstd_wrapper.c + * + * Support for ZSTD compression http://zstd.net + */ + +#include +#include +#include +#include +#include + +#include "squashfs_fs.h" +#include "zstd_wrapper.h" +#include "compressor.h" + +static int compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; + +/* + * This function is called by the options parsing code in mksquashfs.c + * to parse any -X compressor option. + * + * This function returns: + * >=0 (number of additional args parsed) on success + * -1 if the option was unrecognised, or + * -2 if the option was recognised, but otherwise bad in + * some way (e.g. invalid parameter) + * + * Note: this function sets internal compressor state, but does not + * pass back the results of the parsing other than success/failure. + * The zstd_dump_options() function is called later to get the options in + * a format suitable for writing to the filesystem. + */ +static int zstd_options(char *argv[], int argc) +{ + if (strcmp(argv[0], "-Xcompression-level") == 0) { + if (argc < 2) { + fprintf(stderr, "zstd: -Xcompression-level missing " + "compression level\n"); + fprintf(stderr, "zstd: -Xcompression-level it should " + "be 1 <= n <= %d\n", ZSTD_maxCLevel()); + goto failed; + } + + compression_level = atoi(argv[1]); + if (compression_level < 1 || + compression_level > ZSTD_maxCLevel()) { + fprintf(stderr, "zstd: -Xcompression-level invalid, it " + "should be 1 <= n <= %d\n", ZSTD_maxCLevel()); + goto failed; + } + + return 1; + } + + return -1; +failed: + return -2; +} + +/* + * This function is called by mksquashfs to dump the parsed + * compressor options in a format suitable for writing to the + * compressor options field in the filesystem (stored immediately + * after the superblock). + * + * This function returns a pointer to the compression options structure + * to be stored (and the size), or NULL if there are no compression + * options. + */ +static void *zstd_dump_options(int block_size, int *size) +{ + static struct zstd_comp_opts comp_opts; + + /* don't return anything if the options are all default */ + if (compression_level == ZSTD_DEFAULT_COMPRESSION_LEVEL) + return NULL; + + comp_opts.compression_level = compression_level; + + SQUASHFS_INSWAP_COMP_OPTS(&comp_opts); + + *size = sizeof(comp_opts); + return &comp_opts; +} + +/* + * This function is a helper specifically for the append mode of + * mksquashfs. Its purpose is to set the internal compressor state + * to the stored compressor options in the passed compressor options + * structure. + * + * In effect this function sets up the compressor options + * to the same state they were when the filesystem was originally + * generated, this is to ensure on appending, the compressor uses + * the same compression options that were used to generate the + * original filesystem. + * + * Note, even if there are no compressor options, this function is still + * called with an empty compressor structure (size == 0), to explicitly + * set the default options, this is to ensure any user supplied + * -X options on the appending mksquashfs command line are over-ridden. + * + * This function returns 0 on sucessful extraction of options, and -1 on error. + */ +static int zstd_extract_options(int block_size, void *buffer, int size) +{ + struct zstd_comp_opts *comp_opts = buffer; + + if (size == 0) { + /* Set default values */ + compression_level = ZSTD_DEFAULT_COMPRESSION_LEVEL; + return 0; + } + + /* we expect a comp_opts structure of sufficient size to be present */ + if (size < sizeof(*comp_opts)) + goto failed; + + SQUASHFS_INSWAP_COMP_OPTS(comp_opts); + + if (comp_opts->compression_level < 1 || + comp_opts->compression_level > ZSTD_maxCLevel()) { + fprintf(stderr, "zstd: bad compression level in compression " + "options structure\n"); + goto failed; + } + + compression_level = comp_opts->compression_level; + + return 0; + +failed: + fprintf(stderr, "zstd: error reading stored compressor options from " + "filesystem!\n"); + + return -1; +} + +static void zstd_display_options(void *buffer, int size) +{ + struct zstd_comp_opts *comp_opts = buffer; + + /* we expect a comp_opts structure of sufficient size to be present */ + if (size < sizeof(*comp_opts)) + goto failed; + + SQUASHFS_INSWAP_COMP_OPTS(comp_opts); + + if (comp_opts->compression_level < 1 || + comp_opts->compression_level > ZSTD_maxCLevel()) { + fprintf(stderr, "zstd: bad compression level in compression " + "options structure\n"); + goto failed; + } + + printf("\tcompression-level %d\n", comp_opts->compression_level); + + return; + +failed: + fprintf(stderr, "zstd: error reading stored compressor options from " + "filesystem!\n"); +} + +/* + * This function is called by mksquashfs to initialise the + * compressor, before compress() is called. + * + * This function returns 0 on success, and -1 on error. + */ +static int zstd_init(void **strm, int block_size, int datablock) +{ + ZSTD_CCtx *cctx = ZSTD_createCCtx(); + + if (!cctx) { + fprintf(stderr, "zstd: failed to allocate compression " + "context!\n"); + return -1; + } + + *strm = cctx; + return 0; +} + +static int zstd_compress(void *strm, void *dest, void *src, int size, + int block_size, int *error) +{ + const size_t res = ZSTD_compressCCtx((ZSTD_CCtx*)strm, dest, block_size, + src, size, compression_level); + + if (ZSTD_isError(res)) { + /* FIXME: + * zstd does not expose stable error codes. The error enum may + * change between versions. Until upstream zstd stablizes the + * error codes, we have no way of knowing why the error occurs. + * zstd shouldn't fail to compress any input unless there isn't + * enough output space. We assume that is the cause and return + * the special error code for not enough output space. + */ + return 0; + } + + return (int)res; +} + +static int zstd_uncompress(void *dest, void *src, int size, int outsize, + int *error) +{ + const size_t res = ZSTD_decompress(dest, outsize, src, size); + + if (ZSTD_isError(res)) { + fprintf(stderr, "\t%d %d\n", outsize, size); + + *error = (int)ZSTD_getErrorCode(res); + return -1; + } + + return (int)res; +} + +static void zstd_usage(void) +{ + fprintf(stderr, "\t -Xcompression-level \n"); + fprintf(stderr, "\t\t should be 1 .. %d (default " + "%d)\n", ZSTD_maxCLevel(), ZSTD_DEFAULT_COMPRESSION_LEVEL); +} + +struct compressor zstd_comp_ops = { + .init = zstd_init, + .compress = zstd_compress, + .uncompress = zstd_uncompress, + .options = zstd_options, + .dump_options = zstd_dump_options, + .extract_options = zstd_extract_options, + .display_options = zstd_display_options, + .usage = zstd_usage, + .id = ZSTD_COMPRESSION, + .name = "zstd", + .supported = 1 +}; diff --git a/squashfs-tools/zstd_wrapper.h b/squashfs-tools/zstd_wrapper.h new file mode 100644 index 0000000..4fbef0a --- /dev/null +++ b/squashfs-tools/zstd_wrapper.h @@ -0,0 +1,48 @@ +#ifndef ZSTD_WRAPPER_H +#define ZSTD_WRAPPER_H +/* + * Squashfs + * + * Copyright (c) 2017 + * Phillip Lougher + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2, + * or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * zstd_wrapper.h + * + */ + +#ifndef linux +#define __BYTE_ORDER BYTE_ORDER +#define __BIG_ENDIAN BIG_ENDIAN +#define __LITTLE_ENDIAN LITTLE_ENDIAN +#else +#include +#endif + +#if __BYTE_ORDER == __BIG_ENDIAN +extern unsigned int inswap_le16(unsigned short); +extern unsigned int inswap_le32(unsigned int); + +#define SQUASHFS_INSWAP_COMP_OPTS(s) { \ + (s)->compression_level = inswap_le32((s)->compression_level); \ +} +#else +#define SQUASHFS_INSWAP_COMP_OPTS(s) +#endif + +/* Default compression */ +#define ZSTD_DEFAULT_COMPRESSION_LEVEL 15 + +struct zstd_comp_opts { + int compression_level; +}; +#endif -- 2.19.1