From 4e65ca20fc242e4a03471558a357d7809adeb9c4 Mon Sep 17 00:00:00 2001

From: IBM developers

Date: Thu, 1 Aug 2019 09:02:01 +0200

Subject: [PATCH] Add support for IBM Z hardware-accelerated deflate

Future versions of IBM Z mainframes will provide DFLTCC instruction,

which implements deflate algorithm in hardware with estimated

compression and decompression performance orders of magnitude faster

than the current zlib and ratio comparable with that of level 1.

This patch adds DFLTCC support to zlib. In order to enable it, the

following build commands should be used:

    $ CFLAGS=-DDFLTCC ./configure

    $ make OBJA=dfltcc.o PIC_OBJA=dfltcc.lo

When built like this, zlib would compress in hardware on level 1, and in

software on all other levels. Decompression will always happen in

hardware. In order to enable DFLTCC compression for levels 1-6 (i.e. to

make it used by default) one could either add -DDFLTCC_LEVEL_MASK=0x7e

at compile time, or set the environment variable DFLTCC_LEVEL_MASK to

0x7e at run time.

Two DFLTCC compression calls produce the same results only when they

both are made on machines of the same generation, and when the

respective buffers have the same offset relative to the start of the

page. Therefore care should be taken when using hardware compression

when reproducible results are desired. One such use case - reproducible

software builds - is handled explicitly: when SOURCE_DATE_EPOCH

environment variable is set, the hardware compression is disabled.

DFLTCC does not support every single zlib feature, in particular:

    * inflate(Z_BLOCK) and inflate(Z_TREES)

    * inflateMark()

    * inflatePrime()

    * deflateParams() after the first deflate() call

When used, these functions will either switch to software, or, in case

this is not possible, gracefully fail.

This patch tries to add DFLTCC support in a least intrusive way.

All SystemZ-specific code was placed into a separate file, but

unfortunately there is still a noticeable amount of changes in the

main zlib code. Below is the summary of those changes.

DFLTCC takes as arguments a parameter block, an input buffer, an output

buffer and a window. Since DFLTCC requires parameter block to be

doubleword-aligned, and it's reasonable to allocate it alongside

deflate and inflate states, ZALLOC_STATE, ZFREE_STATE and ZCOPY_STATE

macros were introduced in order to encapsulate the allocation details.

The same is true for window, for which ZALLOC_WINDOW and

TRY_FREE_WINDOW macros were introduced.

While for inflate software and hardware window formats match, this is

not the case for deflate. Therefore, deflateSetDictionary and

deflateGetDictionary need special handling, which is triggered using the

new DEFLATE_SET_DICTIONARY_HOOK and DEFLATE_GET_DICTIONARY_HOOK macros.

deflateResetKeep() and inflateResetKeep() now update the DFLTCC

parameter block, which is allocated alongside zlib state, using

the new DEFLATE_RESET_KEEP_HOOK and INFLATE_RESET_KEEP_HOOK macros.

In order to make unsupported deflateParams(), inflatePrime() and

inflateMark() calls to fail gracefully, the new DEFLATE_PARAMS_HOOK,

INFLATE_PRIME_HOOK and INFLATE_MARK_HOOK macros were introduced.

The algorithm implemented in hardware has different compression ratio

than the one implemented in software. In order for deflateBound() to

return the correct results for the hardware implementation, the new

DEFLATE_BOUND_ADJUST_COMPLEN and DEFLATE_NEED_CONSERVATIVE_BOUND macros

were introduced.

Actual compression and decompression are handled by the new DEFLATE_HOOK

and INFLATE_TYPEDO_HOOK macros. Since inflation with DFLTCC manages the

window on its own, calling updatewindow() is suppressed using the new

INFLATE_NEED_UPDATEWINDOW() macro.

In addition to compression, DFLTCC computes CRC-32 and Adler-32

checksums, therefore, whenever it's used, software checksumming needs to

be suppressed using the new DEFLATE_NEED_CHECKSUM and

INFLATE_NEED_CHECKSUM macros.

DFLTCC will refuse to write an End-of-block Symbol if there is no input

data, thus in some cases it is necessary to do this manually. In order

to achieve this, send_bits, bi_reverse, bi_windup and flush_pending

were promoted from local to ZLIB_INTERNAL. Furthermore, since block and

stream termination must be handled in software as well, block_state enum

was moved to deflate.h.

Since the first call to dfltcc_inflate already needs the window, and it

might be not allocated yet, inflate_ensure_window was factored out of

updatewindow and made ZLIB_INTERNAL.

---

 Makefile.in                   |   8 +

 configure                     |  13 +

 contrib/README.contrib        |   4 +

 contrib/s390/dfltcc.c         | 901 ++++++++++++++++++++++++++++++++++

 contrib/s390/dfltcc.h         |  55 +++

 contrib/s390/dfltcc_deflate.h |  50 ++

 deflate.c                     |  60 ++-

 deflate.h                     |  12 +

 gzguts.h                      |   4 +

 inflate.c                     |  84 +++-

 inflate.h                     |   2 +

 test/infcover.c               |   2 +-

 test/minigzip.c               |   4 +

 trees.c                       |  13 +-

 14 files changed, 1161 insertions(+), 51 deletions(-)

 create mode 100644 contrib/s390/dfltcc.c

 create mode 100644 contrib/s390/dfltcc.h

 create mode 100644 contrib/s390/dfltcc_deflate.h

diff --git a/Makefile.in b/Makefile.in

index 5a77949..e756e2f 100644

--- a/Makefile.in

+++ b/Makefile.in

@@ -143,6 +143,14 @@ match.lo: match.S

 	mv _match.o match.lo

 	rm -f _match.s

+dfltcc.o: $(SRCDIR)contrib/s390/dfltcc.c $(SRCDIR)zlib.h zconf.h

+	$(CC) $(CFLAGS) $(ZINC) -c -o $@ $(SRCDIR)contrib/s390/dfltcc.c

+

+dfltcc.lo: $(SRCDIR)contrib/s390/dfltcc.c $(SRCDIR)zlib.h zconf.h

+	-@mkdir objs 2>/dev/null || test -d objs

+	$(CC) $(SFLAGS) $(ZINC) -DPIC -c -o objs/dfltcc.o $(SRCDIR)contrib/s390/dfltcc.c

+	-@mv objs/dfltcc.o $@

+

 example.o: $(SRCDIR)test/example.c $(SRCDIR)zlib.h zconf.h

 	$(CC) $(CFLAGS) $(ZINCOUT) -c -o $@ $(SRCDIR)test/example.c

diff --git a/configure b/configure

index e974d1f..8fab355 100755

--- a/configure

+++ b/configure

@@ -826,6 +826,19 @@ EOF

   fi

 fi

+# Check whether sys/sdt.h is available

+cat > $test.c << EOF

+#include <sys/sdt.h>

+int main() { return 0; }

+EOF

+if try ${CC} ${CFLAGS} $test.c; then

+    echo "Checking for sys/sdt.h ... Yes." | tee -a configure.log

+    CFLAGS="$CFLAGS -DHAVE_SYS_SDT_H"

+    SFLAGS="$SFLAGS -DHAVE_SYS_SDT_H"

+else

+    echo "Checking for sys/sdt.h ... No." | tee -a configure.log

+fi

+

 # show the results in the log

 echo >> configure.log

 echo ALL = $ALL >> configure.log

diff --git a/contrib/README.contrib b/contrib/README.contrib

index a411d5c..b4d3b18 100644

--- a/contrib/README.contrib

+++ b/contrib/README.contrib

@@ -67,6 +67,10 @@ puff/       by Mark Adler <madler@alumni.caltech.edu>

         Small, low memory usage inflate.  Also serves to provide an

         unambiguous description of the deflate format.

+s390/       by Ilya Leoshkevich <iii@linux.ibm.com>

+        Hardware-accelerated deflate on IBM Z with DEFLATE CONVERSION CALL

+        instruction.

+

 testzlib/   by Gilles Vollant <info@winimage.com>

         Example of the use of zlib

diff --git a/contrib/s390/dfltcc.c b/contrib/s390/dfltcc.c

new file mode 100644

index 0000000..d187796

--- /dev/null

+++ b/contrib/s390/dfltcc.c

@@ -0,0 +1,901 @@

+/* dfltcc.c - SystemZ DEFLATE CONVERSION CALL support. */

+

+/*

+   Use the following commands to build zlib with DFLTCC support:

+        $ CFLAGS=-DDFLTCC ./configure

+        $ make OBJA=dfltcc.o PIC_OBJA=dfltcc.lo

+*/

+

+#define _GNU_SOURCE

+#include <ctype.h>

+#include <inttypes.h>

+#include <stddef.h>

+#include <stdio.h>

+#include <stdint.h>

+#include <stdlib.h>

+#include "../../zutil.h"

+#include "../../deflate.h"

+#include "../../inftrees.h"

+#include "../../inflate.h"

+#include "dfltcc.h"

+#include "dfltcc_deflate.h"

+#ifdef HAVE_SYS_SDT_H

+#include <sys/sdt.h>

+#endif

+

+/*

+   C wrapper for the DEFLATE CONVERSION CALL instruction.

+ */

+typedef enum {

+    DFLTCC_CC_OK = 0,

+    DFLTCC_CC_OP1_TOO_SHORT = 1,

+    DFLTCC_CC_OP2_TOO_SHORT = 2,

+    DFLTCC_CC_OP2_CORRUPT = 2,

+    DFLTCC_CC_AGAIN = 3,

+} dfltcc_cc;

+

+#define DFLTCC_QAF 0

+#define DFLTCC_GDHT 1

+#define DFLTCC_CMPR 2

+#define DFLTCC_XPND 4

+#define HBT_CIRCULAR (1 << 7)

+#define HB_BITS 15

+#define HB_SIZE (1 << HB_BITS)

+#define DFLTCC_FACILITY 151

+

+local inline dfltcc_cc dfltcc OF((int fn, void *param,

+                                  Bytef **op1, size_t *len1,

+                                  z_const Bytef **op2, size_t *len2,

+                                  void *hist));

+local inline dfltcc_cc dfltcc(fn, param, op1, len1, op2, len2, hist)

+    int fn;

+    void *param;

+    Bytef **op1;

+    size_t *len1;

+    z_const Bytef **op2;

+    size_t *len2;

+    void *hist;

+{

+    Bytef *t2 = op1 ? *op1 : NULL;

+    size_t t3 = len1 ? *len1 : 0;

+    z_const Bytef *t4 = op2 ? *op2 : NULL;

+    size_t t5 = len2 ? *len2 : 0;

+    register int r0 __asm__("r0") = fn;

+    register void *r1 __asm__("r1") = param;

+    register Bytef *r2 __asm__("r2") = t2;

+    register size_t r3 __asm__("r3") = t3;

+    register z_const Bytef *r4 __asm__("r4") = t4;

+    register size_t r5 __asm__("r5") = t5;

+    int cc;

+

+    __asm__ volatile(

+#ifdef HAVE_SYS_SDT_H

+                     STAP_PROBE_ASM(zlib, dfltcc_entry,

+                                    STAP_PROBE_ASM_TEMPLATE(5))

+#endif

+                     ".insn rrf,0xb9390000,%[r2],%[r4],%[hist],0\n"

+#ifdef HAVE_SYS_SDT_H

+                     STAP_PROBE_ASM(zlib, dfltcc_exit,

+                                    STAP_PROBE_ASM_TEMPLATE(5))

+#endif

+                     "ipm %[cc]\n"

+                     : [r2] "+r" (r2)

+                     , [r3] "+r" (r3)

+                     , [r4] "+r" (r4)

+                     , [r5] "+r" (r5)

+                     , [cc] "=r" (cc)

+                     : [r0] "r" (r0)

+                     , [r1] "r" (r1)

+                     , [hist] "r" (hist)

+#ifdef HAVE_SYS_SDT_H

+                     , STAP_PROBE_ASM_OPERANDS(5, r2, r3, r4, r5, hist)

+#endif

+                     : "cc", "memory");

+    t2 = r2; t3 = r3; t4 = r4; t5 = r5;

+

+    if (op1)

+        *op1 = t2;

+    if (len1)

+        *len1 = t3;

+    if (op2)

+        *op2 = t4;

+    if (len2)

+        *len2 = t5;

+    return (cc >> 28) & 3;

+}

+

+/*

+   Parameter Block for Query Available Functions.

+ */

+#define static_assert(c, msg) \

+        __attribute__((unused)) \

+        static char static_assert_failed_ ## msg[c ? 1 : -1]

+

+struct dfltcc_qaf_param {

+    char fns[16];

+    char reserved1[8];

+    char fmts[2];

+    char reserved2[6];

+};

+

+static_assert(sizeof(struct dfltcc_qaf_param) == 32,

+              sizeof_struct_dfltcc_qaf_param_is_32);

+

+local inline int is_bit_set OF((const char *bits, int n));

+local inline int is_bit_set(bits, n)

+    const char *bits;

+    int n;

+{

+    return bits[n / 8] & (1 << (7 - (n % 8)));

+}

+

+local inline void clear_bit OF((char *bits, int n));

+local inline void clear_bit(bits, n)

+    char *bits;

+    int n;

+{

+    bits[n / 8] &= ~(1 << (7 - (n % 8)));

+}

+

+#define DFLTCC_FMT0 0

+

+/*

+   Parameter Block for Generate Dynamic-Huffman Table, Compress and Expand.

+ */

+#define CVT_CRC32 0

+#define CVT_ADLER32 1

+#define HTT_FIXED 0

+#define HTT_DYNAMIC 1

+

+struct dfltcc_param_v0 {

+    uint16_t pbvn;                     /* Parameter-Block-Version Number */

+    uint8_t mvn;                       /* Model-Version Number */

+    uint8_t ribm;                      /* Reserved for IBM use */

+    unsigned reserved32 : 31;

+    unsigned cf : 1;                   /* Continuation Flag */

+    uint8_t reserved64[8];

+    unsigned nt : 1;                   /* New Task */

+    unsigned reserved129 : 1;

+    unsigned cvt : 1;                  /* Check Value Type */

+    unsigned reserved131 : 1;

+    unsigned htt : 1;                  /* Huffman-Table Type */

+    unsigned bcf : 1;                  /* Block-Continuation Flag */

+    unsigned bcc : 1;                  /* Block Closing Control */

+    unsigned bhf : 1;                  /* Block Header Final */

+    unsigned reserved136 : 1;

+    unsigned reserved137 : 1;

+    unsigned dhtgc : 1;                /* DHT Generation Control */

+    unsigned reserved139 : 5;

+    unsigned reserved144 : 5;

+    unsigned sbb : 3;                  /* Sub-Byte Boundary */

+    uint8_t oesc;                      /* Operation-Ending-Supplemental Code */

+    unsigned reserved160 : 12;

+    unsigned ifs : 4;                  /* Incomplete-Function Status */

+    uint16_t ifl;                      /* Incomplete-Function Length */

+    uint8_t reserved192[8];

+    uint8_t reserved256[8];

+    uint8_t reserved320[4];

+    uint16_t hl;                       /* History Length */

+    unsigned reserved368 : 1;

+    uint16_t ho : 15;                  /* History Offset */

+    uint32_t cv;                       /* Check Value */

+    unsigned eobs : 15;                /* End-of-block Symbol */

+    unsigned reserved431: 1;

+    uint8_t eobl : 4;                  /* End-of-block Length */

+    unsigned reserved436 : 12;

+    unsigned reserved448 : 4;

+    uint16_t cdhtl : 12;               /* Compressed-Dynamic-Huffman Table

+                                          Length */

+    uint8_t reserved464[6];

+    uint8_t cdht[288];

+    uint8_t reserved[32];

+    uint8_t csb[1152];

+};

+

+static_assert(sizeof(struct dfltcc_param_v0) == 1536,

+              sizeof_struct_dfltcc_param_v0_is_1536);

+

+local z_const char *oesc_msg OF((char *buf, int oesc));

+local z_const char *oesc_msg(buf, oesc)

+    char *buf;

+    int oesc;

+{

+    if (oesc == 0x00)

+        return NULL; /* Successful completion */

+    else {

+        sprintf(buf, "Operation-Ending-Supplemental Code is 0x%.2X", oesc);

+        return buf;

+    }

+}

+

+/*

+   Extension of inflate_state and deflate_state. Must be doubleword-aligned.

+*/

+struct dfltcc_state {

+    struct dfltcc_param_v0 param;      /* Parameter block. */

+    struct dfltcc_qaf_param af;        /* Available functions. */

+    uLong level_mask;                  /* Levels on which to use DFLTCC */

+    uLong block_size;                  /* New block each X bytes */

+    uLong block_threshold;             /* New block after total_in > X */

+    uLong dht_threshold;               /* New block only if avail_in >= X */

+    char msg[64];                      /* Buffer for strm->msg */

+};

+

+#define ALIGN_UP(p, size) \

+        (__typeof__(p))(((uintptr_t)(p) + ((size) - 1)) & ~((size) - 1))

+

+#define GET_DFLTCC_STATE(state) ((struct dfltcc_state FAR *)( \

+        (char FAR *)(state) + ALIGN_UP(sizeof(*state), 8)))

+

+/*

+   Compress.

+ */

+local inline int dfltcc_are_params_ok(int level,

+                                      uInt window_bits,

+                                      int strategy,

+                                      uLong level_mask);

+local inline int dfltcc_are_params_ok(level, window_bits, strategy, level_mask)

+    int level;

+    uInt window_bits;

+    int strategy;

+    uLong level_mask;

+{

+    return (level_mask & (1 << level)) != 0 &&

+        (window_bits == HB_BITS) &&

+        (strategy == Z_FIXED || strategy == Z_DEFAULT_STRATEGY);

+}

+

+

+int ZLIB_INTERNAL dfltcc_can_deflate(strm)

+    z_streamp strm;

+{

+    deflate_state FAR *state = (deflate_state FAR *)strm->state;

+    struct dfltcc_state FAR *dfltcc_state = GET_DFLTCC_STATE(state);

+

+    /* Unsupported compression settings */

+    if (!dfltcc_are_params_ok(state->level, state->w_bits, state->strategy,

+                              dfltcc_state->level_mask))

+        return 0;

+

+    /* Unsupported hardware */

+    if (!is_bit_set(dfltcc_state->af.fns, DFLTCC_GDHT) ||

+            !is_bit_set(dfltcc_state->af.fns, DFLTCC_CMPR) ||

+            !is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0))

+        return 0;

+

+    return 1;

+}

+

+local void dfltcc_gdht OF((z_streamp strm));

+local void dfltcc_gdht(strm)

+    z_streamp strm;

+{

+    deflate_state FAR *state = (deflate_state FAR *)strm->state;

+    struct dfltcc_param_v0 FAR *param = &GET_DFLTCC_STATE(state)->param;

+    size_t avail_in = avail_in = strm->avail_in;

+

+    dfltcc(DFLTCC_GDHT,

+           param, NULL, NULL,

+           &strm->next_in, &avail_in, NULL);

+}

+

+local dfltcc_cc dfltcc_cmpr OF((z_streamp strm));

+local dfltcc_cc dfltcc_cmpr(strm)

+    z_streamp strm;

+{

+    deflate_state FAR *state = (deflate_state FAR *)strm->state;

+    struct dfltcc_param_v0 FAR *param = &GET_DFLTCC_STATE(state)->param;

+    size_t avail_in = strm->avail_in;

+    size_t avail_out = strm->avail_out;

+    dfltcc_cc cc;

+

+    cc = dfltcc(DFLTCC_CMPR | HBT_CIRCULAR,

+                param, &strm->next_out, &avail_out,

+                &strm->next_in, &avail_in, state->window);

+    strm->total_in += (strm->avail_in - avail_in);

+    strm->total_out += (strm->avail_out - avail_out);

+    strm->avail_in = avail_in;

+    strm->avail_out = avail_out;

+    return cc;

+}

+

+local void send_eobs OF((z_streamp strm,

+                         z_const struct dfltcc_param_v0 FAR *param));

+local void send_eobs(strm, param)

+    z_streamp strm;

+    z_const struct dfltcc_param_v0 FAR *param;

+{

+    deflate_state FAR *state = (deflate_state FAR *)strm->state;

+

+    _tr_send_bits(

+          state,

+          bi_reverse(param->eobs >> (15 - param->eobl), param->eobl),

+          param->eobl);

+    flush_pending(strm);

+    if (state->pending != 0) {

+        /* The remaining data is located in pending_out[0:pending]. If someone

+         * calls put_byte() - this might happen in deflate() - the byte will be

+         * placed into pending_buf[pending], which is incorrect. Move the

+         * remaining data to the beginning of pending_buf so that put_byte() is

+         * usable again.

+         */

+        memmove(state->pending_buf, state->pending_out, state->pending);

+        state->pending_out = state->pending_buf;

+    }

+#ifdef ZLIB_DEBUG

+    state->compressed_len += param->eobl;

+#endif

+}

+

+int ZLIB_INTERNAL dfltcc_deflate(strm, flush, result)

+    z_streamp strm;

+    int flush;

+    block_state *result;

+{

+    deflate_state FAR *state = (deflate_state FAR *)strm->state;

+    struct dfltcc_state FAR *dfltcc_state = GET_DFLTCC_STATE(state);

+    struct dfltcc_param_v0 FAR *param = &dfltcc_state->param;

+    uInt masked_avail_in;

+    dfltcc_cc cc;

+    int need_empty_block;

+    int soft_bcc;

+    int no_flush;

+

+    if (!dfltcc_can_deflate(strm))

+        return 0;

+

+again:

+    masked_avail_in = 0;

+    soft_bcc = 0;

+    no_flush = flush == Z_NO_FLUSH;

+

+    /* Trailing empty block. Switch to software, except when Continuation Flag

+     * is set, which means that DFLTCC has buffered some output in the

+     * parameter block and needs to be called again in order to flush it.

+     */

+    if (flush == Z_FINISH && strm->avail_in == 0 && !param->cf) {

+        if (param->bcf) {

+            /* A block is still open, and the hardware does not support closing

+             * blocks without adding data. Thus, close it manually.

+             */

+            send_eobs(strm, param);

+            param->bcf = 0;

+        }

+        return 0;

+    }

+

+    if (strm->avail_in == 0 && !param->cf) {

+        *result = need_more;

+        return 1;

+    }

+

+    /* There is an open non-BFINAL block, we are not going to close it just

+     * yet, we have compressed more than DFLTCC_BLOCK_SIZE bytes and we see

+     * more than DFLTCC_DHT_MIN_SAMPLE_SIZE bytes. Open a new block with a new

+     * DHT in order to adapt to a possibly changed input data distribution.

+     */

+    if (param->bcf && no_flush &&

+            strm->total_in > dfltcc_state->block_threshold &&

+            strm->avail_in >= dfltcc_state->dht_threshold) {

+        if (param->cf) {

+            /* We need to flush the DFLTCC buffer before writing the

+             * End-of-block Symbol. Mask the input data and proceed as usual.

+             */

+            masked_avail_in += strm->avail_in;

+            strm->avail_in = 0;

+            no_flush = 0;

+        } else {

+            /* DFLTCC buffer is empty, so we can manually write the

+             * End-of-block Symbol right away.

+             */

+            send_eobs(strm, param);

+            param->bcf = 0;

+            dfltcc_state->block_threshold =

+                strm->total_in + dfltcc_state->block_size;

+            if (strm->avail_out == 0) {

+                *result = need_more;

+                return 1;

+            }

+        }

+    }

+

+    /* The caller gave us too much data. Pass only one block worth of

+     * uncompressed data to DFLTCC and mask the rest, so that on the next

+     * iteration we start a new block.

+     */

+    if (no_flush && strm->avail_in > dfltcc_state->block_size) {

+        masked_avail_in += (strm->avail_in - dfltcc_state->block_size);

+        strm->avail_in = dfltcc_state->block_size;

+    }

+

+    /* When we have an open non-BFINAL deflate block and caller indicates that

+     * the stream is ending, we need to close an open deflate block and open a

+     * BFINAL one.

+     */

+    need_empty_block = flush == Z_FINISH && param->bcf && !param->bhf;

+

+    /* Translate stream to parameter block */

+    param->cvt = state->wrap == 2 ? CVT_CRC32 : CVT_ADLER32;

+    if (!no_flush)

+        /* We need to close a block. Always do this in software - when there is

+         * no input data, the hardware will not nohor BCC. */

+        soft_bcc = 1;

+    if (flush == Z_FINISH && !param->bcf)

+        /* We are about to open a BFINAL block, set Block Header Final bit

+         * until the stream ends.

+         */

+        param->bhf = 1;

+    /* DFLTCC-CMPR will write to next_out, so make sure that buffers with

+     * higher precedence are empty.

+     */

+    Assert(state->pending == 0, "There must be no pending bytes");

+    Assert(state->bi_valid < 8, "There must be less than 8 pending bits");

+    param->sbb = (unsigned int)state->bi_valid;

+    if (param->sbb > 0)

+        *strm->next_out = (Bytef)state->bi_buf;

+    if (param->hl)

+        param->nt = 0; /* Honor history */

+    param->cv = state->wrap == 2 ? ZSWAP32(strm->adler) : strm->adler;

+

+    /* When opening a block, choose a Huffman-Table Type */

+    if (!param->bcf) {

+        if (state->strategy == Z_FIXED ||

+                (strm->total_in == 0 && dfltcc_state->block_threshold > 0))

+            param->htt = HTT_FIXED;

+        else {

+            param->htt = HTT_DYNAMIC;

+            dfltcc_gdht(strm);

+        }

+    }

+

+    /* Deflate */

+    do {

+        cc = dfltcc_cmpr(strm);

+        if (strm->avail_in < 4096 && masked_avail_in > 0)

+            /* We are about to call DFLTCC with a small input buffer, which is

+             * inefficient. Since there is masked data, there will be at least

+             * one more DFLTCC call, so skip the current one and make the next

+             * one handle more data.

+             */

+            break;

+    } while (cc == DFLTCC_CC_AGAIN);

+

+    /* Translate parameter block to stream */

+    strm->msg = oesc_msg(dfltcc_state->msg, param->oesc);

+    state->bi_valid = param->sbb;

+    if (state->bi_valid == 0)

+        state->bi_buf = 0; /* Avoid accessing next_out */

+    else

+        state->bi_buf = *strm->next_out & ((1 << state->bi_valid) - 1);

+    strm->adler = state->wrap == 2 ? ZSWAP32(param->cv) : param->cv;

+

+    /* Unmask the input data */

+    strm->avail_in += masked_avail_in;

+    masked_avail_in = 0;

+

+    /* If we encounter an error, it means there is a bug in DFLTCC call */

+    Assert(cc != DFLTCC_CC_OP2_CORRUPT || param->oesc == 0, "BUG");

+

+    /* Update Block-Continuation Flag. It will be used to check whether to call

+     * GDHT the next time.

+     */

+    if (cc == DFLTCC_CC_OK) {

+        if (soft_bcc) {

+            send_eobs(strm, param);

+            param->bcf = 0;

+            dfltcc_state->block_threshold =

+                strm->total_in + dfltcc_state->block_size;

+        } else

+            param->bcf = 1;

+        if (flush == Z_FINISH) {

+            if (need_empty_block)

+                /* Make the current deflate() call also close the stream */

+                return 0;

+            else {

+                bi_windup(state);

+                *result = finish_done;

+            }

+        } else {

+            if (flush == Z_FULL_FLUSH)

+                param->hl = 0; /* Clear history */

+            *result = flush == Z_NO_FLUSH ? need_more : block_done;

+        }

+    } else {

+        param->bcf = 1;

+        *result = need_more;

+    }

+    if (strm->avail_in != 0 && strm->avail_out != 0)

+        goto again; /* deflate() must use all input or all output */

+    return 1;

+}

+

+/*

+   Expand.

+ */

+int ZLIB_INTERNAL dfltcc_can_inflate(strm)

+    z_streamp strm;

+{

+    struct inflate_state FAR *state = (struct inflate_state FAR *)strm->state;

+    struct dfltcc_state FAR *dfltcc_state = GET_DFLTCC_STATE(state);

+

+    /* Unsupported compression settings */

+    if (state->wbits != HB_BITS)

+        return 0;

+

+    /* Unsupported hardware */

+    return is_bit_set(dfltcc_state->af.fns, DFLTCC_XPND) &&

+               is_bit_set(dfltcc_state->af.fmts, DFLTCC_FMT0);

+}

+

+local dfltcc_cc dfltcc_xpnd OF((z_streamp strm));

+local dfltcc_cc dfltcc_xpnd(strm)

+    z_streamp strm;

+{

+    struct inflate_state FAR *state = (struct inflate_state FAR *)strm->state;

+    struct dfltcc_param_v0 FAR *param = &GET_DFLTCC_STATE(state)->param;

+    size_t avail_in = strm->avail_in;

+    size_t avail_out = strm->avail_out;

+    dfltcc_cc cc;

+

+    cc = dfltcc(DFLTCC_XPND | HBT_CIRCULAR,

+                param, &strm->next_out, &avail_out,

+                &strm->next_in, &avail_in, state->window);