summaryrefslogtreecommitdiffstats
path: root/src/isa-l/igzip/igzip.c
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-27 18:24:20 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-27 18:24:20 +0000
commit483eb2f56657e8e7f419ab1a4fab8dce9ade8609 (patch)
treee5d88d25d870d5dedacb6bbdbe2a966086a0a5cf /src/isa-l/igzip/igzip.c
parentInitial commit. (diff)
downloadceph-483eb2f56657e8e7f419ab1a4fab8dce9ade8609.tar.xz
ceph-483eb2f56657e8e7f419ab1a4fab8dce9ade8609.zip
Adding upstream version 14.2.21.upstream/14.2.21upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/isa-l/igzip/igzip.c')
-rw-r--r--src/isa-l/igzip/igzip.c1180
1 files changed, 1180 insertions, 0 deletions
diff --git a/src/isa-l/igzip/igzip.c b/src/isa-l/igzip/igzip.c
new file mode 100644
index 00000000..6ac1c150
--- /dev/null
+++ b/src/isa-l/igzip/igzip.c
@@ -0,0 +1,1180 @@
+/**********************************************************************
+ Copyright(c) 2011-2016 Intel Corporation All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions
+ are met:
+ * Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the
+ distribution.
+ * Neither the name of Intel Corporation nor the names of its
+ contributors may be used to endorse or promote products derived
+ from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+**********************************************************************/
+
+#define ASM
+
+#include <assert.h>
+#include <string.h>
+#ifdef _WIN32
+# include <intrin.h>
+#endif
+
+#define MAX_WRITE_BITS_SIZE 8
+#define FORCE_FLUSH 64
+#define MIN_OBUF_SIZE 224
+#define NON_EMPTY_BLOCK_SIZE 6
+#define MAX_SYNC_FLUSH_SIZE NON_EMPTY_BLOCK_SIZE + MAX_WRITE_BITS_SIZE
+
+#define MAX_TOKENS (16 * 1024)
+
+#include "huffman.h"
+#include "bitbuf2.h"
+#include "igzip_lib.h"
+#include "repeated_char_result.h"
+#include "huff_codes.h"
+#include "encode_df.h"
+#include "igzip_level_buf_structs.h"
+
+extern const uint8_t gzip_hdr[];
+extern const uint32_t gzip_hdr_bytes;
+extern const uint32_t gzip_trl_bytes;
+extern const struct isal_hufftables hufftables_default;
+extern const struct isal_hufftables hufftables_static;
+
+extern uint32_t crc32_gzip(uint32_t init_crc, const unsigned char *buf, uint64_t len);
+
+static int write_stored_block_stateless(struct isal_zstream *stream, uint32_t stored_len,
+ uint32_t crc32);
+
+static int write_gzip_header_stateless(struct isal_zstream *stream);
+static void write_gzip_header(struct isal_zstream *stream);
+static int write_deflate_header_stateless(struct isal_zstream *stream);
+static int write_deflate_header_unaligned_stateless(struct isal_zstream *stream);
+
+unsigned int detect_repeated_char(uint8_t * buf, uint32_t size);
+
+#define STORED_BLK_HDR_BZ 5
+#define STORED_BLK_MAX_BZ 65535
+
+void isal_deflate_body(struct isal_zstream *stream);
+void isal_deflate_finish(struct isal_zstream *stream);
+
+void isal_deflate_icf_body(struct isal_zstream *stream);
+void isal_deflate_icf_finish(struct isal_zstream *stream);
+/*****************************************************************/
+
+/* Forward declarations */
+static inline void reset_match_history(struct isal_zstream *stream);
+void write_header(struct isal_zstream *stream, uint8_t * deflate_hdr,
+ uint32_t deflate_hdr_count, uint32_t extra_bits_count, uint32_t next_state,
+ uint32_t toggle_end_of_stream);
+void write_deflate_header(struct isal_zstream *stream);
+void write_trailer(struct isal_zstream *stream);
+
+struct slver {
+ uint16_t snum;
+ uint8_t ver;
+ uint8_t core;
+};
+
+/* Version info */
+struct slver isal_deflate_init_slver_01030081;
+struct slver isal_deflate_init_slver = { 0x0081, 0x03, 0x01 };
+
+struct slver isal_deflate_stateless_init_slver_00010084;
+struct slver isal_deflate_stateless_init_slver = { 0x0084, 0x01, 0x00 };
+
+struct slver isal_deflate_slver_01030082;
+struct slver isal_deflate_slver = { 0x0082, 0x03, 0x01 };
+
+struct slver isal_deflate_stateless_slver_01010083;
+struct slver isal_deflate_stateless_slver = { 0x0083, 0x01, 0x01 };
+
+struct slver isal_deflate_set_hufftables_slver_00_01_008b;
+struct slver isal_deflate_set_hufftables_slver = { 0x008b, 0x01, 0x00 };
+
+/*****************************************************************/
+static
+void sync_flush(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ uint64_t bits_to_write = 0xFFFF0000, bits_len;
+ uint64_t bytes;
+ int flush_size;
+
+ if (stream->avail_out >= 8) {
+ set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
+
+ flush_size = (-(state->bitbuf.m_bit_count + 3)) % 8;
+
+ bits_to_write <<= flush_size + 3;
+ bits_len = 32 + flush_size + 3;
+
+#ifdef USE_BITBUFB /* Write Bits Always */
+ state->state = ZSTATE_NEW_HDR;
+#else /* Not Write Bits Always */
+ state->state = ZSTATE_FLUSH_WRITE_BUFFER;
+#endif
+ state->has_eob = 0;
+
+ write_bits(&state->bitbuf, bits_to_write, bits_len);
+
+ bytes = buffer_used(&state->bitbuf);
+ stream->next_out = buffer_ptr(&state->bitbuf);
+ stream->avail_out -= bytes;
+ stream->total_out += bytes;
+
+ if (stream->flush == FULL_FLUSH) {
+ /* Clear match history so there are no cross
+ * block length distance pairs */
+ state->file_start -= state->b_bytes_processed;
+ state->b_bytes_valid -= state->b_bytes_processed;
+ state->b_bytes_processed = 0;
+ reset_match_history(stream);
+ }
+ }
+}
+
+static void flush_write_buffer(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ int bytes = 0;
+ if (stream->avail_out >= 8) {
+ set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
+ flush(&state->bitbuf);
+ stream->next_out = buffer_ptr(&state->bitbuf);
+ bytes = buffer_used(&state->bitbuf);
+ stream->avail_out -= bytes;
+ stream->total_out += bytes;
+ state->state = ZSTATE_NEW_HDR;
+ }
+}
+
+static void flush_icf_block(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ struct level_2_buf *level_buf = (struct level_2_buf *)stream->level_buf;
+ struct BitBuf2 *write_buf = &state->bitbuf;
+ struct deflate_icf *icf_buf_encoded_next;
+
+ set_buf(write_buf, stream->next_out, stream->avail_out);
+
+#if defined (USE_BITBUF8) || (USE_BITBUF_ELSE)
+ if (!is_full(write_buf))
+ flush_bits(write_buf);
+#endif
+
+ icf_buf_encoded_next = encode_deflate_icf(level_buf->icf_buf_start + state->count,
+ level_buf->icf_buf_next, write_buf,
+ &level_buf->encode_tables);
+
+ state->count = icf_buf_encoded_next - level_buf->icf_buf_start;
+ stream->next_out = buffer_ptr(write_buf);
+ stream->total_out += buffer_used(write_buf);
+ stream->avail_out -= buffer_used(write_buf);
+
+ if (level_buf->icf_buf_next <= icf_buf_encoded_next) {
+ state->count = 0;
+ if (stream->avail_in == 0 && stream->end_of_stream)
+ state->state = ZSTATE_TRL;
+ else if (stream->avail_in == 0 && stream->flush != NO_FLUSH)
+ state->state = ZSTATE_SYNC_FLUSH;
+ else
+ state->state = ZSTATE_NEW_HDR;
+ }
+}
+
+static void init_new_icf_block(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ struct level_2_buf *level_buf = (struct level_2_buf *)stream->level_buf;
+
+ if (stream->level_buf_size >=
+ sizeof(struct level_2_buf) + 100 * sizeof(struct deflate_icf)) {
+ level_buf->icf_buf_next = level_buf->icf_buf_start;
+ level_buf->icf_buf_avail_out =
+ stream->level_buf_size - sizeof(struct level_2_buf) -
+ sizeof(struct deflate_icf);
+ memset(&state->hist, 0, sizeof(struct isal_mod_hist));
+ state->state = ZSTATE_BODY;
+ }
+}
+
+static void create_icf_block_hdr(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ struct level_2_buf *level_buf = (struct level_2_buf *)stream->level_buf;
+ struct BitBuf2 *write_buf = &state->bitbuf;
+ struct BitBuf2 write_buf_tmp;
+ uint32_t out_size = stream->avail_out;
+ uint8_t *end_out = stream->next_out + out_size;
+ /* Write EOB in icf_buf */
+ state->hist.ll_hist[256] = 1;
+ level_buf->icf_buf_next->lit_len = 0x100;
+ level_buf->icf_buf_next->lit_dist = NULL_DIST_SYM;
+ level_buf->icf_buf_next->dist_extra = 0;
+ level_buf->icf_buf_next++;
+
+ state->has_eob_hdr = stream->end_of_stream && !stream->avail_in;
+ if (end_out - stream->next_out >= ISAL_DEF_MAX_HDR_SIZE) {
+ /* Determine whether this is the final block */
+
+ if (stream->gzip_flag == IGZIP_GZIP)
+ write_gzip_header_stateless(stream);
+
+ set_buf(write_buf, stream->next_out, stream->avail_out);
+
+ create_hufftables_icf(write_buf, &level_buf->encode_tables, &state->hist,
+ state->has_eob_hdr);
+ state->state = ZSTATE_FLUSH_ICF_BUFFER;
+ stream->next_out = buffer_ptr(write_buf);
+ stream->total_out += buffer_used(write_buf);
+ stream->avail_out -= buffer_used(write_buf);
+ } else {
+ /* Start writing into temporary buffer */
+ write_buf_tmp.m_bits = write_buf->m_bits;
+ write_buf_tmp.m_bit_count = write_buf->m_bit_count;
+
+ write_buf->m_bits = 0;
+ write_buf->m_bit_count = 0;
+
+ set_buf(&write_buf_tmp, level_buf->deflate_hdr, ISAL_DEF_MAX_HDR_SIZE);
+
+ create_hufftables_icf(&write_buf_tmp, &level_buf->encode_tables,
+ &state->hist, state->has_eob_hdr);
+
+ level_buf->deflate_hdr_count = buffer_used(&write_buf_tmp);
+ level_buf->deflate_hdr_extra_bits = write_buf_tmp.m_bit_count;
+ flush(&write_buf_tmp);
+
+ state->state = ZSTATE_HDR;
+ }
+}
+
+static void isal_deflate_pass(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ struct isal_hufftables *hufftables = stream->hufftables;
+ uint8_t *start_in = stream->next_in;
+
+ if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) {
+ if (state->count == 0)
+ /* Assume the final header is being written since the header
+ * stored in hufftables is the final header. */
+ state->has_eob_hdr = 1;
+ write_header(stream, hufftables->deflate_hdr, hufftables->deflate_hdr_count,
+ hufftables->deflate_hdr_extra_bits, ZSTATE_BODY,
+ !stream->end_of_stream);
+ }
+
+ if (state->state == ZSTATE_BODY)
+ isal_deflate_body(stream);
+
+ if (state->state == ZSTATE_FLUSH_READ_BUFFER)
+ isal_deflate_finish(stream);
+
+ if (state->state == ZSTATE_SYNC_FLUSH)
+ sync_flush(stream);
+
+ if (state->state == ZSTATE_FLUSH_WRITE_BUFFER)
+ flush_write_buffer(stream);
+
+ if (stream->gzip_flag)
+ state->crc = crc32_gzip(state->crc, start_in, stream->next_in - start_in);
+
+ if (state->state == ZSTATE_TRL)
+ write_trailer(stream);
+}
+
+static void isal_deflate_icf_pass(struct isal_zstream *stream)
+{
+ uint8_t *start_in = stream->next_in;
+ struct isal_zstate *state = &stream->internal_state;
+ struct level_2_buf *level_buf = (struct level_2_buf *)stream->level_buf;
+
+ do {
+ if (state->state == ZSTATE_NEW_HDR)
+ init_new_icf_block(stream);
+
+ if (state->state == ZSTATE_BODY)
+ isal_deflate_icf_body(stream);
+
+ if (state->state == ZSTATE_FLUSH_READ_BUFFER)
+ isal_deflate_icf_finish(stream);
+
+ if (state->state == ZSTATE_CREATE_HDR)
+ create_icf_block_hdr(stream);
+
+ if (state->state == ZSTATE_HDR)
+ /* Note that the header may be prepended by the
+ * remaining bits in the previous block, as such the
+ * toggle header flag cannot be used */
+ write_header(stream, level_buf->deflate_hdr,
+ level_buf->deflate_hdr_count,
+ level_buf->deflate_hdr_extra_bits,
+ ZSTATE_FLUSH_ICF_BUFFER, 0);
+
+ if (state->state == ZSTATE_FLUSH_ICF_BUFFER)
+ flush_icf_block(stream);
+
+ } while (state->state == ZSTATE_NEW_HDR);
+
+ if (state->state == ZSTATE_SYNC_FLUSH)
+ sync_flush(stream);
+
+ if (state->state == ZSTATE_FLUSH_WRITE_BUFFER)
+ flush_write_buffer(stream);
+
+ if (stream->gzip_flag)
+ state->crc = crc32_gzip(state->crc, start_in, stream->next_in - start_in);
+
+ if (state->state == ZSTATE_TRL)
+ write_trailer(stream);
+}
+
+static void isal_deflate_int(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ uint32_t size;
+
+ /* Move data from temporary output buffer to output buffer */
+ if (state->state >= ZSTATE_TMP_OFFSET) {
+ size = state->tmp_out_end - state->tmp_out_start;
+ if (size > stream->avail_out)
+ size = stream->avail_out;
+ memcpy(stream->next_out, state->tmp_out_buff + state->tmp_out_start, size);
+ stream->next_out += size;
+ stream->avail_out -= size;
+ stream->total_out += size;
+ state->tmp_out_start += size;
+
+ if (state->tmp_out_start == state->tmp_out_end)
+ state->state -= ZSTATE_TMP_OFFSET;
+
+ if (stream->avail_out == 0 || state->state == ZSTATE_END
+ // or do not write out empty blocks since the outbuffer was processed
+ || (state->state == ZSTATE_NEW_HDR && stream->avail_out == 0))
+ return;
+ }
+ assert(state->tmp_out_start == state->tmp_out_end);
+
+ if (stream->level == 0)
+ isal_deflate_pass(stream);
+ else
+ isal_deflate_icf_pass(stream);
+
+ /* Fill temporary output buffer then complete filling output buffer */
+ if (stream->avail_out > 0 && stream->avail_out < 8 && state->state != ZSTATE_NEW_HDR) {
+ uint8_t *next_out;
+ uint32_t avail_out;
+ uint32_t total_out;
+
+ next_out = stream->next_out;
+ avail_out = stream->avail_out;
+ total_out = stream->total_out;
+
+ stream->next_out = state->tmp_out_buff;
+ stream->avail_out = sizeof(state->tmp_out_buff);
+ stream->total_out = 0;
+
+ if (stream->level == 0)
+ isal_deflate_pass(stream);
+ else
+ isal_deflate_icf_pass(stream);
+
+ state->tmp_out_start = 0;
+ state->tmp_out_end = stream->total_out;
+
+ stream->next_out = next_out;
+ stream->avail_out = avail_out;
+ stream->total_out = total_out;
+ if (state->tmp_out_end) {
+ size = state->tmp_out_end;
+ if (size > stream->avail_out)
+ size = stream->avail_out;
+ memcpy(stream->next_out, state->tmp_out_buff, size);
+ stream->next_out += size;
+ stream->avail_out -= size;
+ stream->total_out += size;
+ state->tmp_out_start += size;
+ if (state->tmp_out_start != state->tmp_out_end)
+ state->state += ZSTATE_TMP_OFFSET;
+
+ }
+ }
+
+}
+
+static void write_constant_compressed_stateless(struct isal_zstream *stream,
+ uint32_t repeated_length)
+{
+ /* Assumes repeated_length is at least 1.
+ * Assumes the input end_of_stream is either 0 or 1. */
+ struct isal_zstate *state = &stream->internal_state;
+ uint32_t rep_bits = ((repeated_length - 1) / 258) * 2;
+ uint32_t rep_bytes = rep_bits / 8;
+ uint32_t rep_extra = (repeated_length - 1) % 258;
+ uint32_t bytes;
+ uint32_t repeated_char = *stream->next_in;
+ uint8_t *start_in = stream->next_in;
+
+ /* Guarantee there is enough space for the header even in the worst case */
+ if (stream->avail_out < HEADER_LENGTH + MAX_FIXUP_CODE_LENGTH + rep_bytes + 8)
+ return;
+
+ /* Assumes the repeated char is either 0 or 0xFF. */
+ memcpy(stream->next_out, repeated_char_header[repeated_char & 1], HEADER_LENGTH);
+
+ if (stream->avail_in == repeated_length && stream->end_of_stream > 0) {
+ stream->next_out[0] |= 1;
+ state->has_eob_hdr = 1;
+ state->has_eob = 1;
+ state->state = ZSTATE_TRL;
+ } else {
+ state->state = ZSTATE_NEW_HDR;
+ }
+
+ memset(stream->next_out + HEADER_LENGTH, 0, rep_bytes);
+ stream->avail_out -= HEADER_LENGTH + rep_bytes;
+ stream->next_out += HEADER_LENGTH + rep_bytes;
+ stream->total_out += HEADER_LENGTH + rep_bytes;
+
+ set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
+
+ /* These two lines are basically a modified version of init. */
+ state->bitbuf.m_bits = 0;
+ state->bitbuf.m_bit_count = rep_bits % 8;
+
+ /* Add smaller repeat codes as necessary. Code280 can describe repeat
+ * lengths of 115-130 bits. Code10 can describe repeat lengths of 10
+ * bits. If more than 230 bits, fill code with two code280s. Else if
+ * more than 115 repeates, fill with code10s until one code280 can
+ * finish the rest of the repeats. Else, fill with code10s and
+ * literals */
+ if (rep_extra > 115) {
+ while (rep_extra > 130 && rep_extra < 230) {
+ write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH);
+ rep_extra -= 10;
+ }
+
+ if (rep_extra >= 230) {
+ write_bits(&state->bitbuf,
+ CODE_280 | ((rep_extra / 2 - 115) <<
+ CODE_280_LENGTH), CODE_280_TOTAL_LENGTH);
+ rep_extra -= rep_extra / 2;
+ }
+
+ write_bits(&state->bitbuf,
+ CODE_280 | ((rep_extra - 115) << CODE_280_LENGTH),
+ CODE_280_TOTAL_LENGTH);
+
+ } else {
+ while (rep_extra >= 10) {
+
+ write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH);
+ rep_extra -= 10;
+ }
+
+ for (; rep_extra > 0; rep_extra--)
+ write_bits(&state->bitbuf, CODE_LIT, CODE_LIT_LENGTH);
+ }
+
+ write_bits(&state->bitbuf, END_OF_BLOCK, END_OF_BLOCK_LEN);
+
+ stream->next_in += repeated_length;
+ stream->avail_in -= repeated_length;
+ stream->total_in += repeated_length;
+
+ bytes = buffer_used(&state->bitbuf);
+ stream->next_out = buffer_ptr(&state->bitbuf);
+ stream->avail_out -= bytes;
+ stream->total_out += bytes;
+
+ if (stream->gzip_flag)
+ state->crc = crc32_gzip(state->crc, start_in, stream->next_in - start_in);
+
+ return;
+}
+
+int detect_repeated_char_length(uint8_t * in, uint32_t length)
+{
+ /* This currently assumes the first 8 bytes are the same character.
+ * This won't work effectively if the input stream isn't aligned well. */
+ uint8_t *p_8, *end = in + length;
+ uint64_t *p_64 = (uint64_t *) in;
+ uint64_t w = *p_64;
+ uint8_t c = (uint8_t) w;
+
+ for (; (p_64 <= (uint64_t *) (end - 8)) && (w == *p_64); p_64++) ;
+
+ p_8 = (uint8_t *) p_64;
+
+ for (; (p_8 < end) && (c == *p_8); p_8++) ;
+
+ return p_8 - in;
+}
+
+static int isal_deflate_int_stateless(struct isal_zstream *stream)
+{
+ uint32_t repeat_length;
+ struct isal_zstate *state = &stream->internal_state;
+
+ if (stream->gzip_flag == IGZIP_GZIP)
+ if (write_gzip_header_stateless(stream))
+ return STATELESS_OVERFLOW;
+
+ if (stream->avail_in >= 8
+ && (*(uint64_t *) stream->next_in == 0
+ || *(uint64_t *) stream->next_in == ~(uint64_t) 0)) {
+ repeat_length = detect_repeated_char_length(stream->next_in, stream->avail_in);
+
+ if (stream->avail_in == repeat_length || repeat_length >= MIN_REPEAT_LEN)
+ write_constant_compressed_stateless(stream, repeat_length);
+ }
+
+ if (stream->level == 0) {
+ if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) {
+ write_deflate_header_unaligned_stateless(stream);
+ if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR)
+ return STATELESS_OVERFLOW;
+
+ reset_match_history(stream);
+ }
+
+ state->file_start = stream->next_in - stream->total_in;
+ isal_deflate_pass(stream);
+
+ } else if (stream->level == 1) {
+ if (stream->level_buf == NULL || stream->level_buf_size < ISAL_DEF_LVL1_MIN) {
+ /* Default to internal buffer if invalid size is supplied */
+ stream->level_buf = state->buffer;
+ stream->level_buf_size = sizeof(state->buffer);
+ }
+
+ if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR)
+ reset_match_history(stream);
+
+ state->count = 0;
+ state->file_start = stream->next_in - stream->total_in;
+ isal_deflate_icf_pass(stream);
+
+ } else
+ return ISAL_INVALID_LEVEL;
+
+ if (state->state == ZSTATE_END
+ || (state->state == ZSTATE_NEW_HDR && stream->flush == FULL_FLUSH))
+ return COMP_OK;
+ else
+ return STATELESS_OVERFLOW;
+}
+
+static int write_stored_block_stateless(struct isal_zstream *stream,
+ uint32_t stored_len, uint32_t crc32)
+{
+ uint64_t stored_blk_hdr;
+ uint32_t copy_size;
+ uint32_t avail_in;
+ uint64_t gzip_trl;
+
+ if (stream->avail_out < stored_len)
+ return STATELESS_OVERFLOW;
+
+ stream->avail_out -= stored_len;
+ stream->total_out += stored_len;
+ avail_in = stream->avail_in;
+
+ if (stream->gzip_flag == IGZIP_GZIP) {
+ memcpy(stream->next_out, gzip_hdr, gzip_hdr_bytes);
+ stream->next_out += gzip_hdr_bytes;
+ stream->gzip_flag = IGZIP_GZIP_NO_HDR;
+ }
+
+ do {
+ if (avail_in >= STORED_BLK_MAX_BZ) {
+ stored_blk_hdr = 0xFFFF00;
+ copy_size = STORED_BLK_MAX_BZ;
+ } else {
+ stored_blk_hdr = ~avail_in;
+ stored_blk_hdr <<= 24;
+ stored_blk_hdr |= (avail_in & 0xFFFF) << 8;
+ copy_size = avail_in;
+ }
+
+ avail_in -= copy_size;
+
+ /* Handle BFINAL bit */
+ if (avail_in == 0) {
+ if (stream->flush == NO_FLUSH || stream->end_of_stream) {
+ stored_blk_hdr |= 0x1;
+ stream->internal_state.has_eob_hdr = 1;
+ }
+ }
+ memcpy(stream->next_out, &stored_blk_hdr, STORED_BLK_HDR_BZ);
+ stream->next_out += STORED_BLK_HDR_BZ;
+
+ memcpy(stream->next_out, stream->next_in, copy_size);
+ stream->next_out += copy_size;
+ stream->next_in += copy_size;
+ stream->total_in += copy_size;
+ } while (avail_in != 0);
+
+ if (stream->gzip_flag && stream->internal_state.has_eob_hdr) {
+ gzip_trl = stream->avail_in;
+ gzip_trl <<= 32;
+ gzip_trl |= crc32 & 0xFFFFFFFF;
+ memcpy(stream->next_out, &gzip_trl, gzip_trl_bytes);
+ stream->next_out += gzip_trl_bytes;
+ }
+
+ stream->avail_in = 0;
+ return COMP_OK;
+}
+
+static inline void reset_match_history(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ uint16_t *head = stream->internal_state.head;
+ int i = 0;
+
+ state->has_hist = 0;
+
+ if (stream->total_in == 0)
+ memset(stream->internal_state.head, 0, sizeof(stream->internal_state.head));
+ else {
+ for (i = 0; i < sizeof(state->head) / 2; i++) {
+ head[i] = (uint16_t) (stream->total_in);
+ }
+ }
+}
+
+void isal_deflate_init(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+
+ stream->total_in = 0;
+ stream->total_out = 0;
+ stream->hufftables = (struct isal_hufftables *)&hufftables_default;
+ stream->level = 0;
+ stream->level_buf = NULL;
+ stream->level_buf_size = 0;
+ stream->end_of_stream = 0;
+ stream->flush = NO_FLUSH;
+ stream->gzip_flag = 0;
+
+ state->b_bytes_valid = 0;
+ state->b_bytes_processed = 0;
+ state->has_eob = 0;
+ state->has_eob_hdr = 0;
+ state->has_hist = 0;
+ state->state = ZSTATE_NEW_HDR;
+ state->count = 0;
+
+ state->tmp_out_start = 0;
+ state->tmp_out_end = 0;
+
+ state->file_start = stream->next_in;
+
+ init(&state->bitbuf);
+
+ state->crc = 0;
+
+ memset(state->head, 0, sizeof(state->head));
+
+ return;
+}
+
+int isal_deflate_set_hufftables(struct isal_zstream *stream,
+ struct isal_hufftables *hufftables, int type)
+{
+ if (stream->internal_state.state != ZSTATE_NEW_HDR)
+ return ISAL_INVALID_OPERATION;
+
+ switch (type) {
+ case IGZIP_HUFFTABLE_DEFAULT:
+ stream->hufftables = (struct isal_hufftables *)&hufftables_default;
+ break;
+ case IGZIP_HUFFTABLE_STATIC:
+ stream->hufftables = (struct isal_hufftables *)&hufftables_static;
+ break;
+ case IGZIP_HUFFTABLE_CUSTOM:
+ if (hufftables != NULL) {
+ stream->hufftables = hufftables;
+ break;
+ }
+ default:
+ return ISAL_INVALID_OPERATION;
+ }
+
+ return COMP_OK;
+}
+
+void isal_deflate_stateless_init(struct isal_zstream *stream)
+{
+ stream->total_in = 0;
+ stream->total_out = 0;
+ stream->hufftables = (struct isal_hufftables *)&hufftables_default;
+ stream->level = 0;
+ stream->level_buf = NULL;
+ stream->level_buf_size = 0;
+ stream->end_of_stream = 0;
+ stream->flush = NO_FLUSH;
+ stream->gzip_flag = 0;
+ stream->internal_state.state = ZSTATE_NEW_HDR;
+ return;
+}
+
+int isal_deflate_stateless(struct isal_zstream *stream)
+{
+ uint8_t *next_in = stream->next_in;
+ const uint32_t avail_in = stream->avail_in;
+ const uint32_t total_in = stream->total_in;
+
+ uint8_t *next_out = stream->next_out;
+ const uint32_t avail_out = stream->avail_out;
+ const uint32_t total_out = stream->total_out;
+ const uint32_t gzip_flag = stream->gzip_flag;
+
+ uint32_t crc32 = 0;
+ uint32_t stored_len;
+
+ /* Final block has already been written */
+ stream->internal_state.has_eob_hdr = 0;
+ init(&stream->internal_state.bitbuf);
+ stream->internal_state.state = ZSTATE_NEW_HDR;
+ stream->internal_state.crc = 0;
+
+ if (stream->flush == NO_FLUSH)
+ stream->end_of_stream = 1;
+
+ if (stream->flush != NO_FLUSH && stream->flush != FULL_FLUSH)
+ return INVALID_FLUSH;
+
+ if (stream->level != 0 && stream->level != 1)
+ return ISAL_INVALID_LEVEL;
+
+ if (avail_in == 0)
+ stored_len = STORED_BLK_HDR_BZ;
+ else
+ stored_len =
+ STORED_BLK_HDR_BZ * ((avail_in + STORED_BLK_MAX_BZ - 1) /
+ STORED_BLK_MAX_BZ) + avail_in;
+
+ /*
+ at least 1 byte compressed data in the case of empty dynamic block which only
+ contains the EOB
+ */
+
+ if (stream->gzip_flag == IGZIP_GZIP)
+ stored_len += gzip_hdr_bytes + gzip_trl_bytes;
+
+ else if (stream->gzip_flag == IGZIP_GZIP_NO_HDR)
+ stored_len += gzip_trl_bytes;
+
+ /*
+ the output buffer should be no less than 8 bytes
+ while empty stored deflate block is 5 bytes only
+ */
+ if (stream->avail_out < 8)
+ return STATELESS_OVERFLOW;
+
+ if (isal_deflate_int_stateless(stream) == COMP_OK)
+ return COMP_OK;
+ else {
+ if (stream->flush == FULL_FLUSH) {
+ stream->internal_state.file_start =
+ (uint8_t *) & stream->internal_state.buffer;
+ reset_match_history(stream);
+ }
+ stream->internal_state.has_eob_hdr = 0;
+ }
+
+ if (avail_out < stored_len)
+ return STATELESS_OVERFLOW;
+
+ stream->next_in = next_in;
+ stream->avail_in = avail_in;
+ stream->total_in = total_in;
+
+ stream->next_out = next_out;
+ stream->avail_out = avail_out;
+ stream->total_out = total_out;
+
+ stream->gzip_flag = gzip_flag;
+
+ if (stream->gzip_flag)
+ crc32 = crc32_gzip(0x0, next_in, avail_in);
+
+ return write_stored_block_stateless(stream, stored_len, crc32);
+}
+
+int isal_deflate(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ int ret = COMP_OK;
+ uint8_t *next_in;
+ uint32_t avail_in, avail_in_start;
+ uint32_t flush_type = stream->flush;
+ uint32_t end_of_stream = stream->end_of_stream;
+ int size = 0;
+ uint8_t *copy_down_src = NULL;
+ uint64_t copy_down_size = 0;
+ int32_t processed = -(state->b_bytes_valid - state->b_bytes_processed);
+
+ if (stream->flush >= 3)
+ return INVALID_FLUSH;
+
+ next_in = stream->next_in;
+ avail_in = stream->avail_in;
+ stream->total_in -= state->b_bytes_valid - state->b_bytes_processed;
+
+ do {
+ size = avail_in;
+ if (size > sizeof(state->buffer) - state->b_bytes_valid) {
+ size = sizeof(state->buffer) - state->b_bytes_valid;
+ stream->flush = NO_FLUSH;
+ stream->end_of_stream = 0;
+ }
+ memcpy(&state->buffer[state->b_bytes_valid], next_in, size);
+
+ next_in += size;
+ avail_in -= size;
+ state->b_bytes_valid += size;
+
+ stream->next_in = &state->buffer[state->b_bytes_processed];
+ stream->avail_in = state->b_bytes_valid - state->b_bytes_processed;
+ state->file_start = stream->next_in - stream->total_in;
+ processed += stream->avail_in;
+
+ if (stream->avail_in > IGZIP_HIST_SIZE
+ || stream->end_of_stream || stream->flush != NO_FLUSH) {
+ avail_in_start = stream->avail_in;
+ isal_deflate_int(stream);
+ state->b_bytes_processed += avail_in_start - stream->avail_in;
+
+ if (state->b_bytes_processed > IGZIP_HIST_SIZE) {
+ copy_down_src =
+ &state->buffer[state->b_bytes_processed - IGZIP_HIST_SIZE];
+ copy_down_size =
+ state->b_bytes_valid - state->b_bytes_processed +
+ IGZIP_HIST_SIZE;
+ memmove(state->buffer, copy_down_src, copy_down_size);
+
+ state->b_bytes_valid -= copy_down_src - state->buffer;
+ state->b_bytes_processed -= copy_down_src - state->buffer;
+ }
+
+ }
+
+ stream->flush = flush_type;
+ stream->end_of_stream = end_of_stream;
+ processed -= stream->avail_in;
+ } while (processed < IGZIP_HIST_SIZE + ISAL_LOOK_AHEAD && avail_in > 0
+ && stream->avail_out > 0);
+
+ if (processed >= IGZIP_HIST_SIZE + ISAL_LOOK_AHEAD) {
+ stream->next_in = next_in - stream->avail_in;
+ stream->avail_in = avail_in + stream->avail_in;
+
+ state->file_start = stream->next_in - stream->total_in;
+
+ if (stream->avail_in > 0 && stream->avail_out > 0)
+ isal_deflate_int(stream);
+
+ size = stream->avail_in;
+ if (stream->avail_in > IGZIP_HIST_SIZE)
+ size = 0;
+
+ memmove(state->buffer, stream->next_in - IGZIP_HIST_SIZE,
+ size + IGZIP_HIST_SIZE);
+ state->b_bytes_processed = IGZIP_HIST_SIZE;
+ state->b_bytes_valid = size + IGZIP_HIST_SIZE;
+
+ stream->next_in += size;
+ stream->avail_in -= size;
+ stream->total_in += size;
+
+ } else {
+ stream->total_in += state->b_bytes_valid - state->b_bytes_processed;
+ stream->next_in = next_in;
+ stream->avail_in = avail_in;
+ state->file_start = stream->next_in - stream->total_in;
+
+ }
+
+ return ret;
+}
+
+static int write_gzip_header_stateless(struct isal_zstream *stream)
+{
+ if (gzip_hdr_bytes >= stream->avail_out)
+ return STATELESS_OVERFLOW;
+
+ stream->avail_out -= gzip_hdr_bytes;
+ stream->total_out += gzip_hdr_bytes;
+
+ memcpy(stream->next_out, gzip_hdr, gzip_hdr_bytes);
+
+ stream->next_out += gzip_hdr_bytes;
+ stream->gzip_flag = IGZIP_GZIP_NO_HDR;
+
+ return COMP_OK;
+}
+
+static void write_gzip_header(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ int bytes_to_write = gzip_hdr_bytes;
+
+ bytes_to_write -= state->count;
+
+ if (bytes_to_write > stream->avail_out)
+ bytes_to_write = stream->avail_out;
+
+ memcpy(stream->next_out, gzip_hdr + state->count, bytes_to_write);
+ state->count += bytes_to_write;
+
+ if (state->count == gzip_hdr_bytes) {
+ state->count = 0;
+ stream->gzip_flag = IGZIP_GZIP_NO_HDR;
+ }
+
+ stream->avail_out -= bytes_to_write;
+ stream->total_out += bytes_to_write;
+ stream->next_out += bytes_to_write;
+
+}
+
+static int write_deflate_header_stateless(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ struct isal_hufftables *hufftables = stream->hufftables;
+ uint64_t hdr_extra_bits = hufftables->deflate_hdr[hufftables->deflate_hdr_count];
+ uint32_t count;
+
+ if (hufftables->deflate_hdr_count + 8 >= stream->avail_out)
+ return STATELESS_OVERFLOW;
+
+ memcpy(stream->next_out, hufftables->deflate_hdr, hufftables->deflate_hdr_count);
+
+ if (stream->end_of_stream == 0) {
+ if (hufftables->deflate_hdr_count > 0)
+ *stream->next_out -= 1;
+ else
+ hdr_extra_bits -= 1;
+ } else
+ state->has_eob_hdr = 1;
+
+ stream->avail_out -= hufftables->deflate_hdr_count;
+ stream->total_out += hufftables->deflate_hdr_count;
+ stream->next_out += hufftables->deflate_hdr_count;
+
+ set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
+
+ write_bits(&state->bitbuf, hdr_extra_bits, hufftables->deflate_hdr_extra_bits);
+
+ count = buffer_used(&state->bitbuf);
+ stream->next_out = buffer_ptr(&state->bitbuf);
+ stream->avail_out -= count;
+ stream->total_out += count;
+
+ state->state = ZSTATE_BODY;
+
+ return COMP_OK;
+}
+
+static int write_deflate_header_unaligned_stateless(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ struct isal_hufftables *hufftables = stream->hufftables;
+ unsigned int count;
+ uint64_t bit_count;
+ uint64_t *header_next;
+ uint64_t *header_end;
+ uint64_t header_bits;
+
+ if (state->bitbuf.m_bit_count == 0)
+ return write_deflate_header_stateless(stream);
+
+ if (hufftables->deflate_hdr_count + 16 >= stream->avail_out)
+ return STATELESS_OVERFLOW;
+
+ set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
+
+ header_next = (uint64_t *) hufftables->deflate_hdr;
+ header_end = header_next + hufftables->deflate_hdr_count / 8;
+
+ header_bits = *header_next;
+
+ if (stream->end_of_stream == 0)
+ header_bits--;
+ else
+ state->has_eob_hdr = 1;
+
+ header_next++;
+
+ /* Write out Complete Header bits */
+ for (; header_next <= header_end; header_next++) {
+ write_bits(&state->bitbuf, header_bits, 32);
+ header_bits >>= 32;
+ write_bits(&state->bitbuf, header_bits, 32);
+ header_bits = *header_next;
+ }
+
+ bit_count =
+ (hufftables->deflate_hdr_count & 0x7) * 8 + hufftables->deflate_hdr_extra_bits;
+
+ if (bit_count > MAX_BITBUF_BIT_WRITE) {
+ write_bits(&state->bitbuf, header_bits, MAX_BITBUF_BIT_WRITE);
+ header_bits >>= MAX_BITBUF_BIT_WRITE;
+ bit_count -= MAX_BITBUF_BIT_WRITE;
+
+ }
+
+ write_bits(&state->bitbuf, header_bits, bit_count);
+
+ /* check_space flushes extra bytes in bitbuf. Required because
+ * write_bits_always fails when the next commit makes the buffer
+ * length exceed 64 bits */
+ check_space(&state->bitbuf, FORCE_FLUSH);
+
+ count = buffer_used(&state->bitbuf);
+ stream->next_out = buffer_ptr(&state->bitbuf);
+ stream->avail_out -= count;
+ stream->total_out += count;
+
+ state->state = ZSTATE_BODY;
+
+ return COMP_OK;
+}
+
+/* Toggle end of stream only works when deflate header is aligned */
+void write_header(struct isal_zstream *stream, uint8_t * deflate_hdr,
+ uint32_t deflate_hdr_count, uint32_t extra_bits_count, uint32_t next_state,
+ uint32_t toggle_end_of_stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ uint32_t hdr_extra_bits = deflate_hdr[deflate_hdr_count];
+ uint32_t count;
+ state->state = ZSTATE_HDR;
+
+ if (state->bitbuf.m_bit_count != 0) {
+ if (stream->avail_out < 8)
+ return;
+ set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
+ flush(&state->bitbuf);
+ count = buffer_used(&state->bitbuf);
+ stream->next_out = buffer_ptr(&state->bitbuf);
+ stream->avail_out -= count;
+ stream->total_out += count;
+ }
+
+ if (stream->gzip_flag == IGZIP_GZIP)
+ write_gzip_header(stream);
+
+ count = deflate_hdr_count - state->count;
+
+ if (count != 0) {
+ if (count > stream->avail_out)
+ count = stream->avail_out;
+
+ memcpy(stream->next_out, deflate_hdr + state->count, count);
+
+ if (toggle_end_of_stream && state->count == 0 && count > 0) {
+ /* Assumes the final block bit is the first bit */
+ *stream->next_out ^= 1;
+ state->has_eob_hdr = !state->has_eob_hdr;
+ }
+
+ stream->next_out += count;
+ stream->avail_out -= count;
+ stream->total_out += count;
+ state->count += count;
+
+ count = deflate_hdr_count - state->count;
+ } else if (toggle_end_of_stream && deflate_hdr_count == 0) {
+ /* Assumes the final block bit is the first bit */
+ hdr_extra_bits ^= 1;
+ state->has_eob_hdr = !state->has_eob_hdr;
+ }
+
+ if ((count == 0) && (stream->avail_out >= 8)) {
+
+ set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
+
+ write_bits(&state->bitbuf, hdr_extra_bits, extra_bits_count);
+
+ state->state = next_state;
+ state->count = 0;
+
+ count = buffer_used(&state->bitbuf);
+ stream->next_out = buffer_ptr(&state->bitbuf);
+ stream->avail_out -= count;
+ stream->total_out += count;
+ }
+
+}
+
+void write_trailer(struct isal_zstream *stream)
+{
+ struct isal_zstate *state = &stream->internal_state;
+ unsigned int bytes;
+ uint32_t crc = state->crc;
+
+ if (stream->avail_out >= 8) {
+ set_buf(&state->bitbuf, stream->next_out, stream->avail_out);
+
+ /* the flush() will pad to the next byte and write up to 8 bytes
+ * to the output stream/buffer.
+ */
+ if (!state->has_eob_hdr) {
+ /* If the final header has not been written, write a
+ * final block. This block is a static huffman block
+ * which only contains the end of block symbol. The code
+ * that happens to do this is the fist 10 bits of
+ * 0x003 */
+ state->has_eob_hdr = 1;
+ write_bits(&state->bitbuf, 0x003, 10);
+ if (is_full(&state->bitbuf)) {
+ stream->next_out = buffer_ptr(&state->bitbuf);
+ bytes = buffer_used(&state->bitbuf);
+ stream->avail_out -= bytes;
+ stream->total_out += bytes;
+ return;
+ }
+ }
+
+ flush(&state->bitbuf);
+ stream->next_out = buffer_ptr(&state->bitbuf);
+ bytes = buffer_used(&state->bitbuf);
+
+ if (stream->gzip_flag) {
+ if (!is_full(&state->bitbuf)) {
+ *(uint64_t *) stream->next_out =
+ ((uint64_t) stream->total_in << 32) | crc;
+ stream->next_out += 8;
+ bytes += 8;
+ state->state = ZSTATE_END;
+ }
+ } else
+ state->state = ZSTATE_END;
+
+ stream->avail_out -= bytes;
+ stream->total_out += bytes;
+ }
+}