diff options
Diffstat (limited to 'src/isa-l/igzip/igzip.c')
| -rw-r--r-- | src/isa-l/igzip/igzip.c | 1180 |
1 files changed, 1180 insertions, 0 deletions
diff --git a/src/isa-l/igzip/igzip.c b/src/isa-l/igzip/igzip.c new file mode 100644 index 00000000..6ac1c150 --- /dev/null +++ b/src/isa-l/igzip/igzip.c @@ -0,0 +1,1180 @@ +/********************************************************************** + Copyright(c) 2011-2016 Intel Corporation All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +**********************************************************************/ + +#define ASM + +#include <assert.h> +#include <string.h> +#ifdef _WIN32 +# include <intrin.h> +#endif + +#define MAX_WRITE_BITS_SIZE 8 +#define FORCE_FLUSH 64 +#define MIN_OBUF_SIZE 224 +#define NON_EMPTY_BLOCK_SIZE 6 +#define MAX_SYNC_FLUSH_SIZE NON_EMPTY_BLOCK_SIZE + MAX_WRITE_BITS_SIZE + +#define MAX_TOKENS (16 * 1024) + +#include "huffman.h" +#include "bitbuf2.h" +#include "igzip_lib.h" +#include "repeated_char_result.h" +#include "huff_codes.h" +#include "encode_df.h" +#include "igzip_level_buf_structs.h" + +extern const uint8_t gzip_hdr[]; +extern const uint32_t gzip_hdr_bytes; +extern const uint32_t gzip_trl_bytes; +extern const struct isal_hufftables hufftables_default; +extern const struct isal_hufftables hufftables_static; + +extern uint32_t crc32_gzip(uint32_t init_crc, const unsigned char *buf, uint64_t len); + +static int write_stored_block_stateless(struct isal_zstream *stream, uint32_t stored_len, + uint32_t crc32); + +static int write_gzip_header_stateless(struct isal_zstream *stream); +static void write_gzip_header(struct isal_zstream *stream); +static int write_deflate_header_stateless(struct isal_zstream *stream); +static int write_deflate_header_unaligned_stateless(struct isal_zstream *stream); + +unsigned int detect_repeated_char(uint8_t * buf, uint32_t size); + +#define STORED_BLK_HDR_BZ 5 +#define STORED_BLK_MAX_BZ 65535 + +void isal_deflate_body(struct isal_zstream *stream); +void isal_deflate_finish(struct isal_zstream *stream); + +void isal_deflate_icf_body(struct isal_zstream *stream); +void isal_deflate_icf_finish(struct isal_zstream *stream); +/*****************************************************************/ + +/* Forward declarations */ +static inline void reset_match_history(struct isal_zstream *stream); +void write_header(struct isal_zstream *stream, uint8_t * deflate_hdr, + uint32_t deflate_hdr_count, uint32_t extra_bits_count, uint32_t next_state, + uint32_t toggle_end_of_stream); +void write_deflate_header(struct isal_zstream *stream); +void write_trailer(struct isal_zstream *stream); + +struct slver { + uint16_t snum; + uint8_t ver; + uint8_t core; +}; + +/* Version info */ +struct slver isal_deflate_init_slver_01030081; +struct slver isal_deflate_init_slver = { 0x0081, 0x03, 0x01 }; + +struct slver isal_deflate_stateless_init_slver_00010084; +struct slver isal_deflate_stateless_init_slver = { 0x0084, 0x01, 0x00 }; + +struct slver isal_deflate_slver_01030082; +struct slver isal_deflate_slver = { 0x0082, 0x03, 0x01 }; + +struct slver isal_deflate_stateless_slver_01010083; +struct slver isal_deflate_stateless_slver = { 0x0083, 0x01, 0x01 }; + +struct slver isal_deflate_set_hufftables_slver_00_01_008b; +struct slver isal_deflate_set_hufftables_slver = { 0x008b, 0x01, 0x00 }; + +/*****************************************************************/ +static +void sync_flush(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + uint64_t bits_to_write = 0xFFFF0000, bits_len; + uint64_t bytes; + int flush_size; + + if (stream->avail_out >= 8) { + set_buf(&state->bitbuf, stream->next_out, stream->avail_out); + + flush_size = (-(state->bitbuf.m_bit_count + 3)) % 8; + + bits_to_write <<= flush_size + 3; + bits_len = 32 + flush_size + 3; + +#ifdef USE_BITBUFB /* Write Bits Always */ + state->state = ZSTATE_NEW_HDR; +#else /* Not Write Bits Always */ + state->state = ZSTATE_FLUSH_WRITE_BUFFER; +#endif + state->has_eob = 0; + + write_bits(&state->bitbuf, bits_to_write, bits_len); + + bytes = buffer_used(&state->bitbuf); + stream->next_out = buffer_ptr(&state->bitbuf); + stream->avail_out -= bytes; + stream->total_out += bytes; + + if (stream->flush == FULL_FLUSH) { + /* Clear match history so there are no cross + * block length distance pairs */ + state->file_start -= state->b_bytes_processed; + state->b_bytes_valid -= state->b_bytes_processed; + state->b_bytes_processed = 0; + reset_match_history(stream); + } + } +} + +static void flush_write_buffer(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + int bytes = 0; + if (stream->avail_out >= 8) { + set_buf(&state->bitbuf, stream->next_out, stream->avail_out); + flush(&state->bitbuf); + stream->next_out = buffer_ptr(&state->bitbuf); + bytes = buffer_used(&state->bitbuf); + stream->avail_out -= bytes; + stream->total_out += bytes; + state->state = ZSTATE_NEW_HDR; + } +} + +static void flush_icf_block(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + struct level_2_buf *level_buf = (struct level_2_buf *)stream->level_buf; + struct BitBuf2 *write_buf = &state->bitbuf; + struct deflate_icf *icf_buf_encoded_next; + + set_buf(write_buf, stream->next_out, stream->avail_out); + +#if defined (USE_BITBUF8) || (USE_BITBUF_ELSE) + if (!is_full(write_buf)) + flush_bits(write_buf); +#endif + + icf_buf_encoded_next = encode_deflate_icf(level_buf->icf_buf_start + state->count, + level_buf->icf_buf_next, write_buf, + &level_buf->encode_tables); + + state->count = icf_buf_encoded_next - level_buf->icf_buf_start; + stream->next_out = buffer_ptr(write_buf); + stream->total_out += buffer_used(write_buf); + stream->avail_out -= buffer_used(write_buf); + + if (level_buf->icf_buf_next <= icf_buf_encoded_next) { + state->count = 0; + if (stream->avail_in == 0 && stream->end_of_stream) + state->state = ZSTATE_TRL; + else if (stream->avail_in == 0 && stream->flush != NO_FLUSH) + state->state = ZSTATE_SYNC_FLUSH; + else + state->state = ZSTATE_NEW_HDR; + } +} + +static void init_new_icf_block(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + struct level_2_buf *level_buf = (struct level_2_buf *)stream->level_buf; + + if (stream->level_buf_size >= + sizeof(struct level_2_buf) + 100 * sizeof(struct deflate_icf)) { + level_buf->icf_buf_next = level_buf->icf_buf_start; + level_buf->icf_buf_avail_out = + stream->level_buf_size - sizeof(struct level_2_buf) - + sizeof(struct deflate_icf); + memset(&state->hist, 0, sizeof(struct isal_mod_hist)); + state->state = ZSTATE_BODY; + } +} + +static void create_icf_block_hdr(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + struct level_2_buf *level_buf = (struct level_2_buf *)stream->level_buf; + struct BitBuf2 *write_buf = &state->bitbuf; + struct BitBuf2 write_buf_tmp; + uint32_t out_size = stream->avail_out; + uint8_t *end_out = stream->next_out + out_size; + /* Write EOB in icf_buf */ + state->hist.ll_hist[256] = 1; + level_buf->icf_buf_next->lit_len = 0x100; + level_buf->icf_buf_next->lit_dist = NULL_DIST_SYM; + level_buf->icf_buf_next->dist_extra = 0; + level_buf->icf_buf_next++; + + state->has_eob_hdr = stream->end_of_stream && !stream->avail_in; + if (end_out - stream->next_out >= ISAL_DEF_MAX_HDR_SIZE) { + /* Determine whether this is the final block */ + + if (stream->gzip_flag == IGZIP_GZIP) + write_gzip_header_stateless(stream); + + set_buf(write_buf, stream->next_out, stream->avail_out); + + create_hufftables_icf(write_buf, &level_buf->encode_tables, &state->hist, + state->has_eob_hdr); + state->state = ZSTATE_FLUSH_ICF_BUFFER; + stream->next_out = buffer_ptr(write_buf); + stream->total_out += buffer_used(write_buf); + stream->avail_out -= buffer_used(write_buf); + } else { + /* Start writing into temporary buffer */ + write_buf_tmp.m_bits = write_buf->m_bits; + write_buf_tmp.m_bit_count = write_buf->m_bit_count; + + write_buf->m_bits = 0; + write_buf->m_bit_count = 0; + + set_buf(&write_buf_tmp, level_buf->deflate_hdr, ISAL_DEF_MAX_HDR_SIZE); + + create_hufftables_icf(&write_buf_tmp, &level_buf->encode_tables, + &state->hist, state->has_eob_hdr); + + level_buf->deflate_hdr_count = buffer_used(&write_buf_tmp); + level_buf->deflate_hdr_extra_bits = write_buf_tmp.m_bit_count; + flush(&write_buf_tmp); + + state->state = ZSTATE_HDR; + } +} + +static void isal_deflate_pass(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + struct isal_hufftables *hufftables = stream->hufftables; + uint8_t *start_in = stream->next_in; + + if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) { + if (state->count == 0) + /* Assume the final header is being written since the header + * stored in hufftables is the final header. */ + state->has_eob_hdr = 1; + write_header(stream, hufftables->deflate_hdr, hufftables->deflate_hdr_count, + hufftables->deflate_hdr_extra_bits, ZSTATE_BODY, + !stream->end_of_stream); + } + + if (state->state == ZSTATE_BODY) + isal_deflate_body(stream); + + if (state->state == ZSTATE_FLUSH_READ_BUFFER) + isal_deflate_finish(stream); + + if (state->state == ZSTATE_SYNC_FLUSH) + sync_flush(stream); + + if (state->state == ZSTATE_FLUSH_WRITE_BUFFER) + flush_write_buffer(stream); + + if (stream->gzip_flag) + state->crc = crc32_gzip(state->crc, start_in, stream->next_in - start_in); + + if (state->state == ZSTATE_TRL) + write_trailer(stream); +} + +static void isal_deflate_icf_pass(struct isal_zstream *stream) +{ + uint8_t *start_in = stream->next_in; + struct isal_zstate *state = &stream->internal_state; + struct level_2_buf *level_buf = (struct level_2_buf *)stream->level_buf; + + do { + if (state->state == ZSTATE_NEW_HDR) + init_new_icf_block(stream); + + if (state->state == ZSTATE_BODY) + isal_deflate_icf_body(stream); + + if (state->state == ZSTATE_FLUSH_READ_BUFFER) + isal_deflate_icf_finish(stream); + + if (state->state == ZSTATE_CREATE_HDR) + create_icf_block_hdr(stream); + + if (state->state == ZSTATE_HDR) + /* Note that the header may be prepended by the + * remaining bits in the previous block, as such the + * toggle header flag cannot be used */ + write_header(stream, level_buf->deflate_hdr, + level_buf->deflate_hdr_count, + level_buf->deflate_hdr_extra_bits, + ZSTATE_FLUSH_ICF_BUFFER, 0); + + if (state->state == ZSTATE_FLUSH_ICF_BUFFER) + flush_icf_block(stream); + + } while (state->state == ZSTATE_NEW_HDR); + + if (state->state == ZSTATE_SYNC_FLUSH) + sync_flush(stream); + + if (state->state == ZSTATE_FLUSH_WRITE_BUFFER) + flush_write_buffer(stream); + + if (stream->gzip_flag) + state->crc = crc32_gzip(state->crc, start_in, stream->next_in - start_in); + + if (state->state == ZSTATE_TRL) + write_trailer(stream); +} + +static void isal_deflate_int(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + uint32_t size; + + /* Move data from temporary output buffer to output buffer */ + if (state->state >= ZSTATE_TMP_OFFSET) { + size = state->tmp_out_end - state->tmp_out_start; + if (size > stream->avail_out) + size = stream->avail_out; + memcpy(stream->next_out, state->tmp_out_buff + state->tmp_out_start, size); + stream->next_out += size; + stream->avail_out -= size; + stream->total_out += size; + state->tmp_out_start += size; + + if (state->tmp_out_start == state->tmp_out_end) + state->state -= ZSTATE_TMP_OFFSET; + + if (stream->avail_out == 0 || state->state == ZSTATE_END + // or do not write out empty blocks since the outbuffer was processed + || (state->state == ZSTATE_NEW_HDR && stream->avail_out == 0)) + return; + } + assert(state->tmp_out_start == state->tmp_out_end); + + if (stream->level == 0) + isal_deflate_pass(stream); + else + isal_deflate_icf_pass(stream); + + /* Fill temporary output buffer then complete filling output buffer */ + if (stream->avail_out > 0 && stream->avail_out < 8 && state->state != ZSTATE_NEW_HDR) { + uint8_t *next_out; + uint32_t avail_out; + uint32_t total_out; + + next_out = stream->next_out; + avail_out = stream->avail_out; + total_out = stream->total_out; + + stream->next_out = state->tmp_out_buff; + stream->avail_out = sizeof(state->tmp_out_buff); + stream->total_out = 0; + + if (stream->level == 0) + isal_deflate_pass(stream); + else + isal_deflate_icf_pass(stream); + + state->tmp_out_start = 0; + state->tmp_out_end = stream->total_out; + + stream->next_out = next_out; + stream->avail_out = avail_out; + stream->total_out = total_out; + if (state->tmp_out_end) { + size = state->tmp_out_end; + if (size > stream->avail_out) + size = stream->avail_out; + memcpy(stream->next_out, state->tmp_out_buff, size); + stream->next_out += size; + stream->avail_out -= size; + stream->total_out += size; + state->tmp_out_start += size; + if (state->tmp_out_start != state->tmp_out_end) + state->state += ZSTATE_TMP_OFFSET; + + } + } + +} + +static void write_constant_compressed_stateless(struct isal_zstream *stream, + uint32_t repeated_length) +{ + /* Assumes repeated_length is at least 1. + * Assumes the input end_of_stream is either 0 or 1. */ + struct isal_zstate *state = &stream->internal_state; + uint32_t rep_bits = ((repeated_length - 1) / 258) * 2; + uint32_t rep_bytes = rep_bits / 8; + uint32_t rep_extra = (repeated_length - 1) % 258; + uint32_t bytes; + uint32_t repeated_char = *stream->next_in; + uint8_t *start_in = stream->next_in; + + /* Guarantee there is enough space for the header even in the worst case */ + if (stream->avail_out < HEADER_LENGTH + MAX_FIXUP_CODE_LENGTH + rep_bytes + 8) + return; + + /* Assumes the repeated char is either 0 or 0xFF. */ + memcpy(stream->next_out, repeated_char_header[repeated_char & 1], HEADER_LENGTH); + + if (stream->avail_in == repeated_length && stream->end_of_stream > 0) { + stream->next_out[0] |= 1; + state->has_eob_hdr = 1; + state->has_eob = 1; + state->state = ZSTATE_TRL; + } else { + state->state = ZSTATE_NEW_HDR; + } + + memset(stream->next_out + HEADER_LENGTH, 0, rep_bytes); + stream->avail_out -= HEADER_LENGTH + rep_bytes; + stream->next_out += HEADER_LENGTH + rep_bytes; + stream->total_out += HEADER_LENGTH + rep_bytes; + + set_buf(&state->bitbuf, stream->next_out, stream->avail_out); + + /* These two lines are basically a modified version of init. */ + state->bitbuf.m_bits = 0; + state->bitbuf.m_bit_count = rep_bits % 8; + + /* Add smaller repeat codes as necessary. Code280 can describe repeat + * lengths of 115-130 bits. Code10 can describe repeat lengths of 10 + * bits. If more than 230 bits, fill code with two code280s. Else if + * more than 115 repeates, fill with code10s until one code280 can + * finish the rest of the repeats. Else, fill with code10s and + * literals */ + if (rep_extra > 115) { + while (rep_extra > 130 && rep_extra < 230) { + write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH); + rep_extra -= 10; + } + + if (rep_extra >= 230) { + write_bits(&state->bitbuf, + CODE_280 | ((rep_extra / 2 - 115) << + CODE_280_LENGTH), CODE_280_TOTAL_LENGTH); + rep_extra -= rep_extra / 2; + } + + write_bits(&state->bitbuf, + CODE_280 | ((rep_extra - 115) << CODE_280_LENGTH), + CODE_280_TOTAL_LENGTH); + + } else { + while (rep_extra >= 10) { + + write_bits(&state->bitbuf, CODE_10, CODE_10_LENGTH); + rep_extra -= 10; + } + + for (; rep_extra > 0; rep_extra--) + write_bits(&state->bitbuf, CODE_LIT, CODE_LIT_LENGTH); + } + + write_bits(&state->bitbuf, END_OF_BLOCK, END_OF_BLOCK_LEN); + + stream->next_in += repeated_length; + stream->avail_in -= repeated_length; + stream->total_in += repeated_length; + + bytes = buffer_used(&state->bitbuf); + stream->next_out = buffer_ptr(&state->bitbuf); + stream->avail_out -= bytes; + stream->total_out += bytes; + + if (stream->gzip_flag) + state->crc = crc32_gzip(state->crc, start_in, stream->next_in - start_in); + + return; +} + +int detect_repeated_char_length(uint8_t * in, uint32_t length) +{ + /* This currently assumes the first 8 bytes are the same character. + * This won't work effectively if the input stream isn't aligned well. */ + uint8_t *p_8, *end = in + length; + uint64_t *p_64 = (uint64_t *) in; + uint64_t w = *p_64; + uint8_t c = (uint8_t) w; + + for (; (p_64 <= (uint64_t *) (end - 8)) && (w == *p_64); p_64++) ; + + p_8 = (uint8_t *) p_64; + + for (; (p_8 < end) && (c == *p_8); p_8++) ; + + return p_8 - in; +} + +static int isal_deflate_int_stateless(struct isal_zstream *stream) +{ + uint32_t repeat_length; + struct isal_zstate *state = &stream->internal_state; + + if (stream->gzip_flag == IGZIP_GZIP) + if (write_gzip_header_stateless(stream)) + return STATELESS_OVERFLOW; + + if (stream->avail_in >= 8 + && (*(uint64_t *) stream->next_in == 0 + || *(uint64_t *) stream->next_in == ~(uint64_t) 0)) { + repeat_length = detect_repeated_char_length(stream->next_in, stream->avail_in); + + if (stream->avail_in == repeat_length || repeat_length >= MIN_REPEAT_LEN) + write_constant_compressed_stateless(stream, repeat_length); + } + + if (stream->level == 0) { + if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) { + write_deflate_header_unaligned_stateless(stream); + if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) + return STATELESS_OVERFLOW; + + reset_match_history(stream); + } + + state->file_start = stream->next_in - stream->total_in; + isal_deflate_pass(stream); + + } else if (stream->level == 1) { + if (stream->level_buf == NULL || stream->level_buf_size < ISAL_DEF_LVL1_MIN) { + /* Default to internal buffer if invalid size is supplied */ + stream->level_buf = state->buffer; + stream->level_buf_size = sizeof(state->buffer); + } + + if (state->state == ZSTATE_NEW_HDR || state->state == ZSTATE_HDR) + reset_match_history(stream); + + state->count = 0; + state->file_start = stream->next_in - stream->total_in; + isal_deflate_icf_pass(stream); + + } else + return ISAL_INVALID_LEVEL; + + if (state->state == ZSTATE_END + || (state->state == ZSTATE_NEW_HDR && stream->flush == FULL_FLUSH)) + return COMP_OK; + else + return STATELESS_OVERFLOW; +} + +static int write_stored_block_stateless(struct isal_zstream *stream, + uint32_t stored_len, uint32_t crc32) +{ + uint64_t stored_blk_hdr; + uint32_t copy_size; + uint32_t avail_in; + uint64_t gzip_trl; + + if (stream->avail_out < stored_len) + return STATELESS_OVERFLOW; + + stream->avail_out -= stored_len; + stream->total_out += stored_len; + avail_in = stream->avail_in; + + if (stream->gzip_flag == IGZIP_GZIP) { + memcpy(stream->next_out, gzip_hdr, gzip_hdr_bytes); + stream->next_out += gzip_hdr_bytes; + stream->gzip_flag = IGZIP_GZIP_NO_HDR; + } + + do { + if (avail_in >= STORED_BLK_MAX_BZ) { + stored_blk_hdr = 0xFFFF00; + copy_size = STORED_BLK_MAX_BZ; + } else { + stored_blk_hdr = ~avail_in; + stored_blk_hdr <<= 24; + stored_blk_hdr |= (avail_in & 0xFFFF) << 8; + copy_size = avail_in; + } + + avail_in -= copy_size; + + /* Handle BFINAL bit */ + if (avail_in == 0) { + if (stream->flush == NO_FLUSH || stream->end_of_stream) { + stored_blk_hdr |= 0x1; + stream->internal_state.has_eob_hdr = 1; + } + } + memcpy(stream->next_out, &stored_blk_hdr, STORED_BLK_HDR_BZ); + stream->next_out += STORED_BLK_HDR_BZ; + + memcpy(stream->next_out, stream->next_in, copy_size); + stream->next_out += copy_size; + stream->next_in += copy_size; + stream->total_in += copy_size; + } while (avail_in != 0); + + if (stream->gzip_flag && stream->internal_state.has_eob_hdr) { + gzip_trl = stream->avail_in; + gzip_trl <<= 32; + gzip_trl |= crc32 & 0xFFFFFFFF; + memcpy(stream->next_out, &gzip_trl, gzip_trl_bytes); + stream->next_out += gzip_trl_bytes; + } + + stream->avail_in = 0; + return COMP_OK; +} + +static inline void reset_match_history(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + uint16_t *head = stream->internal_state.head; + int i = 0; + + state->has_hist = 0; + + if (stream->total_in == 0) + memset(stream->internal_state.head, 0, sizeof(stream->internal_state.head)); + else { + for (i = 0; i < sizeof(state->head) / 2; i++) { + head[i] = (uint16_t) (stream->total_in); + } + } +} + +void isal_deflate_init(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + + stream->total_in = 0; + stream->total_out = 0; + stream->hufftables = (struct isal_hufftables *)&hufftables_default; + stream->level = 0; + stream->level_buf = NULL; + stream->level_buf_size = 0; + stream->end_of_stream = 0; + stream->flush = NO_FLUSH; + stream->gzip_flag = 0; + + state->b_bytes_valid = 0; + state->b_bytes_processed = 0; + state->has_eob = 0; + state->has_eob_hdr = 0; + state->has_hist = 0; + state->state = ZSTATE_NEW_HDR; + state->count = 0; + + state->tmp_out_start = 0; + state->tmp_out_end = 0; + + state->file_start = stream->next_in; + + init(&state->bitbuf); + + state->crc = 0; + + memset(state->head, 0, sizeof(state->head)); + + return; +} + +int isal_deflate_set_hufftables(struct isal_zstream *stream, + struct isal_hufftables *hufftables, int type) +{ + if (stream->internal_state.state != ZSTATE_NEW_HDR) + return ISAL_INVALID_OPERATION; + + switch (type) { + case IGZIP_HUFFTABLE_DEFAULT: + stream->hufftables = (struct isal_hufftables *)&hufftables_default; + break; + case IGZIP_HUFFTABLE_STATIC: + stream->hufftables = (struct isal_hufftables *)&hufftables_static; + break; + case IGZIP_HUFFTABLE_CUSTOM: + if (hufftables != NULL) { + stream->hufftables = hufftables; + break; + } + default: + return ISAL_INVALID_OPERATION; + } + + return COMP_OK; +} + +void isal_deflate_stateless_init(struct isal_zstream *stream) +{ + stream->total_in = 0; + stream->total_out = 0; + stream->hufftables = (struct isal_hufftables *)&hufftables_default; + stream->level = 0; + stream->level_buf = NULL; + stream->level_buf_size = 0; + stream->end_of_stream = 0; + stream->flush = NO_FLUSH; + stream->gzip_flag = 0; + stream->internal_state.state = ZSTATE_NEW_HDR; + return; +} + +int isal_deflate_stateless(struct isal_zstream *stream) +{ + uint8_t *next_in = stream->next_in; + const uint32_t avail_in = stream->avail_in; + const uint32_t total_in = stream->total_in; + + uint8_t *next_out = stream->next_out; + const uint32_t avail_out = stream->avail_out; + const uint32_t total_out = stream->total_out; + const uint32_t gzip_flag = stream->gzip_flag; + + uint32_t crc32 = 0; + uint32_t stored_len; + + /* Final block has already been written */ + stream->internal_state.has_eob_hdr = 0; + init(&stream->internal_state.bitbuf); + stream->internal_state.state = ZSTATE_NEW_HDR; + stream->internal_state.crc = 0; + + if (stream->flush == NO_FLUSH) + stream->end_of_stream = 1; + + if (stream->flush != NO_FLUSH && stream->flush != FULL_FLUSH) + return INVALID_FLUSH; + + if (stream->level != 0 && stream->level != 1) + return ISAL_INVALID_LEVEL; + + if (avail_in == 0) + stored_len = STORED_BLK_HDR_BZ; + else + stored_len = + STORED_BLK_HDR_BZ * ((avail_in + STORED_BLK_MAX_BZ - 1) / + STORED_BLK_MAX_BZ) + avail_in; + + /* + at least 1 byte compressed data in the case of empty dynamic block which only + contains the EOB + */ + + if (stream->gzip_flag == IGZIP_GZIP) + stored_len += gzip_hdr_bytes + gzip_trl_bytes; + + else if (stream->gzip_flag == IGZIP_GZIP_NO_HDR) + stored_len += gzip_trl_bytes; + + /* + the output buffer should be no less than 8 bytes + while empty stored deflate block is 5 bytes only + */ + if (stream->avail_out < 8) + return STATELESS_OVERFLOW; + + if (isal_deflate_int_stateless(stream) == COMP_OK) + return COMP_OK; + else { + if (stream->flush == FULL_FLUSH) { + stream->internal_state.file_start = + (uint8_t *) & stream->internal_state.buffer; + reset_match_history(stream); + } + stream->internal_state.has_eob_hdr = 0; + } + + if (avail_out < stored_len) + return STATELESS_OVERFLOW; + + stream->next_in = next_in; + stream->avail_in = avail_in; + stream->total_in = total_in; + + stream->next_out = next_out; + stream->avail_out = avail_out; + stream->total_out = total_out; + + stream->gzip_flag = gzip_flag; + + if (stream->gzip_flag) + crc32 = crc32_gzip(0x0, next_in, avail_in); + + return write_stored_block_stateless(stream, stored_len, crc32); +} + +int isal_deflate(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + int ret = COMP_OK; + uint8_t *next_in; + uint32_t avail_in, avail_in_start; + uint32_t flush_type = stream->flush; + uint32_t end_of_stream = stream->end_of_stream; + int size = 0; + uint8_t *copy_down_src = NULL; + uint64_t copy_down_size = 0; + int32_t processed = -(state->b_bytes_valid - state->b_bytes_processed); + + if (stream->flush >= 3) + return INVALID_FLUSH; + + next_in = stream->next_in; + avail_in = stream->avail_in; + stream->total_in -= state->b_bytes_valid - state->b_bytes_processed; + + do { + size = avail_in; + if (size > sizeof(state->buffer) - state->b_bytes_valid) { + size = sizeof(state->buffer) - state->b_bytes_valid; + stream->flush = NO_FLUSH; + stream->end_of_stream = 0; + } + memcpy(&state->buffer[state->b_bytes_valid], next_in, size); + + next_in += size; + avail_in -= size; + state->b_bytes_valid += size; + + stream->next_in = &state->buffer[state->b_bytes_processed]; + stream->avail_in = state->b_bytes_valid - state->b_bytes_processed; + state->file_start = stream->next_in - stream->total_in; + processed += stream->avail_in; + + if (stream->avail_in > IGZIP_HIST_SIZE + || stream->end_of_stream || stream->flush != NO_FLUSH) { + avail_in_start = stream->avail_in; + isal_deflate_int(stream); + state->b_bytes_processed += avail_in_start - stream->avail_in; + + if (state->b_bytes_processed > IGZIP_HIST_SIZE) { + copy_down_src = + &state->buffer[state->b_bytes_processed - IGZIP_HIST_SIZE]; + copy_down_size = + state->b_bytes_valid - state->b_bytes_processed + + IGZIP_HIST_SIZE; + memmove(state->buffer, copy_down_src, copy_down_size); + + state->b_bytes_valid -= copy_down_src - state->buffer; + state->b_bytes_processed -= copy_down_src - state->buffer; + } + + } + + stream->flush = flush_type; + stream->end_of_stream = end_of_stream; + processed -= stream->avail_in; + } while (processed < IGZIP_HIST_SIZE + ISAL_LOOK_AHEAD && avail_in > 0 + && stream->avail_out > 0); + + if (processed >= IGZIP_HIST_SIZE + ISAL_LOOK_AHEAD) { + stream->next_in = next_in - stream->avail_in; + stream->avail_in = avail_in + stream->avail_in; + + state->file_start = stream->next_in - stream->total_in; + + if (stream->avail_in > 0 && stream->avail_out > 0) + isal_deflate_int(stream); + + size = stream->avail_in; + if (stream->avail_in > IGZIP_HIST_SIZE) + size = 0; + + memmove(state->buffer, stream->next_in - IGZIP_HIST_SIZE, + size + IGZIP_HIST_SIZE); + state->b_bytes_processed = IGZIP_HIST_SIZE; + state->b_bytes_valid = size + IGZIP_HIST_SIZE; + + stream->next_in += size; + stream->avail_in -= size; + stream->total_in += size; + + } else { + stream->total_in += state->b_bytes_valid - state->b_bytes_processed; + stream->next_in = next_in; + stream->avail_in = avail_in; + state->file_start = stream->next_in - stream->total_in; + + } + + return ret; +} + +static int write_gzip_header_stateless(struct isal_zstream *stream) +{ + if (gzip_hdr_bytes >= stream->avail_out) + return STATELESS_OVERFLOW; + + stream->avail_out -= gzip_hdr_bytes; + stream->total_out += gzip_hdr_bytes; + + memcpy(stream->next_out, gzip_hdr, gzip_hdr_bytes); + + stream->next_out += gzip_hdr_bytes; + stream->gzip_flag = IGZIP_GZIP_NO_HDR; + + return COMP_OK; +} + +static void write_gzip_header(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + int bytes_to_write = gzip_hdr_bytes; + + bytes_to_write -= state->count; + + if (bytes_to_write > stream->avail_out) + bytes_to_write = stream->avail_out; + + memcpy(stream->next_out, gzip_hdr + state->count, bytes_to_write); + state->count += bytes_to_write; + + if (state->count == gzip_hdr_bytes) { + state->count = 0; + stream->gzip_flag = IGZIP_GZIP_NO_HDR; + } + + stream->avail_out -= bytes_to_write; + stream->total_out += bytes_to_write; + stream->next_out += bytes_to_write; + +} + +static int write_deflate_header_stateless(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + struct isal_hufftables *hufftables = stream->hufftables; + uint64_t hdr_extra_bits = hufftables->deflate_hdr[hufftables->deflate_hdr_count]; + uint32_t count; + + if (hufftables->deflate_hdr_count + 8 >= stream->avail_out) + return STATELESS_OVERFLOW; + + memcpy(stream->next_out, hufftables->deflate_hdr, hufftables->deflate_hdr_count); + + if (stream->end_of_stream == 0) { + if (hufftables->deflate_hdr_count > 0) + *stream->next_out -= 1; + else + hdr_extra_bits -= 1; + } else + state->has_eob_hdr = 1; + + stream->avail_out -= hufftables->deflate_hdr_count; + stream->total_out += hufftables->deflate_hdr_count; + stream->next_out += hufftables->deflate_hdr_count; + + set_buf(&state->bitbuf, stream->next_out, stream->avail_out); + + write_bits(&state->bitbuf, hdr_extra_bits, hufftables->deflate_hdr_extra_bits); + + count = buffer_used(&state->bitbuf); + stream->next_out = buffer_ptr(&state->bitbuf); + stream->avail_out -= count; + stream->total_out += count; + + state->state = ZSTATE_BODY; + + return COMP_OK; +} + +static int write_deflate_header_unaligned_stateless(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + struct isal_hufftables *hufftables = stream->hufftables; + unsigned int count; + uint64_t bit_count; + uint64_t *header_next; + uint64_t *header_end; + uint64_t header_bits; + + if (state->bitbuf.m_bit_count == 0) + return write_deflate_header_stateless(stream); + + if (hufftables->deflate_hdr_count + 16 >= stream->avail_out) + return STATELESS_OVERFLOW; + + set_buf(&state->bitbuf, stream->next_out, stream->avail_out); + + header_next = (uint64_t *) hufftables->deflate_hdr; + header_end = header_next + hufftables->deflate_hdr_count / 8; + + header_bits = *header_next; + + if (stream->end_of_stream == 0) + header_bits--; + else + state->has_eob_hdr = 1; + + header_next++; + + /* Write out Complete Header bits */ + for (; header_next <= header_end; header_next++) { + write_bits(&state->bitbuf, header_bits, 32); + header_bits >>= 32; + write_bits(&state->bitbuf, header_bits, 32); + header_bits = *header_next; + } + + bit_count = + (hufftables->deflate_hdr_count & 0x7) * 8 + hufftables->deflate_hdr_extra_bits; + + if (bit_count > MAX_BITBUF_BIT_WRITE) { + write_bits(&state->bitbuf, header_bits, MAX_BITBUF_BIT_WRITE); + header_bits >>= MAX_BITBUF_BIT_WRITE; + bit_count -= MAX_BITBUF_BIT_WRITE; + + } + + write_bits(&state->bitbuf, header_bits, bit_count); + + /* check_space flushes extra bytes in bitbuf. Required because + * write_bits_always fails when the next commit makes the buffer + * length exceed 64 bits */ + check_space(&state->bitbuf, FORCE_FLUSH); + + count = buffer_used(&state->bitbuf); + stream->next_out = buffer_ptr(&state->bitbuf); + stream->avail_out -= count; + stream->total_out += count; + + state->state = ZSTATE_BODY; + + return COMP_OK; +} + +/* Toggle end of stream only works when deflate header is aligned */ +void write_header(struct isal_zstream *stream, uint8_t * deflate_hdr, + uint32_t deflate_hdr_count, uint32_t extra_bits_count, uint32_t next_state, + uint32_t toggle_end_of_stream) +{ + struct isal_zstate *state = &stream->internal_state; + uint32_t hdr_extra_bits = deflate_hdr[deflate_hdr_count]; + uint32_t count; + state->state = ZSTATE_HDR; + + if (state->bitbuf.m_bit_count != 0) { + if (stream->avail_out < 8) + return; + set_buf(&state->bitbuf, stream->next_out, stream->avail_out); + flush(&state->bitbuf); + count = buffer_used(&state->bitbuf); + stream->next_out = buffer_ptr(&state->bitbuf); + stream->avail_out -= count; + stream->total_out += count; + } + + if (stream->gzip_flag == IGZIP_GZIP) + write_gzip_header(stream); + + count = deflate_hdr_count - state->count; + + if (count != 0) { + if (count > stream->avail_out) + count = stream->avail_out; + + memcpy(stream->next_out, deflate_hdr + state->count, count); + + if (toggle_end_of_stream && state->count == 0 && count > 0) { + /* Assumes the final block bit is the first bit */ + *stream->next_out ^= 1; + state->has_eob_hdr = !state->has_eob_hdr; + } + + stream->next_out += count; + stream->avail_out -= count; + stream->total_out += count; + state->count += count; + + count = deflate_hdr_count - state->count; + } else if (toggle_end_of_stream && deflate_hdr_count == 0) { + /* Assumes the final block bit is the first bit */ + hdr_extra_bits ^= 1; + state->has_eob_hdr = !state->has_eob_hdr; + } + + if ((count == 0) && (stream->avail_out >= 8)) { + + set_buf(&state->bitbuf, stream->next_out, stream->avail_out); + + write_bits(&state->bitbuf, hdr_extra_bits, extra_bits_count); + + state->state = next_state; + state->count = 0; + + count = buffer_used(&state->bitbuf); + stream->next_out = buffer_ptr(&state->bitbuf); + stream->avail_out -= count; + stream->total_out += count; + } + +} + +void write_trailer(struct isal_zstream *stream) +{ + struct isal_zstate *state = &stream->internal_state; + unsigned int bytes; + uint32_t crc = state->crc; + + if (stream->avail_out >= 8) { + set_buf(&state->bitbuf, stream->next_out, stream->avail_out); + + /* the flush() will pad to the next byte and write up to 8 bytes + * to the output stream/buffer. + */ + if (!state->has_eob_hdr) { + /* If the final header has not been written, write a + * final block. This block is a static huffman block + * which only contains the end of block symbol. The code + * that happens to do this is the fist 10 bits of + * 0x003 */ + state->has_eob_hdr = 1; + write_bits(&state->bitbuf, 0x003, 10); + if (is_full(&state->bitbuf)) { + stream->next_out = buffer_ptr(&state->bitbuf); + bytes = buffer_used(&state->bitbuf); + stream->avail_out -= bytes; + stream->total_out += bytes; + return; + } + } + + flush(&state->bitbuf); + stream->next_out = buffer_ptr(&state->bitbuf); + bytes = buffer_used(&state->bitbuf); + + if (stream->gzip_flag) { + if (!is_full(&state->bitbuf)) { + *(uint64_t *) stream->next_out = + ((uint64_t) stream->total_in << 32) | crc; + stream->next_out += 8; + bytes += 8; + state->state = ZSTATE_END; + } + } else + state->state = ZSTATE_END; + + stream->avail_out -= bytes; + stream->total_out += bytes; + } +} |