diff options
Diffstat (limited to 'src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm')
| -rw-r--r-- | src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm | 901 |
1 files changed, 901 insertions, 0 deletions
diff --git a/src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm b/src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm new file mode 100644 index 000000000..c059178f8 --- /dev/null +++ b/src/isa-l/igzip/igzip_icf_body_h1_gr_bt.asm @@ -0,0 +1,901 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2011-2016 Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%include "options.asm" + +%include "lz0a_const.asm" +%include "data_struct2.asm" +%include "bitbuf2.asm" +%include "huffman.asm" +%include "igzip_compare_types.asm" +%include "reg_sizes.asm" + +%include "stdmac.asm" +%ifdef DEBUG +%macro MARK 1 +global %1 +%1: +%endm +%else +%macro MARK 1 +%endm +%endif + +%define LARGE_MATCH_HASH_REP 1 ; Hash 4 * LARGE_MATCH_HASH_REP elements +%define LARGE_MATCH_MIN 264 ; Minimum match size to enter large match emit loop +%define MIN_INBUF_PADDING 16 +%define MAX_EMIT_SIZE 258 * 16 + +%define SKIP_SIZE_BASE (2 << 10) ; No match length before starting skipping +%define SKIP_BASE 32 ; Initial skip size +%define SKIP_START 512 ; Start increasing skip size once level is beyond SKIP_START +%define SKIP_RATE 2 ; Rate skip size increases after SKIP_START +%define MAX_SKIP_SIZE 128 ; Maximum skip size +%define MAX_SKIP_LEVEL (((MAX_SKIP_SIZE - SKIP_BASE) / SKIP_RATE) + SKIP_START) ; Maximum skip level +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +%define file_start rdi +%define file_length r15 +%define level_buf r14 +%define f_i r10 +%define m_out_buf r11 + +%define curr_data rax + +%define tmp2 rcx +%define skip_count rcx + +%define dist rbx +%define dist_code2 rbx +%define lit_code2 rbx +%define hmask2 rbx + +%define dist2 r12 +%define dist_code r12 +%define hmask3 r12 + +%define tmp1 rsi +%define lit_code rsi + +%define curr_data2 r8 +%define len2 r8 +%define tmp4 r8 +%define hmask1 r8 +%define len_code2 r8 + +%define len rdx +%define len_code rdx +%define hash3 rdx + +%define stream r13 +%define tmp3 r13 + +%define hash rbp +%define hash2 r9 + +;; GPR r8 & r15 can be used + +%define xtmp0 xmm0 ; tmp +%define xtmp1 xmm1 ; tmp +%define xlow_lit_shuf xmm2 +%define xup_lit_shuf xmm3 +%define xdata xmm4 +%define xlit xmm5 + +%define ytmp0 ymm0 ; tmp +%define ytmp1 ymm1 ; tmp + +%define hash_table level_buf + _hash8k_hash_table +%define lit_len_hist level_buf + _hist_lit_len +%define dist_hist level_buf + _hist_dist + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +m_out_end equ 0 ; local variable (8 bytes) +m_out_start equ 8 +dist_mask_offset equ 16 +hash_mask_offset equ 24 +f_end_i_mem_offset equ 32 +stream_offset equ 40 +inbuf_slop_offset equ 48 +skip_match_offset equ 56 +skip_level_offset equ 64 +gpr_save_mem_offset equ 80 ; gpr save area (8*8 bytes) +xmm_save_mem_offset equ gpr_save_mem_offset + 8*8 ; xmm save area (4*16 bytes) (16 byte aligned) +stack_size equ 11*8 + 8*8 + 4*16 + +;;; 8 because stack address is odd multiple of 8 after a function call and +;;; we want it aligned to 16 bytes + +;; Defines to generate functions for different architecture +%xdefine ARCH 01 +%xdefine ARCH1 02 +%xdefine ARCH2 04 + +%ifndef COMPARE_TYPE +%xdefine COMPARE_TYPE_NOT_DEF +%xdefine COMPARE_TYPE 1 +%xdefine COMPARE_TYPE1 2 +%xdefine COMPARE_TYPE2 3 +%endif + +;; Defines to generate functions for different levels +%xdefine METHOD hash_hist + +%rep 3 +%if ARCH == 04 +%define USE_HSWNI +%endif + +; void isal_deflate_icf_body <hashsize> <arch> ( isal_zstream *stream ) +; we make 6 different versions of this function +; arg 1: rcx: addr of stream +global isal_deflate_icf_body_ %+ METHOD %+ _ %+ ARCH +isal_deflate_icf_body_ %+ METHOD %+ _ %+ ARCH %+ : +%ifidn __OUTPUT_FORMAT__, elf64 + mov rcx, rdi +%endif + + ;; do nothing if (avail_in == 0) + cmp dword [rcx + _avail_in], 0 + jne .skip1 + + ;; Set stream's next state + mov rdx, ZSTATE_CREATE_HDR + mov eax, [rcx + _internal_state_state] + cmp word [rcx + _end_of_stream], 0 + cmovne rax, rdx + cmp word [rcx + _flush], _NO_FLUSH + cmovne rax, rdx + mov dword [rcx + _internal_state_state], eax + ret +.skip1: + +%ifdef ALIGN_STACK + push rbp + mov rbp, rsp + sub rsp, stack_size + and rsp, ~15 +%else + sub rsp, stack_size +%endif + + mov [rsp + gpr_save_mem_offset + 0*8], rbx + mov [rsp + gpr_save_mem_offset + 1*8], rsi + mov [rsp + gpr_save_mem_offset + 2*8], rdi + mov [rsp + gpr_save_mem_offset + 3*8], rbp + mov [rsp + gpr_save_mem_offset + 4*8], r12 + mov [rsp + gpr_save_mem_offset + 5*8], r13 + mov [rsp + gpr_save_mem_offset + 6*8], r14 + mov [rsp + gpr_save_mem_offset + 7*8], r15 + + mov stream, rcx + mov [rsp + stream_offset], stream + + mov byte [stream + _internal_state_has_eob], 0 + + mov tmp1 %+ d, dword[stream + _internal_state_dist_mask] + mov [rsp + dist_mask_offset], tmp1 + mov tmp1 %+ d, dword[stream + _internal_state_hash_mask] + mov [rsp + hash_mask_offset], tmp1 + + ; state->bitbuf.set_buf(stream->next_out, stream->avail_out); + mov level_buf, [stream + _level_buf] + mov m_out_buf, [level_buf + _icf_buf_next] + + mov [rsp + m_out_start], m_out_buf + mov tmp1, [level_buf + _icf_buf_avail_out] + add tmp1, m_out_buf + sub tmp1, SLOP + + mov [rsp + m_out_end], tmp1 + + mov file_start, [stream + _next_in] + + mov f_i %+ d, dword [stream + _total_in] + sub file_start, f_i + + mov file_length %+ d, [stream + _avail_in] + add file_length, f_i + + mov [rsp + skip_match_offset], f_i + add qword [rsp + skip_match_offset], SKIP_SIZE_BASE + mov qword [rsp + skip_level_offset], 0 + + PBROADCASTD xlit, dword [min_lit_dist_syms] + MOVDQU xlow_lit_shuf, [low_lit_shuf] + MOVDQU xup_lit_shuf, [up_lit_shuf] + + mov qword [rsp + inbuf_slop_offset], MIN_INBUF_PADDING + cmp byte [stream + _end_of_stream], 0 + jnz .default_inbuf_padding + cmp byte [stream + _flush], 0 + jnz .default_inbuf_padding + mov qword [rsp + inbuf_slop_offset], LA +.default_inbuf_padding: + + ; file_length -= INBUF_PADDING; + sub file_length, [rsp + inbuf_slop_offset] + ; if (file_length <= 0) continue; + mov hmask1 %+ d, [rsp + hash_mask_offset] + + cmp file_length, f_i + jle .input_end + + ; for (f_i = f_start_i; f_i < file_length; f_i++) { + MOVDQU xdata, [file_start + f_i] + mov curr_data, [file_start + f_i] + mov tmp1, curr_data + + compute_hash hash, curr_data + + shr tmp1, 8 + compute_hash hash2, tmp1 + + and hash, hmask1 + and hash2, hmask1 + + cmp byte [stream + _internal_state_has_hist], IGZIP_NO_HIST + je .write_first_byte + + jmp .loop2 + align 16 + +.loop2: + mov tmp3 %+ d, [rsp + dist_mask_offset] + mov hmask1 %+ d, [rsp + hash_mask_offset] + ; if (state->bitbuf.is_full()) { + cmp m_out_buf, [rsp + m_out_end] + ja .output_end + + xor dist, dist + xor dist2, dist2 + + lea tmp1, [file_start + f_i] + + mov dist %+ w, f_i %+ w + dec dist + sub dist %+ w, word [hash_table + 2 * hash] + mov [hash_table + 2 * hash], f_i %+ w + + inc f_i + + mov tmp2, curr_data + shr curr_data, 16 + compute_hash hash, curr_data + and hash %+ d, hmask1 %+ d + + mov dist2 %+ w, f_i %+ w + dec dist2 + sub dist2 %+ w, word [hash_table + 2 * hash2] + mov [hash_table + 2 * hash2], f_i %+ w + + ; if ((dist-1) < (D-1)) { + and dist %+ d, tmp3 %+ d + neg dist + + shr tmp2, 24 + compute_hash hash2, tmp2 + and hash2 %+ d, hmask1 %+ d + + and dist2 %+ d, tmp3 %+ d + neg dist2 + + ;; Check for long len/dist match (>7) with first literal + MOVQ len, xdata + mov curr_data, len + PSRLDQ xdata, 1 + xor len, [tmp1 + dist - 1] + jz .compare_loop + + ;; Check for len/dist match (>7) with second literal + MOVQ len2, xdata + xor len2, [tmp1 + dist2] + jz .compare_loop2 + + movzx lit_code, curr_data %+ b + shr curr_data, 8 + + ;; Check for len/dist match for first literal + test len %+ d, 0xFFFFFFFF + jz .len_dist_huffman_pre + + PSRLDQ xdata, 1 + inc dword [lit_len_hist + HIST_ELEM_SIZE*lit_code] + movzx lit_code2, curr_data %+ b + ;; Check for len/dist match for second literal + test len2 %+ d, 0xFFFFFFFF + jnz .write_lit_bits + +.len_dist_lit_huffman_pre: + bsf len2, len2 + shr len2, 3 + +.len_dist_lit_huffman: + or lit_code, LIT + mov dword [m_out_buf], lit_code %+ d + + neg dist2 + + get_dist_icf_code dist2, dist_code2, tmp1 + + mov hmask3 %+ d, dword [rsp + hash_mask_offset] + + ;; Setup for updating hash + lea tmp3, [f_i + 1] ; tmp3 <= k + + mov tmp2, f_i + add file_start, f_i + add f_i, len2 + cmp f_i, file_length + jg .len_dist_lit_huffman_finish + + lea tmp1, [f_i + SKIP_SIZE_BASE] + mov qword [rsp + skip_match_offset], tmp1 + sub qword [rsp + skip_level_offset], len2 + + MOVDQU xdata, [file_start + len2] + mov tmp1, [file_start + len2] + sub file_start, tmp2 + + shr curr_data, 24 + compute_hash hash3, curr_data + and hash3 %+ d, hmask3 %+ d + + mov curr_data, tmp1 + shr tmp1, 8 + + mov [hash_table + 2 * hash], tmp3 %+ w + + compute_hash hash, curr_data + + add tmp3,1 + mov [hash_table + 2 * hash2], tmp3 %+ w + + compute_hash hash2, tmp1 + + add tmp3, 1 + mov [hash_table + 2 * hash3], tmp3 %+ w + + add dist_code2, 254 + add dist_code2, len2 + + inc dword [lit_len_hist + HIST_ELEM_SIZE*(len2 + 254)] + + mov dword [m_out_buf + 4], dist_code2 %+ d + add m_out_buf, 8 + + shr dist_code2, DIST_OFFSET + and dist_code2, 0x1F + inc dword [dist_hist + HIST_ELEM_SIZE*dist_code2] + + ; hash = compute_hash(state->file_start + f_i) & hash_mask; + and hash %+ d, hmask3 %+ d + and hash2 %+ d, hmask3 %+ d + + ; continue + jmp .loop2 + +.len_dist_lit_huffman_finish: + sub file_start, tmp2 + + mov [hash_table + 2 * hash], tmp3 %+ w + add tmp3,1 + mov [hash_table + 2 * hash2], tmp3 %+ w + + add dist_code2, 254 + add dist_code2, len2 + + inc dword [lit_len_hist + HIST_ELEM_SIZE*(len2 + 254)] + + mov dword [m_out_buf + 4], dist_code2 %+ d + add m_out_buf, 8 + + shr dist_code2, DIST_OFFSET + and dist_code2, 0x1F + inc dword [dist_hist + HIST_ELEM_SIZE*dist_code2] + + jmp .input_end + +.len_dist_huffman_pre: + bsf len, len + shr len, 3 + +.len_dist_huffman: + dec f_i + ;; Setup for updateing hash + lea tmp3, [f_i + 2] ; tmp3 <= k + + neg dist + + ; get_dist_code(dist, &code2, &code_len2); + get_dist_icf_code dist, dist_code, tmp1 + +.len_dist_huffman_skip: + + mov hmask2 %+ d, [rsp + hash_mask_offset] + + mov tmp1, f_i + add file_start, f_i + + add f_i, len + cmp f_i, file_length + jg .len_dist_huffman_finish + + lea tmp2, [f_i + SKIP_SIZE_BASE] + mov qword [rsp + skip_match_offset], tmp2 + sub qword [rsp + skip_level_offset], len + + MOVDQU xdata, [file_start + len] + mov curr_data2, [file_start + len] + mov curr_data, curr_data2 + sub file_start, tmp1 + ; get_len_code(len, &code, &code_len); + lea len_code, [len + 254] + or dist_code, len_code + + mov [hash_table + 2 * hash], tmp3 %+ w + add tmp3,1 + mov [hash_table + 2 * hash2], tmp3 %+ w + + compute_hash hash, curr_data + + shr curr_data2, 8 + compute_hash hash2, curr_data2 + + inc dword [lit_len_hist + HIST_ELEM_SIZE*len_code] + + mov dword [m_out_buf], dist_code %+ d + add m_out_buf, 4 + + shr dist_code, DIST_OFFSET + and dist_code, 0x1F + inc dword [dist_hist + HIST_ELEM_SIZE*dist_code] + + ; hash = compute_hash(state->file_start + f_i) & hash_mask; + and hash %+ d, hmask2 %+ d + and hash2 %+ d, hmask2 %+ d + + ; continue + jmp .loop2 + +.len_dist_huffman_finish: + sub file_start, tmp1 + + ; get_len_code(len, &code, &code_len); + lea len_code, [len + 254] + or dist_code, len_code + + mov [hash_table + 2 * hash], tmp3 %+ w + add tmp3,1 + mov [hash_table + 2 * hash2], tmp3 %+ w + + inc dword [lit_len_hist + HIST_ELEM_SIZE*len_code] + + mov dword [m_out_buf], dist_code %+ d + add m_out_buf, 4 + + shr dist_code, DIST_OFFSET + and dist_code, 0x1F + inc dword [dist_hist + HIST_ELEM_SIZE*dist_code] + + jmp .input_end + +.write_lit_bits: + MOVQ curr_data, xdata + + add f_i, 1 + cmp f_i, file_length + jg .write_lit_bits_finish + + MOVDQU xdata, [file_start + f_i] + + inc dword [lit_len_hist + HIST_ELEM_SIZE*lit_code2] + + shl lit_code2, DIST_OFFSET + lea lit_code, [lit_code + lit_code2 + (31 << DIST_OFFSET)] + + mov dword [m_out_buf], lit_code %+ d + add m_out_buf, 4 + + cmp f_i, [rsp + skip_match_offset] + jle .loop2 + + xor tmp3, tmp3 + mov rcx, [rsp + skip_level_offset] + add rcx, 1 + cmovl rcx, tmp3 + mov tmp1, MAX_SKIP_LEVEL + cmp rcx, MAX_SKIP_LEVEL + cmovg rcx, tmp1 + + mov tmp1, SKIP_SIZE_BASE + shr tmp1, cl + +%if MAX_SKIP_LEVEL > 63 + cmp rcx, 63 + cmovg tmp1, tmp3 +%endif + mov [rsp + skip_match_offset], tmp1 + mov [rsp + skip_level_offset], rcx + + sub rcx, SKIP_START + cmovl rcx, tmp3 + + lea skip_count, [SKIP_RATE * rcx + SKIP_BASE] + and skip_count, -SKIP_BASE + + mov tmp1, [rsp + m_out_end] + lea tmp1, [tmp1 + 4] + sub tmp1, m_out_buf + shr tmp1, 1 + cmp tmp1, skip_count + jl .skip_forward_short + + mov tmp1, [rsp + inbuf_slop_offset] + add tmp1, file_length + sub tmp1, f_i + cmp tmp1, skip_count + jl .skip_forward_short + +.skip_forward_long: + MOVQ xdata, [file_start + f_i] + + movzx lit_code, byte [file_start + f_i] + movzx lit_code2, byte [file_start + f_i + 1] + + add dword [lit_len_hist + HIST_ELEM_SIZE*lit_code], 1 + add dword [lit_len_hist + HIST_ELEM_SIZE*lit_code2], 1 + + movzx lit_code, byte [file_start + f_i + 2] + movzx lit_code2, byte [file_start + f_i + 3] + + add dword [lit_len_hist + HIST_ELEM_SIZE*lit_code], 1 + add dword [lit_len_hist + HIST_ELEM_SIZE*lit_code2], 1 + + movzx lit_code, byte [file_start + f_i + 4] + movzx lit_code2, byte [file_start + f_i + 5] + + add dword [lit_len_hist + HIST_ELEM_SIZE*lit_code], 1 + add dword [lit_len_hist + HIST_ELEM_SIZE*lit_code2], 1 + + movzx lit_code, byte [file_start + f_i + 6] + movzx lit_code2, byte [file_start + f_i + 7] + + add dword [lit_len_hist + HIST_ELEM_SIZE*lit_code], 1 + add dword [lit_len_hist + HIST_ELEM_SIZE*lit_code2], 1 + + PSHUFB xtmp0, xdata, xlow_lit_shuf + PSHUFB xtmp1, xdata, xup_lit_shuf + PSLLD xtmp1, xtmp1, DIST_OFFSET + POR xtmp0, xtmp0, xtmp1 + PADDD xtmp0, xtmp0, xlit + MOVDQU [m_out_buf], xtmp0 + + add m_out_buf, 16 + add f_i, 8 + + sub skip_count, 8 + jg .skip_forward_long + + cmp file_length, f_i + jle .input_end + + mov curr_data, [file_start + f_i] + MOVDQU xdata, [file_start + f_i] + add [rsp + skip_match_offset], f_i + + jmp .loop2 + +.skip_forward_short: + movzx lit_code, byte [file_start + f_i] + movzx lit_code2, byte [file_start + f_i + 1] + + inc dword [lit_len_hist + HIST_ELEM_SIZE*lit_code] + inc dword [lit_len_hist + HIST_ELEM_SIZE*lit_code2] + + shl lit_code2, DIST_OFFSET + lea lit_code, [lit_code + lit_code2 + (31 << DIST_OFFSET)] + + mov dword [m_out_buf], lit_code %+ d + add m_out_buf, 4 + add f_i, 2 + + cmp m_out_buf, [rsp + m_out_end] + ja .output_end + + cmp file_length, f_i + jle .input_end + + jmp .skip_forward_short + +.write_lit_bits_finish: + inc dword [lit_len_hist + HIST_ELEM_SIZE*lit_code2] + + shl lit_code2, DIST_OFFSET + lea lit_code, [lit_code + lit_code2 + (31 << DIST_OFFSET)] + + mov dword [m_out_buf], lit_code %+ d + add m_out_buf, 4 + +.input_end: + mov stream, [rsp + stream_offset] + mov tmp1, ZSTATE_FLUSH_READ_BUFFER + mov tmp2, ZSTATE_BODY + cmp word [stream + _end_of_stream], 0 + cmovne tmp2, tmp1 + cmp word [stream + _flush], _NO_FLUSH + + cmovne tmp2, tmp1 + mov dword [stream + _internal_state_state], tmp2 %+ d + jmp .end + +.output_end: + mov stream, [rsp + stream_offset] + mov dword [stream + _internal_state_state], ZSTATE_CREATE_HDR + +.end: + ;; update input buffer + add file_length, [rsp + inbuf_slop_offset] + mov [stream + _total_in], f_i %+ d + mov [stream + _internal_state_block_end], f_i %+ d + add file_start, f_i + mov [stream + _next_in], file_start + sub file_length, f_i + mov [stream + _avail_in], file_length %+ d + + ;; update output buffer + mov [level_buf + _icf_buf_next], m_out_buf + sub m_out_buf, [rsp + m_out_start] + sub [level_buf + _icf_buf_avail_out], m_out_buf %+ d + + mov rbx, [rsp + gpr_save_mem_offset + 0*8] + mov rsi, [rsp + gpr_save_mem_offset + 1*8] + mov rdi, [rsp + gpr_save_mem_offset + 2*8] + mov rbp, [rsp + gpr_save_mem_offset + 3*8] + mov r12, [rsp + gpr_save_mem_offset + 4*8] + mov r13, [rsp + gpr_save_mem_offset + 5*8] + mov r14, [rsp + gpr_save_mem_offset + 6*8] + mov r15, [rsp + gpr_save_mem_offset + 7*8] + +%ifndef ALIGN_STACK + add rsp, stack_size +%else + mov rsp, rbp + pop rbp +%endif + ret + +align 16 +.compare_loop: + lea tmp2, [tmp1 + dist - 1] + + mov len2, file_length + sub len2, f_i + add len2, [rsp + inbuf_slop_offset] + add len2, 1 + mov tmp3, MAX_EMIT_SIZE + cmp len2, tmp3 + cmovg len2, tmp3 + + mov len, 8 + compare_large tmp1, tmp2, len, len2, tmp3, ytmp0, ytmp1 + + cmp len, 258 + jle .len_dist_huffman + cmp len, LARGE_MATCH_MIN + jge .do_emit + mov len, 258 + jmp .len_dist_huffman + +align 16 +.compare_loop2: + lea tmp2, [tmp1 + dist2] + add tmp1, 1 + + mov len, file_length + sub len, f_i + add len, [rsp + inbuf_slop_offset] + mov tmp3, MAX_EMIT_SIZE + cmp len, tmp3 + cmovg len, tmp3 + + mov len2, 8 + compare_large tmp1, tmp2, len2, len, tmp3, ytmp0, ytmp1 + + movzx lit_code, curr_data %+ b + shr curr_data, 8 + inc dword [lit_len_hist + HIST_ELEM_SIZE*lit_code] + cmp len2, 258 + jle .len_dist_lit_huffman + cmp len2, LARGE_MATCH_MIN + jge .do_emit2 + mov len2, 258 + jmp .len_dist_lit_huffman + +.do_emit2: + or lit_code, LIT + mov dword [m_out_buf], lit_code %+ d + add m_out_buf, 4 + + inc f_i + mov dist, dist2 + mov len, len2 + +.do_emit: + neg dist + get_dist_icf_code dist, dist_code, tmp1 + + mov len_code2, 258 + 254 + or len_code2, dist_code + mov tmp1, dist_code + shr tmp1, DIST_OFFSET + and tmp1, 0x1F + lea tmp3, [f_i + 1] + dec f_i + + mov [hash_table + 2 * hash], tmp3 %+ w + add tmp3,1 + mov [hash_table + 2 * hash2], tmp3 %+ w +.emit: + sub len, 258 + add f_i, 258 + + inc dword [lit_len_hist + HIST_ELEM_SIZE*(258 + 254)] + inc dword [dist_hist + HIST_ELEM_SIZE*tmp1] + mov dword [m_out_buf], len_code2 %+ d + add m_out_buf, 4 + + cmp m_out_buf, [rsp + m_out_end] + ja .output_end + + cmp len, LARGE_MATCH_MIN + jge .emit + + mov len2, 258 + cmp len, len2 + cmovg len, len2 + + ; get_len_code(len, &code, &code_len); + add f_i, len + lea len_code, [len + 254] + or dist_code, len_code + + inc dword [lit_len_hist + HIST_ELEM_SIZE*len_code] + inc dword [dist_hist + HIST_ELEM_SIZE*tmp1] + + mov dword [m_out_buf], dist_code %+ d + add m_out_buf, 4 + + cmp file_length, f_i + jle .input_end + + lea tmp2, [f_i - 4 * LARGE_MATCH_HASH_REP] + mov hmask2 %+ d, [rsp + hash_mask_offset] + +%rep LARGE_MATCH_HASH_REP + mov curr_data %+ d, dword [file_start + tmp2] + mov curr_data2 %+ d, dword [file_start + tmp2 + 1] + mov tmp3 %+ d, dword [file_start + tmp2 + 2] + mov tmp1 %+ d, dword [file_start + tmp2 + 3] + + compute_hash hash, curr_data + compute_hash hash2, curr_data2 + compute_hash hash3, tmp3 + compute_hash hmask3, tmp1 + + and hash %+ d, hmask2 %+ d + and hash2 %+ d, hmask2 %+ d + and hash3 %+ d, hmask2 %+ d + and hmask3 %+ d, hmask2 %+ d + + mov [hash_table + 2 * hash], tmp2 %+ w + add tmp2, 1 + mov [hash_table + 2 * hash2], tmp2 %+ w + add tmp2, 1 + mov [hash_table + 2 * hash3], tmp2 %+ w + add tmp2, 1 + mov [hash_table + 2 * hmask3], tmp2 %+ w +%if (LARGE_MATCH_HASH_REP > 1) + add tmp2, 1 +%endif +%endrep + ; for (f_i = f_start_i; f_i < file_length; f_i++) { + MOVDQU xdata, [file_start + f_i] + mov curr_data, [file_start + f_i] + mov tmp1, curr_data + + compute_hash hash, curr_data + + shr tmp1, 8 + compute_hash hash2, tmp1 + + and hash, hmask2 + and hash2, hmask2 + + jmp .loop2 + +.write_first_byte: + mov hmask1 %+ d, [rsp + hash_mask_offset] + cmp m_out_buf, [rsp + m_out_end] + ja .output_end + + mov byte [stream + _internal_state_has_hist], IGZIP_HIST + + mov [hash_table + 2 * hash], f_i %+ w + + mov hash, hash2 + shr tmp2, 16 + compute_hash hash2, tmp2 + + and curr_data, 0xff + inc dword [lit_len_hist + HIST_ELEM_SIZE*curr_data] + or curr_data, LIT + + mov dword [m_out_buf], curr_data %+ d + add m_out_buf, 4 + + MOVDQU xdata, [file_start + f_i + 1] + add f_i, 1 + mov curr_data, [file_start + f_i] + and hash %+ d, hmask1 %+ d + and hash2 %+ d, hmask1 %+ d + + cmp f_i, file_length + jl .loop2 + jmp .input_end + +%ifdef USE_HSWNI +%undef USE_HSWNI +%endif + +;; Shift defines over in order to iterate over all versions +%undef ARCH +%xdefine ARCH ARCH1 +%undef ARCH1 +%xdefine ARCH1 ARCH2 + +%ifdef COMPARE_TYPE_NOT_DEF +%undef COMPARE_TYPE +%xdefine COMPARE_TYPE COMPARE_TYPE1 +%undef COMPARE_TYPE1 +%xdefine COMPARE_TYPE1 COMPARE_TYPE2 +%endif +%endrep +min_lit_dist_syms: + dd LIT + (1 << DIST_OFFSET) + +low_lit_shuf: + db 0x00, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff + db 0x04, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff +up_lit_shuf: + db 0x01, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff + db 0x05, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff |