Adding upstream version 1.0~rc2.upstream/1.0_rc2

Signed-off-by: Daniel Baumann <mail@daniel-baumann.ch>

1 files changed, 646 insertions, 0 deletions

diff --git a/encoder.c b/encoder.c
new file mode 100644
index 0000000..a46893a
--- /dev/null
+++ b/encoder.c
@@ -0,0 +1,646 @@
+/*  Clzip - A data compressor based on the LZMA algorithm
+    Copyright (C) 2010 Antonio Diaz Diaz.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#define _FILE_OFFSET_BITS 64
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#include "clzip.h"
+#include "encoder.h"
+
+
+Dis_slots dis_slots;
+Prob_prices prob_prices;
+
+
+bool Mf_read_block( struct Matchfinder * const matchfinder )
+  {
+  const int size = matchfinder->buffer_size - matchfinder->stream_pos;
+  const int rd = readblock( matchfinder->infd_, matchfinder->buffer + matchfinder->stream_pos, size );
+  matchfinder->stream_pos += rd;
+  if( rd < size ) matchfinder->at_stream_end = true;
+  return ( rd == size || !errno );
+  }
+
+
+void Mf_init( struct Matchfinder * const matchfinder,
+              const int dict_size, const int len_limit, const int infd )
+  {
+  matchfinder->partial_data_pos = 0;
+  matchfinder->pos = 0;
+  matchfinder->cyclic_pos = 0;
+  matchfinder->stream_pos = 0;
+  matchfinder->infd_ = infd;
+  matchfinder->match_len_limit_ = len_limit;
+  matchfinder->prev_positions = (int32_t *)malloc( mf_num_prev_positions * sizeof (int32_t) );
+  if( !matchfinder->prev_positions )
+    {
+    show_error( "not enough memory. Try a smaller dictionary size", 0, false );
+    cleanup_and_fail( 1 );
+    }
+  matchfinder->at_stream_end = false;
+
+  const int buffer_size_limit = ( 2 * dict_size ) + mf_before_size + mf_after_size;
+  matchfinder->buffer_size = max( 65536, dict_size );
+  matchfinder->buffer = (uint8_t *)malloc( matchfinder->buffer_size );
+  if( !matchfinder->buffer )
+    {
+    show_error( "not enough memory. Try a smaller dictionary size", 0, false );
+    cleanup_and_fail( 1 );
+    }
+  if( !Mf_read_block( matchfinder ) )
+    { show_error( "read error", errno, false ); cleanup_and_fail( 1 ); }
+  if( !matchfinder->at_stream_end && matchfinder->buffer_size < buffer_size_limit )
+    {
+    matchfinder->buffer_size = buffer_size_limit;
+    matchfinder->buffer = (uint8_t *)realloc( matchfinder->buffer, matchfinder->buffer_size * sizeof (uint8_t) );
+    if( !matchfinder->buffer )
+      {
+      show_error( "not enough memory. Try a smaller dictionary size", 0, false );
+      cleanup_and_fail( 1 );
+      }
+    if( !Mf_read_block( matchfinder ) )
+      { show_error( "read error", errno, false ); cleanup_and_fail( 1 ); }
+    }
+  if( matchfinder->at_stream_end && matchfinder->stream_pos < dict_size )
+    matchfinder->dictionary_size_ = max( min_dictionary_size, matchfinder->stream_pos );
+  else matchfinder->dictionary_size_ = dict_size;
+  matchfinder->pos_limit = matchfinder->buffer_size;
+  if( !matchfinder->at_stream_end ) matchfinder->pos_limit -= mf_after_size;
+  matchfinder->prev_pos_tree = (int32_t *)malloc( 2 * matchfinder->dictionary_size_ * sizeof (int32_t) );
+  if( !matchfinder->prev_pos_tree )
+    {
+    show_error( "not enough memory. Try a smaller dictionary size", 0, false );
+    cleanup_and_fail( 1 );
+    }
+  for( int i = 0; i < mf_num_prev_positions; ++i )
+    matchfinder->prev_positions[i] = -1;
+  }
+
+
+bool Mf_reset( struct Matchfinder * const matchfinder )
+  {
+  const int size = matchfinder->stream_pos - matchfinder->pos;
+  if( size > 0 ) memmove( matchfinder->buffer, matchfinder->buffer + matchfinder->pos, size );
+  matchfinder->partial_data_pos = 0;
+  matchfinder->stream_pos -= matchfinder->pos;
+  matchfinder->pos = 0;
+  matchfinder->cyclic_pos = 0;
+  for( int i = 0; i < mf_num_prev_positions; ++i )
+    matchfinder->prev_positions[i] = -1;
+  return ( matchfinder->at_stream_end || Mf_read_block( matchfinder ) );
+  }
+
+
+bool Mf_move_pos( struct Matchfinder * const matchfinder )
+  {
+  if( ++matchfinder->cyclic_pos >= matchfinder->dictionary_size_ )
+    matchfinder->cyclic_pos = 0;
+  if( ++matchfinder->pos >= matchfinder->pos_limit )
+    {
+    if( matchfinder->pos > matchfinder->stream_pos )
+      { matchfinder->pos = matchfinder->stream_pos; return false; }
+    if( !matchfinder->at_stream_end )
+      {
+      const int offset = matchfinder->pos - matchfinder->dictionary_size_ - mf_before_size;
+      const int size = matchfinder->stream_pos - offset;
+      memmove( matchfinder->buffer, matchfinder->buffer + offset, size );
+      matchfinder->partial_data_pos += offset;
+      matchfinder->pos -= offset;
+      matchfinder->stream_pos -= offset;
+      for( int i = 0; i < mf_num_prev_positions; ++i )
+        if( matchfinder->prev_positions[i] >= 0 ) matchfinder->prev_positions[i] -= offset;
+      for( int i = 0; i < 2 * matchfinder->dictionary_size_; ++i )
+        if( matchfinder->prev_pos_tree[i] >= 0 ) matchfinder->prev_pos_tree[i] -= offset;
+      return Mf_read_block( matchfinder );
+      }
+    }
+  return true;
+  }
+
+
+int Mf_longest_match_len( struct Matchfinder * const matchfinder,
+                          int * const distances )
+  {
+  int len_limit = matchfinder->match_len_limit_;
+  if( len_limit > Mf_available_bytes( matchfinder ) )
+    {
+    len_limit = Mf_available_bytes( matchfinder );
+    if( len_limit < 4 ) return 0;
+    }
+
+  int maxlen = min_match_len - 1;
+  const int min_pos = (matchfinder->pos >= matchfinder->dictionary_size_) ?
+                      (matchfinder->pos - matchfinder->dictionary_size_ + 1) : 0;
+  const uint8_t * const data = matchfinder->buffer + matchfinder->pos;
+  const int key2 = mf_num_prev_positions4 + mf_num_prev_positions3 +
+                   ( ( (int)data[0] << 8 ) | data[1] );
+  const int tmp = crc32[data[0]] ^ data[1] ^ ( (int)data[2] << 8 );
+  const int key3 = mf_num_prev_positions4 + ( tmp & ( mf_num_prev_positions3 - 1 ) );
+  const int key4 = ( tmp ^ ( crc32[data[3]] << 5 ) ) &
+                   ( mf_num_prev_positions4 - 1 );
+
+  if( distances )
+    {
+    int np = matchfinder->prev_positions[key2];
+    if( np >= min_pos )
+      { distances[2] = matchfinder->pos - np - 1; maxlen = 2; }
+    else distances[2] = 0x7FFFFFFF;
+    np = matchfinder->prev_positions[key3];
+    if( np >= min_pos && matchfinder->buffer[np] == data[0] )
+      { distances[3] = matchfinder->pos - np - 1; maxlen = 3; }
+    else distances[3] = 0x7FFFFFFF;
+    distances[4] = 0x7FFFFFFF;
+    }
+
+  matchfinder->prev_positions[key2] = matchfinder->pos;
+  matchfinder->prev_positions[key3] = matchfinder->pos;
+  int newpos = matchfinder->prev_positions[key4];
+  matchfinder->prev_positions[key4] = matchfinder->pos;
+
+  int idx0 = matchfinder->cyclic_pos << 1;
+  int idx1 = idx0 + 1;
+  int len = 0, len0 = 0, len1 = 0;
+
+  for( int count = 16 + ( matchfinder->match_len_limit_ / 2 ); ; )
+    {
+    if( newpos < min_pos || --count < 0 )
+      { matchfinder->prev_pos_tree[idx0] = matchfinder->prev_pos_tree[idx1] = -1; break; }
+    const uint8_t * const newdata = matchfinder->buffer + newpos;
+    while( len < len_limit && newdata[len] == data[len] ) ++len;
+
+    const int delta = matchfinder->pos - newpos;
+    if( distances ) while( maxlen < len ) distances[++maxlen] = delta - 1;
+
+    const int newidx = ( matchfinder->cyclic_pos - delta +
+      ( ( matchfinder->cyclic_pos >= delta ) ? 0 : matchfinder->dictionary_size_ ) ) << 1;
+
+    if( len < len_limit )
+      {
+      if( newdata[len] < data[len] )
+        {
+        matchfinder->prev_pos_tree[idx0] = newpos;
+        idx0 = newidx + 1;
+        newpos = matchfinder->prev_pos_tree[idx0];
+        len0 = len; if( len1 < len ) len = len1;
+        }
+      else
+        {
+        matchfinder->prev_pos_tree[idx1] = newpos;
+        idx1 = newidx;
+        newpos = matchfinder->prev_pos_tree[idx1];
+        len1 = len; if( len0 < len ) len = len0;
+        }
+      }
+    else
+      {
+      matchfinder->prev_pos_tree[idx0] = matchfinder->prev_pos_tree[newidx];
+      matchfinder->prev_pos_tree[idx1] = matchfinder->prev_pos_tree[newidx+1];
+      break;
+      }
+    }
+  if( distances )
+    {
+    if( distances[3] > distances[4] ) distances[3] = distances[4];
+    if( distances[2] > distances[3] ) distances[2] = distances[3];
+    }
+  return maxlen;
+  }
+
+
+void Lee_encode( struct Len_encoder * const len_encoder,
+                 struct Range_encoder * const range_encoder,
+                 int symbol, const int pos_state )
+  {
+  symbol -= min_match_len;
+  if( symbol < len_low_symbols )
+    {
+    Re_encode_bit( range_encoder, &len_encoder->choice1, 0 );
+    Re_encode_tree( range_encoder, len_encoder->bm_low[pos_state], symbol, len_low_bits );
+    }
+  else
+    {
+    Re_encode_bit( range_encoder, &len_encoder->choice1, 1 );
+    if( symbol < len_low_symbols + len_mid_symbols )
+      {
+      Re_encode_bit( range_encoder, &len_encoder->choice2, 0 );
+      Re_encode_tree( range_encoder, len_encoder->bm_mid[pos_state], symbol - len_low_symbols, len_mid_bits );
+      }
+    else
+      {
+      Re_encode_bit( range_encoder, &len_encoder->choice2, 1 );
+      Re_encode_tree( range_encoder, len_encoder->bm_high, symbol - len_low_symbols - len_mid_symbols, len_high_bits );
+      }
+    }
+  if( --len_encoder->counters[pos_state] <= 0 )
+    Lee_update_prices( len_encoder, pos_state );
+  }
+
+
+void LZe_fill_align_prices( struct LZ_encoder * const encoder )
+  {
+  for( int i = 0; i < dis_align_size; ++i )
+    encoder->align_prices[i] = price_symbol_reversed( encoder->bm_align, i, dis_align_bits );
+  encoder->align_price_count = dis_align_size;
+  }
+
+
+void LZe_fill_distance_prices( struct LZ_encoder * const encoder )
+  {
+  for( int dis_state = 0; dis_state < max_dis_states; ++dis_state )
+    {
+    int * const dsp = encoder->dis_slot_prices[dis_state];
+    const Bit_model * const bmds = encoder->bm_dis_slot[dis_state];
+    int slot = 0;
+    for( ; slot < end_dis_model && slot < encoder->num_dis_slots; ++slot )
+      dsp[slot] = price_symbol( bmds, slot, dis_slot_bits );
+    for( ; slot < encoder->num_dis_slots; ++slot )
+      dsp[slot] = price_symbol( bmds, slot, dis_slot_bits ) +
+                  (((( slot >> 1 ) - 1 ) - dis_align_bits ) << price_shift );
+
+    int * const dp = encoder->dis_prices[dis_state];
+    int dis = 0;
+    for( ; dis < start_dis_model; ++dis )
+      dp[dis] = dsp[dis];
+    for( ; dis < modeled_distances; ++dis )
+      {
+      const int dis_slot = get_slot( dis );
+      const int direct_bits = ( dis_slot >> 1 ) - 1;
+      const int base = ( 2 | ( dis_slot & 1 ) ) << direct_bits;
+      dp[dis] = dsp[dis_slot] +
+          price_symbol_reversed( encoder->bm_dis + base - dis_slot, dis - base, direct_bits );
+      }
+    }
+  }
+
+
+// Return value: ( dis == -1 ) && ( len == 1 ) means literal
+int LZe_best_pair_sequence( struct LZ_encoder * const encoder, const int reps[num_rep_distances],
+                            const State state )
+  {
+  int main_len;
+  if( encoder->longest_match_found > 0 )		// from previous call
+    {
+    main_len = encoder->longest_match_found;
+    encoder->longest_match_found = 0;
+    }
+  else main_len = LZe_read_match_distances( encoder );
+
+  int replens[num_rep_distances];
+  int rep_index = 0;
+  for( int i = 0; i < num_rep_distances; ++i )
+    {
+    replens[i] = Mf_true_match_len( encoder->matchfinder, 0, reps[i] + 1, max_match_len );
+    if( replens[i] > replens[rep_index] ) rep_index = i;
+    }
+  if( replens[rep_index] >= Mf_match_len_limit( encoder->matchfinder ) )
+    {
+    encoder->trials[0].dis = rep_index;
+    encoder->trials[0].price = replens[rep_index];
+    if( !LZe_move_pos( encoder, replens[rep_index], true ) ) return 0;
+    return replens[rep_index];
+    }
+
+  if( main_len >= Mf_match_len_limit( encoder->matchfinder ) )
+    {
+    encoder->trials[0].dis = encoder->match_distances[Mf_match_len_limit( encoder->matchfinder )] +
+                             num_rep_distances;
+    encoder->trials[0].price = main_len;
+    if( !LZe_move_pos( encoder, main_len, true ) ) return 0;
+    return main_len;
+    }
+
+  encoder->trials[0].state = state;
+  for( int i = 0; i < num_rep_distances; ++i ) encoder->trials[0].reps[i] = reps[i];
+
+  const uint8_t prev_byte = Mf_peek( encoder->matchfinder, -1 );
+  const uint8_t cur_byte = Mf_peek( encoder->matchfinder, 0 );
+  const uint8_t match_byte = Mf_peek( encoder->matchfinder, -reps[0]-1 );
+  unsigned int position = Mf_data_position( encoder->matchfinder );
+  const int pos_state = position & pos_state_mask;
+
+  encoder->trials[1].dis = -1;
+  encoder->trials[1].prev_index = 0;
+  encoder->trials[1].price = price0( encoder->bm_match[state][pos_state] );
+  if( St_is_char( state ) )
+    encoder->trials[1].price += Lie_price_symbol( &encoder->literal_encoder, prev_byte, cur_byte );
+  else
+    encoder->trials[1].price += Lie_price_matched( &encoder->literal_encoder, prev_byte, cur_byte, match_byte );
+
+  const int match_price = price1( encoder->bm_match[state][pos_state] );
+  const int rep_match_price = match_price + price1( encoder->bm_rep[state] );
+
+  if( match_byte == cur_byte )
+    Tr_update( &encoder->trials[1], 0, 0, rep_match_price + LZe_price_rep_len1( encoder, state, pos_state ) );
+
+  if( main_len < min_match_len )
+    {
+    encoder->trials[0].dis = encoder->trials[1].dis;
+    encoder->trials[0].price = 1;
+    if( !Mf_move_pos( encoder->matchfinder ) ) return 0;
+    return 1;
+    }
+
+  {
+  const int normal_match_price = match_price + price0( encoder->bm_rep[state] );
+  int len = min_match_len;
+  if( main_len <= replens[rep_index] )
+    {
+    main_len = replens[rep_index];
+    for( ; len <= main_len; ++len )
+      encoder->trials[len].price = lze_infinite_price;
+    }
+  else for( ; len <= main_len; ++len )
+    {
+    encoder->trials[len].dis = encoder->match_distances[len] + num_rep_distances;
+    encoder->trials[len].prev_index = 0;
+    encoder->trials[len].price = normal_match_price +
+                                 LZe_price_pair( encoder, encoder->match_distances[len], len, pos_state );
+    }
+  }
+
+  for( int rep = 0; rep < num_rep_distances; ++rep )
+    {
+    const int price = rep_match_price +
+                      LZe_price_rep( encoder, rep, state, pos_state );
+    for( int len = min_match_len; len <= replens[rep]; ++len )
+      Tr_update( &encoder->trials[len], rep, 0, price +
+                 Lee_price( &encoder->rep_match_len_encoder, len, pos_state ) );
+    }
+
+  int cur = 0;
+  int num_trials = main_len;
+  if( !Mf_move_pos( encoder->matchfinder ) ) return 0;
+
+  while( true )
+    {
+    if( ++cur >= num_trials )
+      {
+      LZe_backward( encoder, cur );
+      return cur;
+      }
+    const int newlen = LZe_read_match_distances( encoder );
+    if( newlen >= Mf_match_len_limit( encoder->matchfinder ) )
+      {
+      encoder->longest_match_found = newlen;
+      LZe_backward( encoder, cur );
+      return cur;
+      }
+
+    struct Trial * const cur_trial = &encoder->trials[cur];
+    const int prev_index = cur_trial->prev_index;
+
+    cur_trial->state = encoder->trials[prev_index].state;
+
+    for( int i = 0; i < num_rep_distances; ++i )
+      cur_trial->reps[i] = encoder->trials[prev_index].reps[i];
+    if( prev_index == cur - 1 )
+      {
+      if( cur_trial->dis == 0 ) St_set_short_rep( &cur_trial->state );
+      else St_set_char( &cur_trial->state );
+      }
+    else
+      {
+      if( cur_trial->dis < num_rep_distances ) St_set_rep( &cur_trial->state );
+      else St_set_match( &cur_trial->state );
+      LZe_mtf_reps( cur_trial->dis, cur_trial->reps );
+      }
+
+    const uint8_t prev_byte = Mf_peek( encoder->matchfinder, -1 );
+    const uint8_t cur_byte = Mf_peek( encoder->matchfinder, 0 );
+    const uint8_t match_byte = Mf_peek( encoder->matchfinder, -cur_trial->reps[0]-1 );
+    const int pos_state = ++position & pos_state_mask;
+    int next_price = cur_trial->price + price0( encoder->bm_match[cur_trial->state][pos_state] );
+    if( St_is_char( cur_trial->state ) )
+      next_price += Lie_price_symbol( &encoder->literal_encoder, prev_byte, cur_byte );
+    else
+      next_price += Lie_price_matched( &encoder->literal_encoder, prev_byte, cur_byte, match_byte );
+    if( !Mf_move_pos( encoder->matchfinder ) ) return 0;
+
+    struct Trial * const next_trial = &encoder->trials[cur+1];
+
+    Tr_update( next_trial, -1, cur, next_price );
+
+    const int match_price = cur_trial->price + price1( encoder->bm_match[cur_trial->state][pos_state] );
+    const int rep_match_price = match_price + price1( encoder->bm_rep[cur_trial->state] );
+
+    if( match_byte == cur_byte && next_trial->dis != 0 )
+      Tr_update( next_trial, 0, cur, rep_match_price +
+                 LZe_price_rep_len1( encoder, cur_trial->state, pos_state ) );
+
+    const int len_limit = min( min( max_num_trials - 1 - cur,
+                          Mf_available_bytes( encoder->matchfinder ) ),
+                          Mf_match_len_limit( encoder->matchfinder ) );
+    if( len_limit < min_match_len ) continue;
+
+    for( int rep = 0; rep < num_rep_distances; ++rep )
+      {
+      const int dis = cur_trial->reps[rep] + 1;
+      int len = 0;
+      const uint8_t * const data = Mf_ptr_to_current_pos( encoder->matchfinder ) - 1;
+      while( len < len_limit && data[len] == data[len-dis] ) ++len;
+      if( len >= min_match_len )
+        {
+        const int price = rep_match_price +
+                          LZe_price_rep( encoder, rep, cur_trial->state, pos_state );
+        while( num_trials < cur + len )
+          encoder->trials[++num_trials].price = lze_infinite_price;
+        for( ; len >= min_match_len; --len )
+          Tr_update( &encoder->trials[cur+len], rep, cur, price +
+                     Lee_price( &encoder->rep_match_len_encoder, len, pos_state ) );
+        }
+      }
+
+    if( newlen <= len_limit &&
+        ( newlen > min_match_len ||
+          ( newlen == min_match_len &&
+            encoder->match_distances[newlen] < modeled_distances ) ) )
+      {
+      const int normal_match_price = match_price +
+                                     price0( encoder->bm_rep[cur_trial->state] );
+      while( num_trials < cur + newlen )
+        encoder->trials[++num_trials].price = lze_infinite_price;
+
+      for( int len = min_match_len; len <= newlen; ++len )
+        Tr_update( &encoder->trials[cur+len], encoder->match_distances[len] + num_rep_distances,
+                   cur, normal_match_price +
+                        LZe_price_pair( encoder, encoder->match_distances[len], len, pos_state ) );
+      }
+    }
+  }
+
+
+     // End Of Stream mark => (dis == 0xFFFFFFFF, len == min_match_len)
+void LZe_full_flush( struct LZ_encoder * const encoder, const State state )
+  {
+  const int pos_state = ( Mf_data_position( encoder->matchfinder ) ) & pos_state_mask;
+  Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[state][pos_state], 1 );
+  Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep[state], 0 );
+  LZe_encode_pair( encoder, 0xFFFFFFFF, min_match_len, pos_state );
+  Re_flush( &encoder->range_encoder );
+  File_trailer trailer;
+  Ft_set_data_crc( trailer, LZe_crc( encoder ) );
+  Ft_set_data_size( trailer, Mf_data_position( encoder->matchfinder ) );
+  Ft_set_member_size( trailer, LZe_member_position( encoder ) + sizeof (File_trailer) );
+  for( unsigned int i = 0; i < sizeof (File_trailer); ++i )
+    Re_put_byte( &encoder->range_encoder, trailer[i] );
+  Re_flush_data( &encoder->range_encoder );
+  }
+
+
+void LZe_init( struct LZ_encoder * const encoder, struct Matchfinder * const mf,
+               const File_header header, const int outfd )
+  {
+  encoder->longest_match_found = 0;
+  encoder->crc_ = 0xFFFFFFFF;
+
+  for( int i = 0; i < St_states; ++i )
+    {
+    for( int j = 0; j < pos_states; ++j )
+      {
+      Bm_init( &encoder->bm_match[i][j] );
+      Bm_init( &encoder->bm_len[i][j] );
+      }
+    Bm_init( &encoder->bm_rep[i] );
+    Bm_init( &encoder->bm_rep0[i] );
+    Bm_init( &encoder->bm_rep1[i] );
+    Bm_init( &encoder->bm_rep2[i] );
+    }
+  for( int i = 0; i < max_dis_states; ++i )
+    for( int j = 0; j < 1<<dis_slot_bits; ++j )
+      Bm_init( &encoder->bm_dis_slot[i][j] );
+  for( int i = 0; i < modeled_distances-end_dis_model; ++i )
+    Bm_init( &encoder->bm_dis[i] );
+  for( int i = 0; i < dis_align_size; ++i )
+    Bm_init( &encoder->bm_align[i] );
+
+  encoder->matchfinder = mf;
+  Re_init( &encoder->range_encoder, outfd );
+  Lee_init( &encoder->len_encoder, Mf_match_len_limit( encoder->matchfinder ) ),
+  Lee_init( &encoder->rep_match_len_encoder, Mf_match_len_limit( encoder->matchfinder ) ),
+  Lie_init( &encoder->literal_encoder );
+  encoder->num_dis_slots = 2 * Fh_real_bits( Mf_dictionary_size( encoder->matchfinder ) - 1 );
+
+  LZe_fill_align_prices( encoder );
+
+  for( unsigned int i = 0; i < sizeof (File_header); ++i )
+    Re_put_byte( &encoder->range_encoder, header[i] );
+  }
+
+
+bool LZe_encode_member( struct LZ_encoder * const encoder, const long long member_size )
+  {
+  if( LZe_member_position( encoder ) != sizeof (File_header) )
+    return false;		// can be called only once
+  const long long member_size_limit =
+    member_size - sizeof (File_trailer) - lze_max_marker_size;
+  int fill_counter = 0;
+  int rep_distances[num_rep_distances];
+  State state = 0;
+  for( int i = 0; i < num_rep_distances; ++i ) rep_distances[i] = 0;
+
+  // encode first byte
+  if( Mf_data_position( encoder->matchfinder ) == 0 &&
+      !Mf_finished( encoder->matchfinder ) )
+    {
+    Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[state][0], 0 );
+    const uint8_t prev_byte = 0;
+    const uint8_t cur_byte = Mf_peek( encoder->matchfinder, 0 );
+    Lie_encode( &encoder->literal_encoder, &encoder->range_encoder, prev_byte, cur_byte );
+    CRC32_update_byte( &encoder->crc_, cur_byte );
+    if( !LZe_move_pos( encoder, 1, false ) ) return false;
+    }
+
+  while( true )
+    {
+    if( Mf_finished( encoder->matchfinder ) )
+      { LZe_full_flush( encoder, state ); return true; }
+    if( fill_counter <= 0 )
+      { LZe_fill_distance_prices( encoder ); fill_counter = 512; }
+
+    int ahead = LZe_best_pair_sequence( encoder, rep_distances, state );
+    if( ahead <= 0 ) return false;
+    fill_counter -= ahead;
+
+    for( int i = 0; ; )
+      {
+      const int pos_state = ( Mf_data_position( encoder->matchfinder ) - ahead ) & pos_state_mask;
+      int dis = encoder->trials[i].dis;
+      const int len = encoder->trials[i].price;
+
+      bool bit = ( dis < 0 && len == 1 );
+      Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[state][pos_state], !bit );
+      if( bit )
+        {
+        const uint8_t prev_byte = Mf_peek( encoder->matchfinder, -ahead-1 );
+        const uint8_t cur_byte = Mf_peek( encoder->matchfinder, -ahead );
+        if( St_is_char( state ) )
+          Lie_encode( &encoder->literal_encoder, &encoder->range_encoder, prev_byte, cur_byte );
+        else
+          {
+          const uint8_t match_byte = Mf_peek( encoder->matchfinder, -ahead-rep_distances[0]-1 );
+          Lie_encode_matched( &encoder->literal_encoder, &encoder->range_encoder, prev_byte, match_byte, cur_byte );
+          }
+        St_set_char( &state );
+        }
+      else
+        {
+        LZe_mtf_reps( dis, rep_distances );
+        bit = ( dis < num_rep_distances );
+        Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep[state], bit );
+        if( bit )
+          {
+          bit = ( dis == 0 );
+          Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep0[state], !bit );
+          if( bit )
+            Re_encode_bit( &encoder->range_encoder, &encoder->bm_len[state][pos_state], len > 1 );
+          else
+            {
+            Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep1[state], dis > 1 );
+            if( dis > 1 )
+              Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep2[state], dis > 2 );
+            }
+          if( len == 1 ) St_set_short_rep( &state );
+          else
+            {
+            Lee_encode( &encoder->rep_match_len_encoder, &encoder->range_encoder, len, pos_state );
+            St_set_rep( &state );
+            }
+          }
+        else
+          {
+          LZe_encode_pair( encoder, dis - num_rep_distances, len, pos_state );
+          St_set_match( &state );
+          }
+        }
+      for( int j = 0; j < len; ++j )
+        CRC32_update_byte( &encoder->crc_, Mf_peek( encoder->matchfinder, j-ahead ) );
+      ahead -= len; i += len;
+      if( LZe_member_position( encoder ) >= member_size_limit )
+        {
+        if( !Mf_dec_pos( encoder->matchfinder, ahead ) ) return false;
+        LZe_full_flush( encoder, state );
+        return true;
+        }
+      if( ahead <= 0 ) break;
+      }
+    }
+  }