summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--ChangeLog6
-rw-r--r--Makefile.in16
-rw-r--r--NEWS4
-rw-r--r--README45
-rw-r--r--bbexample.c2
-rwxr-xr-xconfigure12
-rw-r--r--doc/lzlib.info71
-rw-r--r--doc/lzlib.texi54
-rw-r--r--doc/minilzip.16
-rw-r--r--encoder.c106
-rw-r--r--encoder.h125
-rw-r--r--lzcheck.c2
-rw-r--r--lzlib.h2
-rw-r--r--main.c18
-rwxr-xr-xtestsuite/check.sh24
15 files changed, 307 insertions, 186 deletions
diff --git a/ChangeLog b/ChangeLog
index cfb81b6..3582c40 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+2014-03-30 Antonio Diaz Diaz <antonio@gnu.org>
+
+ * Version 1.6-pre3 released.
+ * Compression ratio has been slightly increased.
+ * configure: Added new option '--disable-static'.
+
2014-01-30 Antonio Diaz Diaz <antonio@gnu.org>
* Version 1.6-pre2 released.
diff --git a/Makefile.in b/Makefile.in
index ebdef06..6f69b0a 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -15,7 +15,7 @@ objs = carg_parser.o main.o
install-as-lzip uninstall uninstall-bin uninstall-info uninstall-man \
doc info man check dist clean distclean
-all : $(progname) $(progname_shared)
+all : $(progname_static) $(progname_shared)
lib$(libname).a : lzlib.o
$(AR) -rcs $@ $<
@@ -73,7 +73,7 @@ $(VPATH)/doc/$(progname).1 : $(progname)
Makefile : $(VPATH)/configure $(VPATH)/Makefile.in
./config.status
-check : all bbexample lzcheck
+check : $(progname) bbexample lzcheck
@$(VPATH)/testsuite/check.sh $(VPATH)/testsuite $(pkgversion)
install : install-bin install-info
@@ -82,17 +82,17 @@ install-bin : all
if [ ! -d "$(DESTDIR)$(includedir)" ] ; then $(INSTALL_DIR) "$(DESTDIR)$(includedir)" ; fi
if [ ! -d "$(DESTDIR)$(libdir)" ] ; then $(INSTALL_DIR) "$(DESTDIR)$(libdir)" ; fi
$(INSTALL_DATA) $(VPATH)/$(libname)lib.h "$(DESTDIR)$(includedir)/$(libname)lib.h"
- $(INSTALL_DATA) ./lib$(libname).a "$(DESTDIR)$(libdir)/lib$(libname).a"
+ if [ -n "$(progname_static)" ] ; then \
+ $(INSTALL_DATA) ./lib$(libname).a "$(DESTDIR)$(libdir)/lib$(libname).a" ; \
+ fi
if [ -n "$(progname_shared)" ] ; then \
$(INSTALL_PROGRAM) ./lib$(libname).so.$(pkgversion) "$(DESTDIR)$(libdir)/lib$(libname).so.$(pkgversion)" ; \
if [ -e "$(DESTDIR)$(libdir)/lib$(libname).so.$(soversion)" ] ; then \
- rm -f "$(DESTDIR)$(libdir)/lib$(libname).so.$(soversion)" ; \
run_ldconfig=no ; \
else run_ldconfig=yes ; \
fi ; \
- if [ -e "$(DESTDIR)$(libdir)/lib$(libname).so" ] ; then \
- rm -f "$(DESTDIR)$(libdir)/lib$(libname).so" ; \
- fi ; \
+ rm -f "$(DESTDIR)$(libdir)/lib$(libname).so" ; \
+ rm -f "$(DESTDIR)$(libdir)/lib$(libname).so.$(soversion)" ; \
cd "$(DESTDIR)$(libdir)" && ln -s lib$(libname).so.$(pkgversion) lib$(libname).so ; \
cd "$(DESTDIR)$(libdir)" && ln -s lib$(libname).so.$(pkgversion) lib$(libname).so.$(soversion) ; \
if [ "${disable_ldconfig}" != yes ] && [ $${run_ldconfig} = yes ] && \
@@ -113,7 +113,7 @@ install-strip : all
install-as-lzip : install install-man
if [ ! -d "$(DESTDIR)$(bindir)" ] ; then $(INSTALL_DIR) "$(DESTDIR)$(bindir)" ; fi
- $(INSTALL_PROGRAM) ./$(progname) "$(DESTDIR)$(bindir)/$(progname)"
+ $(INSTALL_PROGRAM) ./$(progname_lzip) "$(DESTDIR)$(bindir)/$(progname)"
-rm -f "$(DESTDIR)$(bindir)/lzip"
cd "$(DESTDIR)$(bindir)" && ln -s $(progname) lzip
diff --git a/NEWS b/NEWS
index b1d1c27..260e7ca 100644
--- a/NEWS
+++ b/NEWS
@@ -1,5 +1,9 @@
Changes in version 1.6:
+Compression ratio has been slightly increased.
+
+The configure script now accepts the option "--disable-static".
+
Improved portability to BSD systems:
The configure script now accepts the option "--disable-ldconfig".
diff --git a/README b/README
index 2e66843..dc98c4f 100644
--- a/README
+++ b/README
@@ -5,9 +5,24 @@ and decompression functions, including integrity checking of the
decompressed data. The compressed data format used by the library is the
lzip format. Lzlib is written in C.
-The lzip file format is designed for long-term data archiving. It is
-clean, provides very safe 4 factor integrity checking, and is backed by
-the recovery capabilities of lziprecover.
+The lzip file format is designed for long-term data archiving, taking
+into account both data integrity and decoder availability:
+
+ * The lzip format provides very safe integrity checking and some data
+ recovery means. The lziprecover program can repair bit-flip errors
+ (one of the most common forms of data corruption) in lzip files,
+ and provides data recovery capabilities, including error-checked
+ merging of damaged copies of a file.
+
+ * The lzip format is as simple as possible (but not simpler). The
+ lzip manual provides the code of a simple decompressor along with a
+ detailed explanation of how it works, so that with the only help of
+ the lzip manual it would be possible for a digital archaeologist to
+ extract the data from a lzip file long after quantum computers
+ eventually render LZMA obsolete.
+
+ * Additionally lzip is copylefted, which guarantees that it will
+ remain free forever.
The functions and variables forming the interface of the compression
library are declared in the file 'lzlib.h'. Usage examples of the
@@ -34,13 +49,23 @@ All the library functions are thread safe. The library does not install
any signal handler. The decoder checks the consistency of the compressed
data, so the library should never crash even in case of corrupted input.
-Lzlib implements a simplified version of the LZMA (Lempel-Ziv-Markov
-chain-Algorithm) algorithm. The high compression of LZMA comes from
-combining two basic, well-proven compression ideas: sliding dictionaries
-(LZ77/78) and markov models (the thing used by every compression
-algorithm that uses a range encoder or similar order-0 entropy coder as
-its last stage) with segregation of contexts according to what the bits
-are used for.
+There is no such thing as a "LZMA algorithm"; it is more like a "LZMA
+coding scheme". For example, the option '-0' of lzip uses the scheme in
+almost the simplest way possible; issuing the longest match it can find,
+or a literal byte if it can't find a match. Conversely, a much more
+elaborated way of finding coding sequences of minimum price than the one
+currently used by lzip could be developed, and the resulting sequence
+could also be coded using the LZMA coding scheme.
+
+Lzip currently implements two variants of the LZMA algorithm; fast (used
+by option -0) and normal (used by all other compression levels). Lzlib
+just implements the "normal" variant.
+
+The high compression of LZMA comes from combining two basic, well-proven
+compression ideas: sliding dictionaries (LZ77/78) and markov models (the
+thing used by every compression algorithm that uses a range encoder or
+similar order-0 entropy coder as its last stage) with segregation of
+contexts according to what the bits are used for.
The ideas embodied in lzlib are due to (at least) the following people:
Abraham Lempel and Jacob Ziv (for the LZ algorithm), Andrey Markov (for
diff --git a/bbexample.c b/bbexample.c
index 7a8e08c..da15738 100644
--- a/bbexample.c
+++ b/bbexample.c
@@ -34,7 +34,7 @@ uint8_t * bbcompress( const uint8_t * const data, const int size,
struct LZ_Encoder * encoder;
uint8_t * new_data;
const int match_len_limit = 36;
- const unsigned long long member_size = INT64_MAX;
+ const unsigned long long member_size = 0x7FFFFFFFFFFFFFFFULL; /* INT64_MAX */
int delta_size, new_data_size;
int new_pos = 0;
int written = 0;
diff --git a/configure b/configure
index 77f568e..e108358 100755
--- a/configure
+++ b/configure
@@ -6,10 +6,12 @@
# to copy, distribute and modify it.
pkgname=lzlib
-pkgversion=1.6-pre2
+pkgversion=1.6-pre3
soversion=1
progname=minilzip
+progname_static=${progname}
progname_shared=
+progname_lzip=${progname}
disable_ldconfig=
libname=lz
srctrigger=doc/${pkgname}.texi
@@ -72,6 +74,8 @@ while [ $# != 0 ] ; do
echo " --infodir=DIR info files directory [${infodir}]"
echo " --libdir=DIR object code libraries [${libdir}]"
echo " --mandir=DIR man pages directory [${mandir}]"
+ echo " --disable-static do not build a static library [enable]"
+ echo " (implies --enable-shared)"
echo " --enable-shared build also a shared library [disable]"
echo " --disable-ldconfig do not run ldconfig after install"
echo " CC=COMPILER C compiler to use [gcc]"
@@ -103,6 +107,10 @@ while [ $# != 0 ] ; do
--libdir=*) libdir=${optarg} ;;
--mandir=*) mandir=${optarg} ;;
--no-create) no_create=yes ;;
+ --disable-static)
+ progname_static=
+ progname_shared=${progname}_shared
+ progname_lzip=${progname}_shared ;;
--enable-shared) progname_shared=${progname}_shared ;;
--disable-ldconfig) disable_ldconfig=yes ;;
@@ -193,7 +201,9 @@ pkgname = ${pkgname}
pkgversion = ${pkgversion}
soversion = ${soversion}
progname = ${progname}
+progname_static = ${progname_static}
progname_shared = ${progname_shared}
+progname_lzip = ${progname_lzip}
disable_ldconfig = ${disable_ldconfig}
libname = ${libname}
VPATH = ${srcdir}
diff --git a/doc/lzlib.info b/doc/lzlib.info
index 1344ced..2017f3f 100644
--- a/doc/lzlib.info
+++ b/doc/lzlib.info
@@ -11,7 +11,7 @@ File: lzlib.info, Node: Top, Next: Introduction, Up: (dir)
Lzlib Manual
************
-This manual is for Lzlib (version 1.6-pre2, 30 January 2014).
+This manual is for Lzlib (version 1.6-pre3, 30 March 2014).
* Menu:
@@ -45,9 +45,24 @@ and decompression functions, including integrity checking of the
decompressed data. The compressed data format used by the library is the
lzip format. Lzlib is written in C.
- The lzip file format is designed for long-term data archiving. It is
-clean, provides very safe 4 factor integrity checking, and is backed by
-the recovery capabilities of lziprecover.
+ The lzip file format is designed for long-term data archiving, taking
+into account both data integrity and decoder availability:
+
+ * The lzip format provides very safe integrity checking and some data
+ recovery means. The lziprecover program can repair bit-flip errors
+ (one of the most common forms of data corruption) in lzip files,
+ and provides data recovery capabilities, including error-checked
+ merging of damaged copies of a file.
+
+ * The lzip format is as simple as possible (but not simpler). The
+ lzip manual provides the code of a simple decompressor along with
+ a detailed explanation of how it works, so that with the only help
+ of the lzip manual it would be possible for a digital
+ archaeologist to extract the data from a lzip file long after
+ quantum computers eventually render LZMA obsolete.
+
+ * Additionally lzip is copylefted, which guarantees that it will
+ remain free forever.
The functions and variables forming the interface of the compression
library are declared in the file 'lzlib.h'. Usage examples of the
@@ -75,13 +90,23 @@ install any signal handler. The decoder checks the consistency of the
compressed data, so the library should never crash even in case of
corrupted input.
- Lzlib implements a simplified version of the LZMA (Lempel-Ziv-Markov
-chain-Algorithm) algorithm. The high compression of LZMA comes from
-combining two basic, well-proven compression ideas: sliding dictionaries
-(LZ77/78) and markov models (the thing used by every compression
-algorithm that uses a range encoder or similar order-0 entropy coder as
-its last stage) with segregation of contexts according to what the bits
-are used for.
+ There is no such thing as a "LZMA algorithm"; it is more like a "LZMA
+coding scheme". For example, the option '-0' of lzip uses the scheme in
+almost the simplest way possible; issuing the longest match it can find,
+or a literal byte if it can't find a match. Conversely, a much more
+elaborated way of finding coding sequences of minimum price than the one
+currently used by lzip could be developed, and the resulting sequence
+could also be coded using the LZMA coding scheme.
+
+ Lzip currently implements two variants of the LZMA algorithm; fast
+(used by option -0) and normal (used by all other compression levels).
+Lzlib just implements the "normal" variant.
+
+ The high compression of LZMA comes from combining two basic,
+well-proven compression ideas: sliding dictionaries (LZ77/78) and
+markov models (the thing used by every compression algorithm that uses
+a range encoder or similar order-0 entropy coder as its last stage)
+with segregation of contexts according to what the bits are used for.
The ideas embodied in lzlib are due to (at least) the following
people: Abraham Lempel and Jacob Ziv (for the LZ algorithm), Andrey
@@ -736,18 +761,18 @@ Concept index

Tag Table:
Node: Top220
-Node: Introduction1335
-Node: Library version3916
-Node: Buffering4561
-Node: Parameter limits5684
-Node: Compression functions6643
-Node: Decompression functions12856
-Node: Error codes19017
-Node: Error messages20956
-Node: Data format21535
-Node: Examples24184
-Node: Problems28267
-Node: Concept index28839
+Node: Introduction1333
+Node: Library version5247
+Node: Buffering5892
+Node: Parameter limits7015
+Node: Compression functions7974
+Node: Decompression functions14187
+Node: Error codes20348
+Node: Error messages22287
+Node: Data format22866
+Node: Examples25515
+Node: Problems29598
+Node: Concept index30170

End Tag Table
diff --git a/doc/lzlib.texi b/doc/lzlib.texi
index 88f0b16..658db32 100644
--- a/doc/lzlib.texi
+++ b/doc/lzlib.texi
@@ -6,8 +6,8 @@
@finalout
@c %**end of header
-@set UPDATED 30 January 2014
-@set VERSION 1.6-pre2
+@set UPDATED 30 March 2014
+@set VERSION 1.6-pre3
@dircategory Data Compression
@direntry
@@ -66,9 +66,29 @@ and decompression functions, including integrity checking of the
decompressed data. The compressed data format used by the library is the
lzip format. Lzlib is written in C.
-The lzip file format is designed for long-term data archiving. It is
-clean, provides very safe 4 factor integrity checking, and is backed by
-the recovery capabilities of lziprecover.
+The lzip file format is designed for long-term data archiving, taking
+into account both data integrity and decoder availability:
+
+@itemize @bullet
+@item
+The lzip format provides very safe integrity checking and some data
+recovery means. The lziprecover program can repair bit-flip errors (one
+of the most common forms of data corruption) in lzip files, and provides
+data recovery capabilities, including error-checked merging of damaged
+copies of a file.
+
+@item
+The lzip format is as simple as possible (but not simpler). The lzip
+manual provides the code of a simple decompressor along with a detailed
+explanation of how it works, so that with the only help of the lzip
+manual it would be possible for a digital archaeologist to extract the
+data from a lzip file long after quantum computers eventually render
+LZMA obsolete.
+
+@item
+Additionally lzip is copylefted, which guarantees that it will remain
+free forever.
+@end itemize
The functions and variables forming the interface of the compression
library are declared in the file @samp{lzlib.h}. Usage examples of the
@@ -95,13 +115,23 @@ All the library functions are thread safe. The library does not install
any signal handler. The decoder checks the consistency of the compressed
data, so the library should never crash even in case of corrupted input.
-Lzlib implements a simplified version of the LZMA (Lempel-Ziv-Markov
-chain-Algorithm) algorithm. The high compression of LZMA comes from
-combining two basic, well-proven compression ideas: sliding dictionaries
-(LZ77/78) and markov models (the thing used by every compression
-algorithm that uses a range encoder or similar order-0 entropy coder as
-its last stage) with segregation of contexts according to what the bits
-are used for.
+There is no such thing as a "LZMA algorithm"; it is more like a "LZMA
+coding scheme". For example, the option '-0' of lzip uses the scheme in
+almost the simplest way possible; issuing the longest match it can find,
+or a literal byte if it can't find a match. Conversely, a much more
+elaborated way of finding coding sequences of minimum price than the one
+currently used by lzip could be developed, and the resulting sequence
+could also be coded using the LZMA coding scheme.
+
+Lzip currently implements two variants of the LZMA algorithm; fast (used
+by option -0) and normal (used by all other compression levels). Lzlib
+just implements the "normal" variant.
+
+The high compression of LZMA comes from combining two basic, well-proven
+compression ideas: sliding dictionaries (LZ77/78) and markov models (the
+thing used by every compression algorithm that uses a range encoder or
+similar order-0 entropy coder as its last stage) with segregation of
+contexts according to what the bits are used for.
The ideas embodied in lzlib are due to (at least) the following people:
Abraham Lempel and Jacob Ziv (for the LZ algorithm), Andrey Markov (for
diff --git a/doc/minilzip.1 b/doc/minilzip.1
index 0535797..b61a79f 100644
--- a/doc/minilzip.1
+++ b/doc/minilzip.1
@@ -1,7 +1,7 @@
.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.37.1.
-.TH MINILZIP "1" "January 2014" "Minilzip 1.6-pre2" "User Commands"
+.TH MINILZIP "1" "March 2014" "minilzip 1.6-pre3" "User Commands"
.SH NAME
-Minilzip \- reduces the size of files
+minilzip \- reduces the size of files
.SH SYNOPSIS
.B minilzip
[\fIoptions\fR] [\fIfiles\fR]
@@ -83,7 +83,7 @@ Report bugs to lzip\-bug@nongnu.org
Lzlib home page: http://www.nongnu.org/lzip/lzlib.html
.SH COPYRIGHT
Copyright \(co 2014 Antonio Diaz Diaz.
-Using Lzlib 1.6\-pre2
+Using lzlib 1.6\-pre3
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
.br
This is free software: you are free to change and redistribute it.
diff --git a/encoder.c b/encoder.c
index 3a22a25..3c28c4b 100644
--- a/encoder.c
+++ b/encoder.c
@@ -231,36 +231,6 @@ static int Mf_get_match_pairs( struct Matchfinder * const mf, struct Pair * pair
}
-static void Lee_encode( struct Len_encoder * const le,
- struct Range_encoder * const renc,
- int symbol, const int pos_state )
- {
- symbol -= min_match_len;
- if( symbol < len_low_symbols )
- {
- Re_encode_bit( renc, &le->lm.choice1, 0 );
- Re_encode_tree( renc, le->lm.bm_low[pos_state], symbol, len_low_bits );
- }
- else
- {
- Re_encode_bit( renc, &le->lm.choice1, 1 );
- if( symbol < len_low_symbols + len_mid_symbols )
- {
- Re_encode_bit( renc, &le->lm.choice2, 0 );
- Re_encode_tree( renc, le->lm.bm_mid[pos_state],
- symbol - len_low_symbols, len_mid_bits );
- }
- else
- {
- Re_encode_bit( renc, &le->lm.choice2, 1 );
- Re_encode_tree( renc, le->lm.bm_high,
- symbol - len_low_symbols - len_mid_symbols, len_high_bits );
- }
- }
- if( --le->counters[pos_state] <= 0 ) Lee_update_prices( le, pos_state );
- }
-
-
/* End Of Stream mark => (dis == 0xFFFFFFFFU, len == min_match_len) */
static bool LZe_full_flush( struct LZ_encoder * const e, const State state )
{
@@ -298,16 +268,7 @@ static bool LZe_sync_flush( struct LZ_encoder * const e )
}
-static void LZe_fill_align_prices( struct LZ_encoder * const e )
- {
- int i;
- for( i = 0; i < dis_align_size; ++i )
- e->align_prices[i] = price_symbol_reversed( e->bm_align, i, dis_align_bits );
- e->align_price_count = dis_align_size;
- }
-
-
-static void LZe_fill_distance_prices( struct LZ_encoder * const e )
+static void LZe_update_distance_prices( struct LZ_encoder * const e )
{
int dis, len_state;
for( dis = start_dis_model; dis < modeled_distances; ++dis )
@@ -364,12 +325,15 @@ static bool LZe_init( struct LZ_encoder * const e,
e->matchfinder = mf;
if( !Re_init( &e->renc ) ) return false;
- Lee_init( &e->match_len_encoder, mf->match_len_limit );
- Lee_init( &e->rep_len_encoder, mf->match_len_limit );
+ Lm_init( &e->match_len_model );
+ Lm_init( &e->rep_len_model );
+ Lp_init( &e->match_len_prices, &e->match_len_model, mf->match_len_limit );
+ Lp_init( &e->rep_len_prices, &e->rep_len_model, mf->match_len_limit );
for( i = 0; i < num_rep_distances; ++i ) e->reps[i] = 0;
- e->align_price_count = 0;
e->num_dis_slots = 2 * real_bits( mf->dictionary_size - 1 );
- e->fill_counter = 0;
+ e->price_counter = 0;
+ e->dis_price_counter = 0;
+ e->align_price_counter = 0;
e->state = 0;
e->member_finished = false;
@@ -382,6 +346,7 @@ static bool LZe_init( struct LZ_encoder * const e,
/* Return value == number of bytes advanced (ahead).
trials[0]..trials[ahead-1] contain the steps to encode.
( trials[0].dis == -1 ) means literal.
+ A match/rep longer or equal than match_len_limit finishes the sequence.
*/
static int LZe_sequence_optimizer( struct LZ_encoder * const e,
const int reps[num_rep_distances],
@@ -468,7 +433,7 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
for( len = min_match_len; len <= replens[rep]; ++len )
Tr_update( &e->trials[len], price +
- Lee_price( &e->rep_len_encoder, len, pos_state ), rep, 0 );
+ Lp_price( &e->rep_len_prices, len, pos_state ), rep, 0 );
}
if( main_len > replens[0] )
@@ -487,8 +452,6 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
}
}
- if( !Mf_move_pos( e->matchfinder ) ) return 0;
-
while( true ) /* price optimization loop */
{
struct Trial *cur_trial, *next_trial;
@@ -498,6 +461,7 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
State cur_state;
uint8_t prev_byte, cur_byte, match_byte;
+ if( !Mf_move_pos( e->matchfinder ) ) return 0;
if( ++cur >= num_trials ) /* no more initialized trials */
{
LZe_backward( e, cur );
@@ -557,7 +521,6 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
prev_byte = Mf_peek( e->matchfinder, 1 );
cur_byte = Mf_peek( e->matchfinder, 0 );
match_byte = Mf_peek( e->matchfinder, cur_trial->reps[0] + 1 );
- if( !Mf_move_pos( e->matchfinder ) ) return 0;
next_price = cur_trial->price +
price0( e->bm_match[cur_state][pos_state] );
@@ -587,7 +550,7 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
}
}
- available_bytes = min( Mf_available_bytes( e->matchfinder ) + 1,
+ available_bytes = min( Mf_available_bytes( e->matchfinder ),
max_num_trials - 1 - cur );
if( available_bytes < min_match_len ) continue;
@@ -596,7 +559,7 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
/* try literal + rep0 */
if( match_byte != cur_byte && next_trial->prev_index != cur )
{
- const uint8_t * const data = Mf_ptr_to_current_pos( e->matchfinder ) - 1;
+ const uint8_t * const data = Mf_ptr_to_current_pos( e->matchfinder );
const int dis = cur_trial->reps[0] + 1;
const int limit = min( e->matchfinder->match_len_limit + 1,
available_bytes );
@@ -619,7 +582,7 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
/* try rep distances */
for( rep = 0; rep < num_rep_distances; ++rep )
{
- const uint8_t * const data = Mf_ptr_to_current_pos( e->matchfinder ) - 1;
+ const uint8_t * const data = Mf_ptr_to_current_pos( e->matchfinder );
int price;
const int dis = cur_trial->reps[rep] + 1;
@@ -631,7 +594,7 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
price = rep_match_price + LZe_price_rep( e, rep, cur_state, pos_state );
for( i = min_match_len; i <= len; ++i )
Tr_update( &e->trials[cur+i], price +
- Lee_price( &e->rep_len_encoder, i, pos_state ), rep, cur );
+ Lp_price( &e->rep_len_prices, i, pos_state ), rep, cur );
if( rep == 0 ) start_len = len + 1; /* discard shorter matches */
@@ -647,7 +610,7 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
pos_state2 = ( pos_state + len ) & pos_state_mask;
state2 = St_set_rep( cur_state );
- price += Lee_price( &e->rep_len_encoder, len, pos_state ) +
+ price += Lp_price( &e->rep_len_prices, len, pos_state ) +
price0( e->bm_match[state2][pos_state2] ) +
LZe_price_matched( e, data[len-1], data[len], data[len-dis] );
pos_state2 = ( pos_state2 + 1 ) & pos_state_mask;
@@ -683,7 +646,7 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
/* try match + literal + rep0 */
if( len == e->pairs[i].len )
{
- const uint8_t * const data = Mf_ptr_to_current_pos( e->matchfinder ) - 1;
+ const uint8_t * const data = Mf_ptr_to_current_pos( e->matchfinder );
const int dis2 = dis + 1;
int len2 = len + 1;
const int limit = min( e->matchfinder->match_len_limit + len2,
@@ -718,7 +681,10 @@ static int LZe_sequence_optimizer( struct LZ_encoder * const e,
static bool LZe_encode_member( struct LZ_encoder * const e )
{
- const int fill_count = ( e->matchfinder->match_len_limit > 12 ) ? 128 : 512;
+ const bool best = ( e->matchfinder->match_len_limit > 12 );
+ const int dis_price_count = best ? 1 : 512;
+ const int align_price_count = best ? 1 : dis_align_size;
+ const int price_count = ( e->matchfinder->match_len_limit > 36 ) ? 1024 : 4096;
int ahead, i;
State * const state = &e->state;
@@ -733,12 +699,13 @@ static bool LZe_encode_member( struct LZ_encoder * const e )
!Mf_finished( e->matchfinder ) ) /* encode first byte */
{
const uint8_t prev_byte = 0;
- const uint8_t cur_byte = Mf_peek( e->matchfinder, 0 );
+ uint8_t cur_byte;
if( !Mf_enough_available_bytes( e->matchfinder ) ||
!Re_enough_free_bytes( &e->renc ) ) return true;
- CRC32_update_byte( &e->crc, cur_byte );
+ cur_byte = Mf_peek( e->matchfinder, 0 );
Re_encode_bit( &e->renc, &e->bm_match[*state][0], 0 );
LZe_encode_literal( e, prev_byte, cur_byte );
+ CRC32_update_byte( &e->crc, cur_byte );
Mf_get_match_pairs( e->matchfinder, 0 );
if( !Mf_move_pos( e->matchfinder ) ) return false;
}
@@ -747,15 +714,24 @@ static bool LZe_encode_member( struct LZ_encoder * const e )
{
if( !Mf_enough_available_bytes( e->matchfinder ) ||
!Re_enough_free_bytes( &e->renc ) ) return true;
- if( e->pending_num_pairs == 0 )
+ if( e->price_counter <= 0 && e->pending_num_pairs == 0 )
{
- if( e->fill_counter <= 0 )
- { LZe_fill_distance_prices( e ); e->fill_counter = fill_count; }
- if( e->align_price_count <= 0 ) LZe_fill_align_prices( e );
+ e->price_counter = price_count; /* recalculate prices every these bytes */
+ if( e->dis_price_counter <= 0 )
+ { e->dis_price_counter = dis_price_count; LZe_update_distance_prices( e ); }
+ if( e->align_price_counter <= 0 )
+ {
+ e->align_price_counter = align_price_count;
+ for( i = 0; i < dis_align_size; ++i )
+ e->align_prices[i] = price_symbol_reversed( e->bm_align, i, dis_align_bits );
+ }
+ Lp_update_prices( &e->match_len_prices );
+ Lp_update_prices( &e->rep_len_prices );
}
ahead = LZe_sequence_optimizer( e, e->reps, *state );
if( ahead <= 0 ) return false; /* can't happen */
+ e->price_counter -= ahead;
for( i = 0; ahead > 0; )
{
@@ -802,14 +778,18 @@ static bool LZe_encode_member( struct LZ_encoder * const e )
if( len == 1 ) *state = St_set_short_rep( *state );
else
{
- Lee_encode( &e->rep_len_encoder, &e->renc, len, pos_state );
+ Re_encode_len( &e->renc, &e->rep_len_model, len, pos_state );
+ Lp_decrement_counter( &e->rep_len_prices, pos_state );
*state = St_set_rep( *state );
}
}
else /* match */
{
LZe_encode_pair( e, dis - num_rep_distances, len, pos_state );
- --e->fill_counter;
+ if( get_slot( dis - num_rep_distances ) >= end_dis_model )
+ --e->align_price_counter;
+ --e->dis_price_counter;
+ Lp_decrement_counter( &e->match_len_prices, pos_state );
*state = St_set_match( *state );
}
}
diff --git a/encoder.h b/encoder.h
index f458bd9..2a92228 100644
--- a/encoder.h
+++ b/encoder.h
@@ -476,52 +476,93 @@ static inline void Re_encode_matched( struct Range_encoder * const renc,
while( symbol < 0x10000 );
}
+static inline void Re_encode_len( struct Range_encoder * const renc,
+ struct Len_model * const lm,
+ int symbol, const int pos_state )
+ {
+ bool bit = ( ( symbol -= min_match_len ) >= len_low_symbols );
+ Re_encode_bit( renc, &lm->choice1, bit );
+ if( !bit )
+ Re_encode_tree( renc, lm->bm_low[pos_state], symbol, len_low_bits );
+ else
+ {
+ bit = ( symbol >= len_low_symbols + len_mid_symbols );
+ Re_encode_bit( renc, &lm->choice2, bit );
+ if( !bit )
+ Re_encode_tree( renc, lm->bm_mid[pos_state],
+ symbol - len_low_symbols, len_mid_bits );
+ else
+ Re_encode_tree( renc, lm->bm_high,
+ symbol - len_low_symbols - len_mid_symbols, len_high_bits );
+ }
+ }
+
-struct Len_encoder
+struct Len_prices
{
- struct Len_model lm;
- int prices[pos_states][max_len_symbols];
+ const struct Len_model * lm;
int len_symbols;
+ int count;
+ int prices[pos_states][max_len_symbols];
int counters[pos_states];
};
-static void Lee_update_prices( struct Len_encoder * const le,
- const int pos_state )
+static inline void Lp_update_low_mid_prices( struct Len_prices * const lp,
+ const int pos_state )
{
- int * const pps = le->prices[pos_state];
- int tmp = price0( le->lm.choice1 );
+ int * const pps = lp->prices[pos_state];
+ int tmp = price0( lp->lm->choice1 );
int len = 0;
- for( ; len < len_low_symbols && len < le->len_symbols; ++len )
- pps[len] = tmp + price_symbol( le->lm.bm_low[pos_state], len, len_low_bits );
- tmp = price1( le->lm.choice1 );
- for( ; len < len_low_symbols + len_mid_symbols && len < le->len_symbols; ++len )
- pps[len] = tmp + price0( le->lm.choice2 ) +
- price_symbol( le->lm.bm_mid[pos_state], len - len_low_symbols, len_mid_bits );
- for( ; len < le->len_symbols; ++len )
- /* using 4 slots per value makes "Lee_price" faster */
- le->prices[3][len] = le->prices[2][len] =
- le->prices[1][len] = le->prices[0][len] =
- tmp + price1( le->lm.choice2 ) +
- price_symbol( le->lm.bm_high, len - len_low_symbols - len_mid_symbols, len_high_bits );
- le->counters[pos_state] = le->len_symbols;
- }
-
-static void Lee_init( struct Len_encoder * const le,
- const int match_len_limit )
+ lp->counters[pos_state] = lp->count;
+ for( ; len < len_low_symbols && len < lp->len_symbols; ++len )
+ pps[len] = tmp + price_symbol( lp->lm->bm_low[pos_state], len, len_low_bits );
+ if( len >= lp->len_symbols ) return;
+ tmp = price1( lp->lm->choice1 ) + price0( lp->lm->choice2 );
+ for( ; len < len_low_symbols + len_mid_symbols && len < lp->len_symbols; ++len )
+ pps[len] = tmp +
+ price_symbol( lp->lm->bm_mid[pos_state], len - len_low_symbols, len_mid_bits );
+ }
+
+static inline void Lp_update_high_prices( struct Len_prices * const lp )
+ {
+ const int tmp = price1( lp->lm->choice1 ) + price1( lp->lm->choice2 );
+ int len;
+ for( len = len_low_symbols + len_mid_symbols; len < lp->len_symbols; ++len )
+ /* using 4 slots per value makes "Lp_price" faster */
+ lp->prices[3][len] = lp->prices[2][len] =
+ lp->prices[1][len] = lp->prices[0][len] = tmp +
+ price_symbol( lp->lm->bm_high, len - len_low_symbols - len_mid_symbols, len_high_bits );
+ }
+
+static inline void Lp_init( struct Len_prices * const lp,
+ const struct Len_model * const lm,
+ const int match_len_limit )
{
int i;
- Lm_init( &le->lm );
- le->len_symbols = match_len_limit + 1 - min_match_len;
- for( i = 0; i < pos_states; ++i ) Lee_update_prices( le, i );
+ lp->lm = lm;
+ lp->len_symbols = match_len_limit + 1 - min_match_len;
+ lp->count = ( match_len_limit > 12 ) ? 1 : lp->len_symbols;
+ for( i = 0; i < pos_states; ++i ) lp->counters[i] = 0;
}
-static void Lee_encode( struct Len_encoder * const le,
- struct Range_encoder * const renc,
- int symbol, const int pos_state );
+static inline void Lp_decrement_counter( struct Len_prices * const lp,
+ const int pos_state )
+ { --lp->counters[pos_state]; }
+
+static inline void Lp_update_prices( struct Len_prices * const lp )
+ {
+ int pos_state;
+ bool high_pending = false;
+ for( pos_state = 0; pos_state < pos_states; ++pos_state )
+ if( lp->counters[pos_state] <= 0 )
+ { Lp_update_low_mid_prices( lp, pos_state ); high_pending = true; }
+ if( high_pending && lp->len_symbols > len_low_symbols + len_mid_symbols )
+ Lp_update_high_prices( lp );
+ }
-static inline int Lee_price( const struct Len_encoder * const le,
- const int symbol, const int pos_state )
- { return le->prices[pos_state][symbol - min_match_len]; }
+static inline int Lp_price( const struct Len_prices * const lp,
+ const int symbol, const int pos_state )
+ { return lp->prices[pos_state][symbol - min_match_len]; }
enum { infinite_price = 0x0FFFFFFF,
@@ -593,8 +634,10 @@ struct LZ_encoder
struct Matchfinder * matchfinder;
struct Range_encoder renc;
- struct Len_encoder match_len_encoder;
- struct Len_encoder rep_len_encoder;
+ struct Len_model match_len_model;
+ struct Len_model rep_len_model;
+ struct Len_prices match_len_prices;
+ struct Len_prices rep_len_prices;
struct Pair pairs[max_match_len+1];
struct Trial trials[max_num_trials];
@@ -603,9 +646,10 @@ struct LZ_encoder
int dis_slot_prices[len_states][2*max_dictionary_bits];
int dis_prices[len_states][modeled_distances];
int align_prices[dis_align_size];
- int align_price_count;
int num_dis_slots;
- int fill_counter;
+ int price_counter;
+ int dis_price_counter;
+ int align_price_counter;
State state;
bool member_finished;
};
@@ -669,14 +713,14 @@ static inline int LZe_price_rep0_len( const struct LZ_encoder * const e,
const State state, const int pos_state )
{
return LZe_price_rep( e, 0, state, pos_state ) +
- Lee_price( &e->rep_len_encoder, len, pos_state );
+ Lp_price( &e->rep_len_prices, len, pos_state );
}
static inline int LZe_price_pair( const struct LZ_encoder * const e,
const int dis, const int len,
const int pos_state )
{
- const int price = Lee_price( &e->match_len_encoder, len, pos_state );
+ const int price = Lp_price( &e->match_len_prices, len, pos_state );
const int len_state = get_len_state( len );
if( dis < modeled_distances )
return price + e->dis_prices[len_state][dis];
@@ -711,7 +755,7 @@ static inline void LZe_encode_pair( struct LZ_encoder * const e,
const int pos_state )
{
const int dis_slot = get_slot( dis );
- Lee_encode( &e->match_len_encoder, &e->renc, len, pos_state );
+ Re_encode_len( &e->renc, &e->match_len_model, len, pos_state );
Re_encode_tree( &e->renc, e->bm_dis_slot[get_len_state(len)], dis_slot,
dis_slot_bits );
@@ -729,7 +773,6 @@ static inline void LZe_encode_pair( struct LZ_encoder * const e,
Re_encode( &e->renc, direct_dis >> dis_align_bits,
direct_bits - dis_align_bits );
Re_encode_tree_reversed( &e->renc, e->bm_align, direct_dis, dis_align_bits );
- --e->align_price_count;
}
}
}
diff --git a/lzcheck.c b/lzcheck.c
index aa9195e..361a98c 100644
--- a/lzcheck.c
+++ b/lzcheck.c
@@ -38,7 +38,7 @@ int main( const int argc, const char * const argv[] )
{
const int dictionary_size = 1 << 20;
const int match_len_limit = 16;
- const unsigned long long member_size = INT64_MAX;
+ const unsigned long long member_size = 0x7FFFFFFFFFFFFFFFULL; /* INT64_MAX */
struct LZ_Encoder * encoder;
struct LZ_Decoder * decoder;
FILE * file;
diff --git a/lzlib.h b/lzlib.h
index e1e91a0..5fcf34d 100644
--- a/lzlib.h
+++ b/lzlib.h
@@ -29,7 +29,7 @@
extern "C" {
#endif
-static const char * const LZ_version_string = "1.6-pre2";
+static const char * const LZ_version_string = "1.6-pre3";
enum LZ_Errno { LZ_ok = 0, LZ_bad_argument, LZ_mem_error,
LZ_sequence_error, LZ_header_error, LZ_unexpected_eof,
diff --git a/main.c b/main.c
index ef12c33..5c53b00 100644
--- a/main.c
+++ b/main.c
@@ -35,7 +35,7 @@
#include <unistd.h>
#include <utime.h>
#include <sys/stat.h>
-#if defined(__MSVCRT__)
+#if defined(__MSVCRT__) || defined(_MSC_VER)
#include <io.h>
#define fchmod(x,y) 0
#define fchown(x,y,z) 0
@@ -202,9 +202,9 @@ static void show_help( void )
static void show_version( void )
{
- printf( "%s %s\n", Program_name, PROGVERSION );
+ printf( "%s %s\n", program_name, PROGVERSION );
printf( "Copyright (C) %s Antonio Diaz Diaz.\n", program_year );
- printf( "Using Lzlib %s\n", LZ_version() );
+ printf( "Using lzlib %s\n", LZ_version() );
printf( "License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>\n"
"This is free software: you are free to change and redistribute it.\n"
"There is NO WARRANTY, to the extent permitted by law.\n" );
@@ -322,8 +322,7 @@ static int open_instream( const char * const name, struct stat * const in_statsp
}
else
{
- do infd = open( name, O_RDONLY | O_BINARY );
- while( infd < 0 && errno == EINTR );
+ infd = open( name, O_RDONLY | O_BINARY );
if( infd < 0 )
{
if( verbosity >= 0 )
@@ -407,8 +406,7 @@ static bool open_outstream( const bool force )
int flags = O_CREAT | O_WRONLY | O_BINARY;
if( force ) flags |= O_TRUNC; else flags |= O_EXCL;
- do outfd = open( output_filename, flags, outfd_mode );
- while( outfd < 0 && errno == EINTR );
+ outfd = open( output_filename, flags, outfd_mode );
if( outfd < 0 && verbosity >= 0 )
{
if( errno == EEXIST )
@@ -497,7 +495,7 @@ static int readblock( const int fd, uint8_t * const buf, const int size )
const int n = read( fd, buf + size - rest, rest );
if( n > 0 ) rest -= n;
else if( n == 0 ) break; /* EOF */
- else if( errno != EINTR && errno != EAGAIN ) break;
+ else if( errno != EINTR ) break;
errno = 0;
}
return size - rest;
@@ -515,7 +513,7 @@ static int writeblock( const int fd, const uint8_t * const buf, const int size )
{
const int n = write( fd, buf + size - rest, rest );
if( n > 0 ) rest -= n;
- else if( n < 0 && errno != EINTR && errno != EAGAIN ) break;
+ else if( n < 0 && errno != EINTR ) break;
errno = 0;
}
return size - rest;
@@ -931,7 +929,7 @@ int main( const int argc, const char * const argv[] )
}
} /* end process options */
-#if defined(__MSVCRT__) || defined(__OS2__)
+#if defined(__MSVCRT__) || defined(__OS2__) || defined(_MSC_VER)
setmode( STDIN_FILENO, O_BINARY );
setmode( STDOUT_FILENO, O_BINARY );
#endif
diff --git a/testsuite/check.sh b/testsuite/check.sh
index 6665469..038c76d 100755
--- a/testsuite/check.sh
+++ b/testsuite/check.sh
@@ -14,7 +14,7 @@ BBEXAMPLE="${objdir}"/bbexample
LZCHECK="${objdir}"/lzcheck
framework_failure() { echo "failure in testing framework" ; exit 1 ; }
-if [ ! -x "${LZIP}" ] ; then
+if [ ! -f "${LZIP}" ] || [ ! -x "${LZIP}" ] ; then
echo "${LZIP}: cannot execute"
exit 1
fi
@@ -66,13 +66,13 @@ printf .
"${LZIP}" -cfq "${in_lz}" > out
if [ $? = 1 ] ; then printf . ; else fail=1 ; printf - ; fi
-"${LZIP}" -cF "${in_lz}" > out || fail=1
+"${LZIP}" -cF -s16 "${in_lz}" > out || fail=1
"${LZIP}" -cd out | "${LZIP}" -d > copy || fail=1
cmp in copy || fail=1
printf .
-for i in s4Ki 0 1 2 3 4 5 6 7s16 8s16 9s16 ; do
- "${LZIP}" -k -$i in || fail=1
+for i in s4Ki 0 1 2 3 4 5 6 7 8 9 ; do
+ "${LZIP}" -k -$i -s16 in || fail=1
mv -f in.lz copy.lz || fail=1
printf "garbage" >> copy.lz || fail=1
"${LZIP}" -df copy.lz || fail=1
@@ -80,35 +80,35 @@ for i in s4Ki 0 1 2 3 4 5 6 7s16 8s16 9s16 ; do
done
printf .
-for i in s4Ki 0 1 2 3 4 5 6 7s16 8s16 9s16 ; do
- "${LZIP}" -c -$i in > out || fail=1
+for i in s4Ki 0 1 2 3 4 5 6 7 8 9 ; do
+ "${LZIP}" -c -$i -s16 in > out || fail=1
printf "g" >> out || fail=1
"${LZIP}" -cd out > copy || fail=1
cmp in copy || fail=1
done
printf .
-for i in s4Ki 0 1 2 3 4 5 6 7s16 8s16 9s16 ; do
- "${LZIP}" -$i < in > out || fail=1
+for i in s4Ki 0 1 2 3 4 5 6 7 8 9 ; do
+ "${LZIP}" -$i -s16 < in > out || fail=1
"${LZIP}" -d < out > copy || fail=1
cmp in copy || fail=1
done
printf .
-for i in s4Ki 0 1 2 3 4 5 6 7s16 8s16 9s16 ; do
- "${LZIP}" -f -$i -o out < in || fail=1
+for i in s4Ki 0 1 2 3 4 5 6 7 8 9 ; do
+ "${LZIP}" -f -$i -s16 -o out < in || fail=1
"${LZIP}" -df -o copy < out.lz || fail=1
cmp in copy || fail=1
done
printf .
-"${LZIP}" < in > anyothername || fail=1
+"${LZIP}" -s16 < in > anyothername || fail=1
"${LZIP}" -d anyothername || fail=1
cmp in anyothername.out || fail=1
printf .
cat in in > in2 || framework_failure
-"${LZIP}" -o copy2 < in2 || fail=1
+"${LZIP}" -s16 -o copy2 < in2 || fail=1
"${LZIP}" -t copy2.lz || fail=1
printf .
"${LZIP}" -cd copy2.lz > copy2 || fail=1
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-08 23:38:51 +0000