diff options
| -rw-r--r-- | ChangeLog | 42 | ||||
| -rw-r--r-- | INSTALL | 28 | ||||
| -rw-r--r-- | Makefile.in | 66 | ||||
| -rw-r--r-- | NEWS | 15 | ||||
| -rw-r--r-- | README | 12 | ||||
| -rw-r--r-- | arg_parser.cc | 203 | ||||
| -rw-r--r-- | arg_parser.h | 105 | ||||
| -rw-r--r-- | bbexample.c (renamed from bbexample.cc) | 57 | ||||
| -rw-r--r-- | carg_parser.c | 294 | ||||
| -rw-r--r-- | carg_parser.h | 102 | ||||
| -rw-r--r-- | clzip.h | 232 | ||||
| -rwxr-xr-x | configure | 105 | ||||
| -rw-r--r-- | decoder.c | 734 | ||||
| -rw-r--r-- | decoder.cc | 289 | ||||
| -rw-r--r-- | decoder.h | 323 | ||||
| -rw-r--r-- | doc/lzlib.info | 58 | ||||
| -rw-r--r-- | doc/lzlib.texinfo | 31 | ||||
| -rw-r--r-- | encoder.c | 1334 | ||||
| -rw-r--r-- | encoder.cc | 663 | ||||
| -rw-r--r-- | encoder.h | 620 | ||||
| -rw-r--r-- | lzcheck.c (renamed from lzcheck.cc) | 128 | ||||
| -rw-r--r-- | lzip.h | 297 | ||||
| -rw-r--r-- | lzlib.c | 543 | ||||
| -rw-r--r-- | lzlib.cc | 540 | ||||
| -rw-r--r-- | lzlib.h | 4 | ||||
| -rw-r--r-- | main.c | 1036 | ||||
| -rw-r--r-- | main.cc | 966 | ||||
| -rw-r--r-- | tables.c | 395 | ||||
| -rwxr-xr-x | testsuite/check.sh | 9 |
29 files changed, 4950 insertions, 4281 deletions
@@ -1,24 +1,30 @@ +2012-02-29 Antonio Diaz Diaz <ant_diaz@teleline.es> + + * Version 1.3 released. + * Translated to C from the C++ source of lzlib 1.2. + * configure: 'datadir' renamed to 'datarootdir'. + 2011-10-25 Antonio Diaz Diaz <ant_diaz@teleline.es> * Version 1.2 released. * encoder.h (Lee_update_prices): Update high length symbol prices - independently of the value of `pos_state'. This gives better - compression for large values of `--match-length' without being + independently of the value of 'pos_state'. This gives better + compression for large values of '--match-length' without being slower. * encoder.h encoder.cc: Optimize pair price calculations. This - reduces compression time for large values of `--match-length' + reduces compression time for large values of '--match-length' by up to 6%. - * main.cc: Added new option `-F, --recompress'. - * Makefile.in: `make install' no more tries to run - `/sbin/ldconfig' on systems lacking it. + * main.cc: Added new option '-F, --recompress'. + * Makefile.in: 'make install' no more tries to run + '/sbin/ldconfig' on systems lacking it. 2011-01-03 Antonio Diaz Diaz <ant_diaz@teleline.es> * Version 1.1 released. * Compression time has been reduced by 2%. * All declarations not belonging to the API have been - encapsulated in the namespace `Lzlib'. - * testsuite: `test1' renamed to `test.txt'. Added new tests. + encapsulated in the namespace 'Lzlib'. + * testsuite: 'test1' renamed to 'test.txt'. Added new tests. * Match length limits set by options -1 to -9 of minilzip have been changed to match those of lzip 1.11. * main.cc: Set stdin/stdout in binary mode on OS2. @@ -32,7 +38,7 @@ * Added new function LZ_decompress_member_version. * Added new function LZ_decompress_dictionary_size. * Added new function LZ_decompress_data_crc. - * Variables declared `extern' have been encapsulated in a + * Variables declared 'extern' have been encapsulated in a namespace. * main.cc: Fixed warning about fchown's return value being ignored. * decoder.h: Input_buffer integrated in Range_decoder. @@ -41,7 +47,7 @@ * Version 0.9 released. * Compression time has been reduced by 8%. - * main.cc: New constant `o_binary'. + * main.cc: New constant 'o_binary'. 2010-01-17 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -50,18 +56,18 @@ * Added new function LZ_decompress_sync_to_member. * Added new function LZ_decompress_write_size. * Added new function LZ_strerror. - * lzlib.h: API change. Replaced `enum' with functions for values + * lzlib.h: API change. Replaced 'enum' with functions for values of dictionary size limits to make interface names consistent. - * lzlib.h: API change. `LZ_errno' replaced with `LZ_Errno'. - * lzlib.h: API change. Replaced `void *' with `struct LZ_Encoder *' - and `struct LZ_Decoder *' to make interface type safe. + * lzlib.h: API change. 'LZ_errno' replaced with 'LZ_Errno'. + * lzlib.h: API change. Replaced 'void *' with 'struct LZ_Encoder *' + and 'struct LZ_Decoder *' to make interface type safe. * decoder.cc: Truncated member trailer is now correctly detected. * encoder.cc: Matchfinder::reset now also clears at_stream_end_, allowing LZ_compress_restart_member to restart a finished stream. * lzlib.cc: Accept only query or close operations after a fatal error has occurred. * Shared version of lzlib is no more built by default. - * testsuite/check.sh: Use `test1' instead of `COPYING' for testing. + * testsuite/check.sh: Use 'test1' instead of 'COPYING' for testing. 2009-10-20 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -79,7 +85,7 @@ * Version 0.5 released. * Decompression speed has been improved. - * main.cc (signal_handler): Declared as `extern "C"'. + * main.cc (signal_handler): Declared as 'extern "C"'. 2009-06-03 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -87,7 +93,7 @@ * Added new function LZ_compress_sync_flush. * Added new function LZ_compress_write_size. * Decompression speed has been improved. - * Added chapter `Buffering' to the manual. + * Added chapter 'Buffering' to the manual. 2009-05-03 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -105,7 +111,7 @@ * Version 0.1 released. -Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. +Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. This file is a collection of facts, and thus it is not copyrightable, but just in case, you have unlimited permission to copy, distribute and @@ -1,6 +1,6 @@ Requirements ------------ -You will need a C++ compiler. +You will need a C compiler. I use gcc 4.3.5 and 3.3.6, but the code should compile with any standards compliant compiler. Gcc is available at http://gcc.gnu.org. @@ -18,7 +18,7 @@ This creates the directory ./lzlib[version] containing the source from the main archive. 2. Change to lzlib directory and run configure. - (Try `configure --help' for usage instructions). + (Try 'configure --help' for usage instructions). cd lzlib[version] ./configure @@ -27,30 +27,30 @@ the main archive. make -4. Optionally, type `make check' to run the tests that come with lzlib. +4. Optionally, type 'make check' to run the tests that come with lzlib. -5. Type `make install' to install the library and any data files and +5. Type 'make install' to install the library and any data files and documentation. (You might have to run ldconfig also). Another way ----------- You can also compile lzlib into a separate directory. To do this, you -must use a version of `make' that supports the `VPATH' variable, such -as GNU `make'. `cd' to the directory where you want the object files -and executables to go and run the `configure' script. `configure' -automatically checks for the source code in `.', in `..' and in the -directory that `configure' is in. - -`configure' recognizes the option `--srcdir=DIR' to control where to -look for the sources. Usually `configure' can determine that directory +must use a version of 'make' that supports the 'VPATH' variable, such +as GNU 'make'. 'cd' to the directory where you want the object files +and executables to go and run the 'configure' script. 'configure' +automatically checks for the source code in '.', in '..' and in the +directory that 'configure' is in. + +'configure' recognizes the option '--srcdir=DIR' to control where to +look for the sources. Usually 'configure' can determine that directory automatically. -After running `configure', you can run `make' and `make install' as +After running 'configure', you can run 'make' and 'make install' as explained above. -Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. +Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. This file is free documentation: you have unlimited permission to copy, distribute and modify it. diff --git a/Makefile.in b/Makefile.in index fd9c6b6..983f9a9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -8,9 +8,7 @@ INSTALL_DIR = $(INSTALL) -d -m 755 LDCONFIG = /sbin/ldconfig SHELL = /bin/sh -lib_objs = decoder.o encoder.o lzlib.o -sh_lib_objs = sh_decoder.o sh_encoder.o sh_lzlib.o -objs = arg_parser.o main.o +objs = carg_parser.o main.o .PHONY : all install install-info install-man install-strip \ @@ -19,55 +17,43 @@ objs = arg_parser.o main.o all : $(progname) $(progname_shared) -lib$(libname).a: $(lib_objs) - $(AR) -rcs $@ $^ +lib$(libname).a : lzlib.o + $(AR) -rcs $@ $< -lib$(libname).so.$(pkgversion) : $(sh_lib_objs) - $(CXX) -shared -Wl,--soname=lib$(libname).so.$(soversion) -o $@ $^ +lib$(libname).so.$(pkgversion) : lzlib_sh.o + $(CC) -shared -Wl,--soname=lib$(libname).so.$(soversion) -o $@ $< $(progname) : $(objs) lib$(libname).a - $(CXX) $(LDFLAGS) -o $@ $^ + $(CC) $(LDFLAGS) -o $@ $(objs) lib$(libname).a $(progname)_shared : $(objs) lib$(libname).so.$(pkgversion) - $(CXX) $(LDFLAGS) -o $@ $^ + $(CC) $(LDFLAGS) -o $@ $(objs) lib$(libname).so.$(pkgversion) $(progname)_profiled : $(objs) lib$(libname).a - $(CXX) $(LDFLAGS) -pg -o $@ $^ + $(CC) $(LDFLAGS) -pg -o $@ $(objs) lib$(libname).a bbexample : bbexample.o lib$(libname).a - $(CXX) $(LDFLAGS) -o $@ $^ + $(CC) $(LDFLAGS) -o $@ bbexample.o lib$(libname).a lzcheck : lzcheck.o lib$(libname).a - $(CXX) $(LDFLAGS) -o $@ $^ + $(CC) $(LDFLAGS) -o $@ lzcheck.o lib$(libname).a -main.o : main.cc - $(CXX) $(CPPFLAGS) $(CXXFLAGS) -DPROGVERSION=\"$(pkgversion)\" -c -o $@ $< +main.o : main.c + $(CC) $(CPPFLAGS) $(CFLAGS) -DPROGVERSION=\"$(pkgversion)\" -c -o $@ $< -%.o : %.cc - $(CXX) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< +%.o : %.c + $(CC) $(CPPFLAGS) $(CFLAGS) -c -o $@ $< -sh_decoder.o : decoder.cc - $(CXX) -fpic -fPIC $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< +lzlib_sh.o : lzlib.c + $(CC) -fpic -fPIC $(CPPFLAGS) $(CFLAGS) -c -o $@ $< -sh_encoder.o : encoder.cc - $(CXX) -fpic -fPIC $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< - -sh_lzlib.o : lzlib.cc - $(CXX) -fpic -fPIC $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< - -$(lib_objs) : Makefile lzlib.h lzip.h -$(sh_lib_objs) : Makefile lzlib.h lzip.h -$(objs) : Makefile -arg_parser.o : arg_parser.h -decoder.o : decoder.h -encoder.o : encoder.h -lzlib.o : decoder.h encoder.h -sh_decoder.o : decoder.h -sh_encoder.o : encoder.h -sh_lzlib.o : decoder.h encoder.h -main.o : arg_parser.h lzlib.h -bbexample.o : Makefile lzlib.h -lzcheck.o : Makefile lzlib.h +$(objs) : Makefile +carg_parser.o : carg_parser.h +lzlib.o : Makefile lzlib.h clzip.h tables.c decoder.c encoder.c +lzlib_sh.o : Makefile lzlib.h clzip.h tables.c decoder.c encoder.c +main.o : carg_parser.h lzlib.h +bbexample.o : Makefile lzlib.h +lzcheck.o : Makefile lzlib.h doc : info man @@ -147,14 +133,14 @@ dist : doc $(DISTNAME)/testsuite/test_sync.lz \ $(DISTNAME)/testsuite/test_v[01].lz \ $(DISTNAME)/*.h \ - $(DISTNAME)/*.cc + $(DISTNAME)/*.c rm -f $(DISTNAME) lzip -v -9 $(DISTNAME).tar clean : -rm -f $(progname) $(progname)_profiled $(objs) - -rm -f $(progname)_shared $(sh_lib_objs) *.so.$(pkgversion) - -rm -f bbexample bbexample.o lzcheck lzcheck.o $(lib_objs) *.a + -rm -f $(progname)_shared lzlib_sh.o *.so.$(pkgversion) + -rm -f bbexample bbexample.o lzcheck lzcheck.o lzlib.o *.a distclean : clean -rm -f Makefile config.status *.tar *.tar.lz @@ -1,10 +1,11 @@ -Changes in version 1.2: +Changes in version 1.3: -For large values of "--match-length", compression ratio has been -slightly increased and compression time has been reduced by up to 6%. +Lzlib has been translated to C from the C++ source of lzlib 1.2. This +has been done to avoid the dependency on libstdc++, making lzlib useful +in more environments. -The option "-F, --recompress", which forces recompression of files whose -name already has the ".lz" or ".tlz" suffix, has been added to minilzip. +Quote characters in messages have been changed as advised by GNU Coding +Standards. -"make install" no more tries to run "/sbin/ldconfig" on systems lacking -it. +Configure option "--datadir" has been renamed to "--datarootdir" to +follow GNU Standards. @@ -3,11 +3,11 @@ Description Lzlib is a data compression library providing in-memory LZMA compression and decompression functions, including integrity checking of the decompressed data. The compressed data format used by the library is the -lzip format. +lzip format. Lzlib is written in C. The functions and variables forming the interface of the compression library are declared in the file lzlib.h. Usage examples of the library -are given in the files main.cc and bbexample.cc from the source +are given in the files main.c and bbexample.c from the source distribution. Compression/decompression is done by repeatedly calling a couple of @@ -15,6 +15,12 @@ read/write functions until all the data has been processed by the library. This interface is safer and less error prone than the traditional zlib interface. +Compression/decompression is done when the read function is called. This +means the value returned by the position functions will not be updated +until some data is read, even if you write a lot of data. If you want +the data to be compressed in advance, just call the read function with a +size equal to 0. + Lzlib will correctly decompress a data stream which is the concatenation of two or more compressed data streams. The result is the concatenation of the corresponding decompressed data streams. Integrity testing of @@ -30,7 +36,7 @@ Igor Pavlov. For a description of the LZMA algorithm, see the Lzip manual. -Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. +Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. This file is free documentation: you have unlimited permission to copy, distribute and modify it. diff --git a/arg_parser.cc b/arg_parser.cc deleted file mode 100644 index bc4b4a3..0000000 --- a/arg_parser.cc +++ /dev/null @@ -1,203 +0,0 @@ -/* Arg_parser - POSIX/GNU command line argument parser. (C++ version) - Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Antonio Diaz Diaz. - - This library is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this library. If not, see <http://www.gnu.org/licenses/>. - - As a special exception, you may use this file as part of a free - software library without restriction. Specifically, if other files - instantiate templates or use macros or inline functions from this - file, or you compile this file and link it with other files to - produce an executable, this file does not by itself cause the - resulting executable to be covered by the GNU General Public - License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General - Public License. -*/ - -#include <cstring> -#include <string> -#include <vector> - -#include "arg_parser.h" - - -bool Arg_parser::parse_long_option( const char * const opt, const char * const arg, - const Option options[], int & argind ) - { - unsigned int len; - int index = -1; - bool exact = false, ambig = false; - - for( len = 0; opt[len+2] && opt[len+2] != '='; ++len ) ; - - // Test all long options for either exact match or abbreviated matches. - for( int i = 0; options[i].code != 0; ++i ) - if( options[i].name && !std::strncmp( options[i].name, &opt[2], len ) ) - { - if( std::strlen( options[i].name ) == len ) // Exact match found - { index = i; exact = true; break; } - else if( index < 0 ) index = i; // First nonexact match found - else if( options[index].code != options[i].code || - options[index].has_arg != options[i].has_arg ) - ambig = true; // Second or later nonexact match found - } - - if( ambig && !exact ) - { - error_ = "option `"; error_ += opt; error_ += "' is ambiguous"; - return false; - } - - if( index < 0 ) // nothing found - { - error_ = "unrecognized option `"; error_ += opt; error_ += '\''; - return false; - } - - ++argind; - data.push_back( Record( options[index].code ) ); - - if( opt[len+2] ) // `--<long_option>=<argument>' syntax - { - if( options[index].has_arg == no ) - { - error_ = "option `--"; error_ += options[index].name; - error_ += "' doesn't allow an argument"; - return false; - } - if( options[index].has_arg == yes && !opt[len+3] ) - { - error_ = "option `--"; error_ += options[index].name; - error_ += "' requires an argument"; - return false; - } - data.back().argument = &opt[len+3]; - return true; - } - - if( options[index].has_arg == yes ) - { - if( !arg || !arg[0] ) - { - error_ = "option `--"; error_ += options[index].name; - error_ += "' requires an argument"; - return false; - } - ++argind; data.back().argument = arg; - return true; - } - - return true; - } - - -bool Arg_parser::parse_short_option( const char * const opt, const char * const arg, - const Option options[], int & argind ) - { - int cind = 1; // character index in opt - - while( cind > 0 ) - { - int index = -1; - const unsigned char c = opt[cind]; - - if( c != 0 ) - for( int i = 0; options[i].code; ++i ) - if( c == options[i].code ) - { index = i; break; } - - if( index < 0 ) - { - error_ = "invalid option -- "; error_ += c; - return false; - } - - data.push_back( Record( c ) ); - if( opt[++cind] == 0 ) { ++argind; cind = 0; } // opt finished - - if( options[index].has_arg != no && cind > 0 && opt[cind] ) - { - data.back().argument = &opt[cind]; ++argind; cind = 0; - } - else if( options[index].has_arg == yes ) - { - if( !arg || !arg[0] ) - { - error_ = "option requires an argument -- "; error_ += c; - return false; - } - data.back().argument = arg; ++argind; cind = 0; - } - } - return true; - } - - -Arg_parser::Arg_parser( const int argc, const char * const argv[], - const Option options[], const bool in_order ) - { - if( argc < 2 || !argv || !options ) return; - - std::vector< std::string > non_options; // skipped non-options - int argind = 1; // index in argv - - while( argind < argc ) - { - const unsigned char ch1 = argv[argind][0]; - const unsigned char ch2 = ( ch1 ? argv[argind][1] : 0 ); - - if( ch1 == '-' && ch2 ) // we found an option - { - const char * const opt = argv[argind]; - const char * const arg = (argind + 1 < argc) ? argv[argind+1] : 0; - if( ch2 == '-' ) - { - if( !argv[argind][2] ) { ++argind; break; } // we found "--" - else if( !parse_long_option( opt, arg, options, argind ) ) break; - } - else if( !parse_short_option( opt, arg, options, argind ) ) break; - } - else - { - if( !in_order ) non_options.push_back( argv[argind++] ); - else { data.push_back( Record() ); data.back().argument = argv[argind++]; } - } - } - if( error_.size() ) data.clear(); - else - { - for( unsigned int i = 0; i < non_options.size(); ++i ) - { data.push_back( Record() ); data.back().argument.swap( non_options[i] ); } - while( argind < argc ) - { data.push_back( Record() ); data.back().argument = argv[argind++]; } - } - } - - -Arg_parser::Arg_parser( const char * const opt, const char * const arg, - const Option options[] ) - { - if( !opt || !opt[0] || !options ) return; - - if( opt[0] == '-' && opt[1] ) // we found an option - { - int argind = 1; // dummy - if( opt[1] == '-' ) - { if( opt[2] ) parse_long_option( opt, arg, options, argind ); } - else - parse_short_option( opt, arg, options, argind ); - if( error_.size() ) data.clear(); - } - else { data.push_back( Record() ); data.back().argument = opt; } - } diff --git a/arg_parser.h b/arg_parser.h deleted file mode 100644 index d1e5c02..0000000 --- a/arg_parser.h +++ /dev/null @@ -1,105 +0,0 @@ -/* Arg_parser - POSIX/GNU command line argument parser. (C++ version) - Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Antonio Diaz Diaz. - - This library is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this library. If not, see <http://www.gnu.org/licenses/>. - - As a special exception, you may use this file as part of a free - software library without restriction. Specifically, if other files - instantiate templates or use macros or inline functions from this - file, or you compile this file and link it with other files to - produce an executable, this file does not by itself cause the - resulting executable to be covered by the GNU General Public - License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General - Public License. -*/ - -/* Arg_parser reads the arguments in `argv' and creates a number of - option codes, option arguments and non-option arguments. - - In case of error, `error' returns a non-empty error message. - - `options' is an array of `struct Option' terminated by an element - containing a code which is zero. A null name means a short-only - option. A code value outside the unsigned char range means a - long-only option. - - Arg_parser normally makes it appear as if all the option arguments - were specified before all the non-option arguments for the purposes - of parsing, even if the user of your program intermixed option and - non-option arguments. If you want the arguments in the exact order - the user typed them, call `Arg_parser' with `in_order' = true. - - The argument `--' terminates all options; any following arguments are - treated as non-option arguments, even if they begin with a hyphen. - - The syntax for optional option arguments is `-<short_option><argument>' - (without whitespace), or `--<long_option>=<argument>'. -*/ - -class Arg_parser - { -public: - enum Has_arg { no, yes, maybe }; - - struct Option - { - int code; // Short option letter or code ( code != 0 ) - const char * name; // Long option name (maybe null) - Has_arg has_arg; - }; - -private: - struct Record - { - int code; - std::string argument; - Record( const int c = 0 ) : code( c ) {} - }; - - std::string error_; - std::vector< Record > data; - - bool parse_long_option( const char * const opt, const char * const arg, - const Option options[], int & argind ); - bool parse_short_option( const char * const opt, const char * const arg, - const Option options[], int & argind ); - -public: - Arg_parser( const int argc, const char * const argv[], - const Option options[], const bool in_order = false ); - - // Restricted constructor. Parses a single token and argument (if any) - Arg_parser( const char * const opt, const char * const arg, - const Option options[] ); - - const std::string & error() const throw() { return error_; } - - // The number of arguments parsed (may be different from argc) - int arguments() const throw() { return data.size(); } - - // If code( i ) is 0, argument( i ) is a non-option. - // Else argument( i ) is the option's argument (or empty). - int code( const int i ) const throw() - { - if( i >= 0 && i < arguments() ) return data[i].code; - else return 0; - } - - const std::string & argument( const int i ) const throw() - { - if( i >= 0 && i < arguments() ) return data[i].argument; - else return error_; - } - }; diff --git a/bbexample.cc b/bbexample.c index df91300..f174875 100644 --- a/bbexample.cc +++ b/bbexample.c @@ -1,5 +1,5 @@ /* Buff to buff example - A test program for the lzlib library - Copyright (C) 2010, 2011 Antonio Diaz Diaz. + Copyright (C) 2010, 2011, 2012 Antonio Diaz Diaz. This program is free software: you have unlimited permission to copy, distribute and modify it. @@ -14,10 +14,10 @@ #ifndef __cplusplus #include <stdbool.h> #endif +#include <stdint.h> #include <stdio.h> #include <stdlib.h> #include <string.h> -#include <stdint.h> #include <unistd.h> #include "lzlib.h" @@ -33,27 +33,27 @@ #endif -// Compresses `size' bytes from `data'. Returns the address of a -// malloc'd buffer containing the compressed data and its size in -// `*out_sizep'. -// In case of error, returns 0 and does not modify `*out_sizep'. - +/* Compresses 'size' bytes from 'data'. Returns the address of a + malloc'd buffer containing the compressed data and its size in + '*out_sizep'. + In case of error, returns 0 and does not modify '*out_sizep'. +*/ uint8_t * bbcompress( const uint8_t * const data, const int size, int * const out_sizep ) { - int dict_size = 8 << 20; // 8 MiB const int match_len_limit = 36; const long long member_size = LLONG_MAX; - if( dict_size > size ) dict_size = size; + int dict_size = 8 << 20; /* 8 MiB */ + if( dict_size > size ) dict_size = size; /* saves memory */ if( dict_size < LZ_min_dictionary_size() ) dict_size = LZ_min_dictionary_size(); - struct LZ_Encoder * encoder = + struct LZ_Encoder * const encoder = LZ_compress_open( dict_size, match_len_limit, member_size ); if( !encoder || LZ_compress_errno( encoder ) != LZ_ok ) { LZ_compress_close( encoder ); return 0; } - const int delta_size = (size < 256) ? 64 : size / 4; // size may be zero - int new_data_size = delta_size; // initial size + const int delta_size = (size < 256) ? 64 : size / 4; /* size may be zero */ + int new_data_size = delta_size; /* initial size */ uint8_t * new_data = (uint8_t *)malloc( new_data_size ); if( !new_data ) { LZ_compress_close( encoder ); return 0; } @@ -81,9 +81,10 @@ uint8_t * bbcompress( const uint8_t * const data, const int size, if( LZ_compress_finished( encoder ) == 1 ) break; if( new_pos >= new_data_size ) { - void * const tmp = realloc( new_data, new_data_size + delta_size ); + uint8_t * const tmp = + (uint8_t *)realloc( new_data, new_data_size + delta_size ); if( !tmp ) { error = true; break; } - new_data = (uint8_t *)tmp; + new_data = tmp; new_data_size += delta_size; } } @@ -95,20 +96,20 @@ uint8_t * bbcompress( const uint8_t * const data, const int size, } -// Decompresses `size' bytes from `data'. Returns the address of a -// malloc'd buffer containing the decompressed data and its size in -// `*out_sizep'. -// In case of error, returns 0 and does not modify `*out_sizep'. - +/* Decompresses 'size' bytes from 'data'. Returns the address of a + malloc'd buffer containing the decompressed data and its size in + '*out_sizep'. + In case of error, returns 0 and does not modify '*out_sizep'. +*/ uint8_t * bbdecompress( const uint8_t * const data, const int size, int * const out_sizep ) { - struct LZ_Decoder * decoder = LZ_decompress_open(); + struct LZ_Decoder * const decoder = LZ_decompress_open(); if( !decoder || LZ_decompress_errno( decoder ) != LZ_ok ) { LZ_decompress_close( decoder ); return 0; } - const int delta_size = size; - int new_data_size = delta_size; // initial size + const int delta_size = size; /* size must be > zero */ + int new_data_size = delta_size; /* initial size */ uint8_t * new_data = (uint8_t *)malloc( new_data_size ); if( !new_data ) { LZ_decompress_close( decoder ); return 0; } @@ -136,9 +137,10 @@ uint8_t * bbdecompress( const uint8_t * const data, const int size, if( LZ_decompress_finished( decoder ) == 1 ) break; if( new_pos >= new_data_size ) { - void * const tmp = realloc( new_data, new_data_size + delta_size ); + uint8_t * const tmp = + (uint8_t *)realloc( new_data, new_data_size + delta_size ); if( !tmp ) { error = true; break; } - new_data = (uint8_t *)tmp; + new_data = tmp; new_data_size += delta_size; } } @@ -161,7 +163,7 @@ int main( const int argc, const char * const argv[] ) FILE *file = fopen( argv[1], "rb" ); if( !file ) { - fprintf( stderr, "bbexample: Can't open file `%s' for reading\n", argv[1] ); + fprintf( stderr, "bbexample: Can't open file '%s' for reading\n", argv[1] ); return 1; } @@ -172,10 +174,11 @@ int main( const int argc, const char * const argv[] ) fprintf( stderr, "bbexample: Not enough memory.\n" ); return 1; } + const int in_size = fread( in_buffer, 1, in_buffer_size, file ); if( in_size >= in_buffer_size ) { - fprintf( stderr, "bbexample: Input file `%s' is too big.\n", argv[1] ); + fprintf( stderr, "bbexample: Input file '%s' is too big.\n", argv[1] ); return 1; } fclose( file ); @@ -197,7 +200,7 @@ int main( const int argc, const char * const argv[] ) } if( in_size != out_size || - ( out_size > 0 && memcmp( in_buffer, out_buffer, out_size ) ) ) + ( in_size > 0 && memcmp( in_buffer, out_buffer, in_size ) != 0 ) ) { fprintf( stderr, "bbexample: Decompressed data differs from original.\n" ); return 1; diff --git a/carg_parser.c b/carg_parser.c new file mode 100644 index 0000000..326bd41 --- /dev/null +++ b/carg_parser.c @@ -0,0 +1,294 @@ +/* Arg_parser - POSIX/GNU command line argument parser. (C version) + Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 + Antonio Diaz Diaz. + + This library is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this library. If not, see <http://www.gnu.org/licenses/>. + + As a special exception, you may use this file as part of a free + software library without restriction. Specifically, if other files + instantiate templates or use macros or inline functions from this + file, or you compile this file and link it with other files to + produce an executable, this file does not by itself cause the + resulting executable to be covered by the GNU General Public + License. This exception does not however invalidate any other + reasons why the executable file might be covered by the GNU General + Public License. +*/ + +#include <stdlib.h> +#include <string.h> + +#include "carg_parser.h" + + +/* assure at least a minimum size for buffer 'buf' */ +static void * ap_resize_buffer( void * buf, const int min_size ) + { + if( buf ) buf = realloc( buf, min_size ); + else buf = malloc( min_size ); + return buf; + } + + +static char push_back_record( struct Arg_parser * const ap, + const int code, const char * const argument ) + { + const int len = strlen( argument ); + struct ap_Record *p; + void * tmp = ap_resize_buffer( ap->data, + ( ap->data_size + 1 ) * sizeof (struct ap_Record) ); + if( !tmp ) return 0; + ap->data = (struct ap_Record *)tmp; + p = &(ap->data[ap->data_size]); + p->code = code; + p->argument = 0; + tmp = ap_resize_buffer( p->argument, len + 1 ); + if( !tmp ) return 0; + p->argument = (char *)tmp; + strncpy( p->argument, argument, len + 1 ); + ++ap->data_size; + return 1; + } + + +static char add_error( struct Arg_parser * const ap, const char * const msg ) + { + const int len = strlen( msg ); + void * tmp = ap_resize_buffer( ap->error, ap->error_size + len + 1 ); + if( !tmp ) return 0; + ap->error = (char *)tmp; + strncpy( ap->error + ap->error_size, msg, len + 1 ); + ap->error_size += len; + return 1; + } + + +static void free_data( struct Arg_parser * const ap ) + { + int i; + for( i = 0; i < ap->data_size; ++i ) free( ap->data[i].argument ); + if( ap->data ) { free( ap->data ); ap->data = 0; } + ap->data_size = 0; + } + + +static char parse_long_option( struct Arg_parser * const ap, + const char * const opt, const char * const arg, + const struct ap_Option options[], + int * const argindp ) + { + unsigned int len; + int index = -1; + int i; + char exact = 0, ambig = 0; + + for( len = 0; opt[len+2] && opt[len+2] != '='; ++len ) ; + + /* Test all long options for either exact match or abbreviated matches. */ + for( i = 0; options[i].code != 0; ++i ) + if( options[i].name && !strncmp( options[i].name, &opt[2], len ) ) + { + if( strlen( options[i].name ) == len ) /* Exact match found */ + { index = i; exact = 1; break; } + else if( index < 0 ) index = i; /* First nonexact match found */ + else if( options[index].code != options[i].code || + options[index].has_arg != options[i].has_arg ) + ambig = 1; /* Second or later nonexact match found */ + } + + if( ambig && !exact ) + { + add_error( ap, "option '" ); add_error( ap, opt ); + add_error( ap, "' is ambiguous" ); + return 1; + } + + if( index < 0 ) /* nothing found */ + { + add_error( ap, "unrecognized option '" ); add_error( ap, opt ); + add_error( ap, "'" ); + return 1; + } + + ++*argindp; + + if( opt[len+2] ) /* '--<long_option>=<argument>' syntax */ + { + if( options[index].has_arg == ap_no ) + { + add_error( ap, "option '--" ); add_error( ap, options[index].name ); + add_error( ap, "' doesn't allow an argument" ); + return 1; + } + if( options[index].has_arg == ap_yes && !opt[len+3] ) + { + add_error( ap, "option '--" ); add_error( ap, options[index].name ); + add_error( ap, "' requires an argument" ); + return 1; + } + return push_back_record( ap, options[index].code, &opt[len+3] ); + } + + if( options[index].has_arg == ap_yes ) + { + if( !arg || !arg[0] ) + { + add_error( ap, "option '--" ); add_error( ap, options[index].name ); + add_error( ap, "' requires an argument" ); + return 1; + } + ++*argindp; + return push_back_record( ap, options[index].code, arg ); + } + + return push_back_record( ap, options[index].code, "" ); + } + + +static char parse_short_option( struct Arg_parser * const ap, + const char * const opt, const char * const arg, + const struct ap_Option options[], + int * const argindp ) + { + int cind = 1; /* character index in opt */ + + while( cind > 0 ) + { + int index = -1; + int i; + const unsigned char code = opt[cind]; + char code_str[2]; + code_str[0] = code; code_str[1] = 0; + + if( code != 0 ) + for( i = 0; options[i].code; ++i ) + if( code == options[i].code ) + { index = i; break; } + + if( index < 0 ) + { + add_error( ap, "invalid option -- " ); add_error( ap, code_str ); + return 1; + } + + if( opt[++cind] == 0 ) { ++*argindp; cind = 0; } /* opt finished */ + + if( options[index].has_arg != ap_no && cind > 0 && opt[cind] ) + { + if( !push_back_record( ap, code, &opt[cind] ) ) return 0; + ++*argindp; cind = 0; + } + else if( options[index].has_arg == ap_yes ) + { + if( !arg || !arg[0] ) + { + add_error( ap, "option requires an argument -- " ); + add_error( ap, code_str ); + return 1; + } + ++*argindp; cind = 0; + if( !push_back_record( ap, code, arg ) ) return 0; + } + else if( !push_back_record( ap, code, "" ) ) return 0; + } + return 1; + } + + +char ap_init( struct Arg_parser * const ap, + const int argc, const char * const argv[], + const struct ap_Option options[], const char in_order ) + { + const char ** non_options = 0; /* skipped non-options */ + int non_options_size = 0; /* number of skipped non-options */ + int argind = 1; /* index in argv */ + int i; + + ap->data = 0; + ap->error = 0; + ap->data_size = 0; + ap->error_size = 0; + if( argc < 2 || !argv || !options ) return 1; + + while( argind < argc ) + { + const unsigned char ch1 = argv[argind][0]; + const unsigned char ch2 = ( ch1 ? argv[argind][1] : 0 ); + + if( ch1 == '-' && ch2 ) /* we found an option */ + { + const char * const opt = argv[argind]; + const char * const arg = (argind + 1 < argc) ? argv[argind+1] : 0; + if( ch2 == '-' ) + { + if( !argv[argind][2] ) { ++argind; break; } /* we found "--" */ + else if( !parse_long_option( ap, opt, arg, options, &argind ) ) return 0; + } + else if( !parse_short_option( ap, opt, arg, options, &argind ) ) return 0; + if( ap->error ) break; + } + else + { + if( !in_order ) + { + void * tmp = ap_resize_buffer( non_options, + ( non_options_size + 1 ) * sizeof *non_options ); + if( !tmp ) return 0; + non_options = (const char **)tmp; + non_options[non_options_size++] = argv[argind++]; + } + else if( !push_back_record( ap, 0, argv[argind++] ) ) return 0; + } + } + if( ap->error ) free_data( ap ); + else + { + for( i = 0; i < non_options_size; ++i ) + if( !push_back_record( ap, 0, non_options[i] ) ) return 0; + while( argind < argc ) + if( !push_back_record( ap, 0, argv[argind++] ) ) return 0; + } + if( non_options ) free( non_options ); + return 1; + } + + +void ap_free( struct Arg_parser * const ap ) + { + free_data( ap ); + if( ap->error ) { free( ap->error ); ap->error = 0; } + ap->error_size = 0; + } + + +const char * ap_error( const struct Arg_parser * const ap ) + { return ap->error; } + + +int ap_arguments( const struct Arg_parser * const ap ) + { return ap->data_size; } + + +int ap_code( const struct Arg_parser * const ap, const int i ) + { + if( i >= 0 && i < ap_arguments( ap ) ) return ap->data[i].code; + else return 0; + } + + +const char * ap_argument( const struct Arg_parser * const ap, const int i ) + { + if( i >= 0 && i < ap_arguments( ap ) ) return ap->data[i].argument; + else return ""; + } diff --git a/carg_parser.h b/carg_parser.h new file mode 100644 index 0000000..3575dd7 --- /dev/null +++ b/carg_parser.h @@ -0,0 +1,102 @@ +/* Arg_parser - POSIX/GNU command line argument parser. (C version) + Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 + Antonio Diaz Diaz. + + This library is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this library. If not, see <http://www.gnu.org/licenses/>. + + As a special exception, you may use this file as part of a free + software library without restriction. Specifically, if other files + instantiate templates or use macros or inline functions from this + file, or you compile this file and link it with other files to + produce an executable, this file does not by itself cause the + resulting executable to be covered by the GNU General Public + License. This exception does not however invalidate any other + reasons why the executable file might be covered by the GNU General + Public License. +*/ + +/* Arg_parser reads the arguments in 'argv' and creates a number of + option codes, option arguments and non-option arguments. + + In case of error, 'ap_error' returns a non-null pointer to an error + message. + + 'options' is an array of 'struct ap_Option' terminated by an element + containing a code which is zero. A null name means a short-only + option. A code value outside the unsigned char range means a + long-only option. + + Arg_parser normally makes it appear as if all the option arguments + were specified before all the non-option arguments for the purposes + of parsing, even if the user of your program intermixed option and + non-option arguments. If you want the arguments in the exact order + the user typed them, call 'ap_init' with 'in_order' = true. + + The argument '--' terminates all options; any following arguments are + treated as non-option arguments, even if they begin with a hyphen. + + The syntax for optional option arguments is '-<short_option><argument>' + (without whitespace), or '--<long_option>=<argument>'. +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +enum ap_Has_arg { ap_no, ap_yes, ap_maybe }; + +struct ap_Option + { + int code; /* Short option letter or code ( code != 0 ) */ + const char * name; /* Long option name (maybe null) */ + enum ap_Has_arg has_arg; + }; + + +struct ap_Record + { + int code; + char * argument; + }; + + +struct Arg_parser + { + struct ap_Record * data; + char * error; + int data_size; + int error_size; + }; + + +char ap_init( struct Arg_parser * const ap, + const int argc, const char * const argv[], + const struct ap_Option options[], const char in_order ); + +void ap_free( struct Arg_parser * const ap ); + +const char * ap_error( const struct Arg_parser * const ap ); + + /* The number of arguments parsed (may be different from argc) */ +int ap_arguments( const struct Arg_parser * const ap ); + + /* If ap_code( i ) is 0, ap_argument( i ) is a non-option. + Else ap_argument( i ) is the option's argument (or empty). */ +int ap_code( const struct Arg_parser * const ap, const int i ); + +const char * ap_argument( const struct Arg_parser * const ap, const int i ); + +#ifdef __cplusplus +} +#endif @@ -0,0 +1,232 @@ +/* Lzlib - A compression library for lzip files + Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. + + This library is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this library. If not, see <http://www.gnu.org/licenses/>. + + As a special exception, you may use this file as part of a free + software library without restriction. Specifically, if other files + instantiate templates or use macros or inline functions from this + file, or you compile this file and link it with other files to + produce an executable, this file does not by itself cause the + resulting executable to be covered by the GNU General Public + License. This exception does not however invalidate any other + reasons why the executable file might be covered by the GNU General + Public License. +*/ + +#ifndef max + #define max(x,y) ((x) >= (y) ? (x) : (y)) +#endif +#ifndef min + #define min(x,y) ((x) <= (y) ? (x) : (y)) +#endif + +typedef uint8_t State; + +enum { states = 12 }; + +static inline bool St_is_char( const State st ) { return st < 7; } + +static inline void St_set_char( State * const st ) + { + static const uint8_t next[states] = + { 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5 }; + *st = next[*st]; + } + +static inline void St_set_match( State * const st ) + { + static const uint8_t next[states] = + { 7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10 }; + *st = next[*st]; + } + +static inline void St_set_rep( State * const st ) + { + static const uint8_t next[states] = + { 8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11 }; + *st = next[*st]; + } + +static inline void St_set_short_rep( State * const st ) + { + static const uint8_t next[states] = + { 9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11 }; + *st = next[*st]; + } + + +enum { + min_dictionary_bits = 12, + min_dictionary_size = 1 << min_dictionary_bits, + max_dictionary_bits = 29, + max_dictionary_size = 1 << max_dictionary_bits, + literal_context_bits = 3, + pos_state_bits = 2, + pos_states = 1 << pos_state_bits, + pos_state_mask = pos_states - 1, + + dis_slot_bits = 6, + start_dis_model = 4, + end_dis_model = 14, + modeled_distances = 1 << (end_dis_model / 2), + dis_align_bits = 4, + dis_align_size = 1 << dis_align_bits, + + len_low_bits = 3, + len_mid_bits = 3, + len_high_bits = 8, + len_low_symbols = 1 << len_low_bits, + len_mid_symbols = 1 << len_mid_bits, + len_high_symbols = 1 << len_high_bits, + max_len_symbols = len_low_symbols + len_mid_symbols + len_high_symbols, + + min_match_len = 2, /* must be 2 */ + max_match_len = min_match_len + max_len_symbols - 1, /* 273 */ + min_match_len_limit = 5, + + max_dis_states = 4 }; + +static inline int get_dis_state( int len ) + { + len -= min_match_len; + if( len >= max_dis_states ) len = max_dis_states - 1; + return len; + } + + +enum { bit_model_move_bits = 5, + bit_model_total_bits = 11, + bit_model_total = 1 << bit_model_total_bits }; + +typedef unsigned int Bit_model; + +static inline void Bm_init( Bit_model * const probability ) + { *probability = bit_model_total / 2; } + + +static inline int real_bits( const unsigned int value ) + { + int bits = 0, i = 1; + unsigned int mask = 1; + for( ; mask > 0; ++i, mask <<= 1 ) if( value & mask ) bits = i; + return bits; + } + + +static const uint8_t magic_string[4] = { 'L', 'Z', 'I', 'P' }; + +typedef uint8_t File_header[6]; /* 0-3 magic bytes */ + /* 4 version */ + /* 5 coded_dict_size */ +enum { Fh_size = 6 }; + +static inline void Fh_set_magic( File_header data ) + { memcpy( data, magic_string, 4 ); data[4] = 1; } + +static inline bool Fh_verify_magic( const File_header data ) + { return ( memcmp( data, magic_string, 4 ) == 0 ); } + +static inline uint8_t Fh_version( const File_header data ) + { return data[4]; } + +static inline bool Fh_verify_version( const File_header data ) + { return ( data[4] <= 1 ); } + +static inline int Fh_get_dictionary_size( const File_header data ) + { + int sz = ( 1 << ( data[5] & 0x1F ) ); + if( sz > min_dictionary_size && sz <= max_dictionary_size ) + sz -= ( sz / 16 ) * ( ( data[5] >> 5 ) & 0x07 ); + return sz; + } + +static inline bool Fh_set_dictionary_size( File_header data, const int sz ) + { + if( sz >= min_dictionary_size && sz <= max_dictionary_size ) + { + data[5] = real_bits( sz - 1 ); + if( sz > min_dictionary_size ) + { + const int base_size = 1 << data[5]; + const int wedge = base_size / 16; + int i; + for( i = 7; i >= 1; --i ) + if( base_size - ( i * wedge ) >= sz ) + { data[5] |= ( i << 5 ); break; } + } + return true; + } + return false; + } + +static inline bool Fh_verify( const File_header data ) + { + return ( Fh_verify_magic( data ) && Fh_verify_version( data ) && + Fh_get_dictionary_size( data ) >= min_dictionary_size && + Fh_get_dictionary_size( data ) <= max_dictionary_size ); + } + + +typedef uint8_t File_trailer[20]; + /* 0-3 CRC32 of the uncompressed data */ + /* 4-11 size of the uncompressed data */ + /* 12-19 member size including header and trailer */ + +enum { Ft_size = 20 }; + +static inline int Ft_versioned_size( const int version ) + { return ( ( version >= 1 ) ? 20 : 12 ); } + +static inline uint32_t Ft_get_data_crc( const File_trailer data ) + { + uint32_t tmp = 0; + int i; + for( i = 3; i >= 0; --i ) { tmp <<= 8; tmp += data[i]; } + return tmp; + } + +static inline void Ft_set_data_crc( File_trailer data, uint32_t crc ) + { + int i; + for( i = 0; i <= 3; ++i ) { data[i] = (uint8_t)crc; crc >>= 8; } + } + +static inline long long Ft_get_data_size( const File_trailer data ) + { + long long tmp = 0; + int i; + for( i = 11; i >= 4; --i ) { tmp <<= 8; tmp += data[i]; } + return tmp; + } + +static inline void Ft_set_data_size( File_trailer data, long long sz ) + { + int i; + for( i = 4; i <= 11; ++i ) { data[i] = (uint8_t)sz; sz >>= 8; } + } + +static inline long long Ft_get_member_size( const File_trailer data ) + { + long long tmp = 0; + int i; + for( i = 19; i >= 12; --i ) { tmp <<= 8; tmp += data[i]; } + return tmp; + } + +static inline void Ft_set_member_size( File_trailer data, long long sz ) + { + int i; + for( i = 12; i <= 19; ++i ) { data[i] = (uint8_t)sz; sz >>= 8; } + } @@ -1,6 +1,6 @@ #! /bin/sh # configure script for Lzlib - A compression library for lzip files -# Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. +# Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. # # This configure script is free software: you have unlimited permission # to copy, distribute and modify it. @@ -8,7 +8,7 @@ args= no_create= pkgname=lzlib -pkgversion=1.2 +pkgversion=1.3 soversion=1 progname=minilzip progname_shared= @@ -22,15 +22,14 @@ srcdir= prefix=/usr/local exec_prefix='$(prefix)' bindir='$(exec_prefix)/bin' -datadir='$(prefix)/share' -includedir='${prefix}/include' -infodir='$(datadir)/info' -libdir='${exec_prefix}/lib' -mandir='$(datadir)/man' -sysconfdir='$(prefix)/etc' -CXX= +datarootdir='$(prefix)/share' +includedir='$(prefix)/include' +infodir='$(datarootdir)/info' +libdir='$(exec_prefix)/lib' +mandir='$(datarootdir)/man' +CC= CPPFLAGS= -CXXFLAGS='-Wall -W -O2' +CFLAGS='-Wall -W -O2' LDFLAGS= # Loop over all args @@ -45,12 +44,12 @@ while [ -n "$1" ] ; do # Split out the argument for options that take them case ${option} in - *=*) optarg=`echo ${option} | sed -e 's,^[^=]*=,,'` ;; + *=*) optarg=`echo ${option} | sed -e 's,^[^=]*=,,;s,/$,,'` ;; esac # Process the options case ${option} in - --help | --he* | -h) + --help | -h) echo "Usage: configure [options]" echo echo "Options: [defaults in brackets]" @@ -60,51 +59,37 @@ while [ -n "$1" ] ; do echo " --prefix=DIR install into DIR [${prefix}]" echo " --exec-prefix=DIR base directory for arch-dependent files [${exec_prefix}]" echo " --bindir=DIR user executables directory [${bindir}]" - echo " --datadir=DIR base directory for doc and data [${datadir}]" + echo " --datarootdir=DIR base directory for doc and data [${datarootdir}]" echo " --includedir=DIR C header files [${includedir}]" echo " --infodir=DIR info files directory [${infodir}]" echo " --libdir=DIR object code libraries [${libdir}]" echo " --mandir=DIR man pages directory [${mandir}]" - echo " --sysconfdir=DIR read-only single-machine data directory [${sysconfdir}]" echo " --enable-shared build also a shared library [disable]" - echo " CXX=COMPILER C++ compiler to use [g++]" + echo " CC=COMPILER C compiler to use [gcc]" echo " CPPFLAGS=OPTIONS command line options for the preprocessor [${CPPFLAGS}]" - echo " CXXFLAGS=OPTIONS command line options for the C++ compiler [${CXXFLAGS}]" + echo " CFLAGS=OPTIONS command line options for the C compiler [${CFLAGS}]" echo " LDFLAGS=OPTIONS command line options for the linker [${LDFLAGS}]" echo exit 0 ;; - --version | --ve* | -V) + --version | -V) echo "Configure script for ${pkgname} version ${pkgversion}" exit 0 ;; - --srcdir* | --sr*) - srcdir=`echo ${optarg} | sed -e 's,/$,,'` ;; - --prefix* | --pr*) - prefix=`echo ${optarg} | sed -e 's,/$,,'` ;; - --exec-prefix* | --ex*) - exec_prefix=`echo ${optarg} | sed -e 's,/$,,'` ;; - --bindir* | --bi*) - bindir=`echo ${optarg} | sed -e 's,/$,,'` ;; - --datadir* | --da*) - datadir=`echo ${optarg} | sed -e 's,/$,,'` ;; - --includedir* | --inc*) - includedir=`echo ${optarg} | sed -e 's,/$,,'` ;; - --infodir* | --inf*) - infodir=`echo ${optarg} | sed -e 's,/$,,'` ;; - --libdir* | --li*) - libdir=`echo ${optarg} | sed -e 's,/$,,'` ;; - --mandir* | --ma*) - mandir=`echo ${optarg} | sed -e 's,/$,,'` ;; - --sysconfdir* | --sy*) - sysconfdir=`echo ${optarg} | sed -e 's,/$,,'` ;; - --no-create | --no-c*) - no_create=yes ;; - --enable-shared | --enable-s*) - progname_shared=${progname}_shared ;; - - CXX=*) CXX=${optarg} ;; + --srcdir=*) srcdir=${optarg} ;; + --prefix=*) prefix=${optarg} ;; + --exec-prefix=*) exec_prefix=${optarg} ;; + --bindir=*) bindir=${optarg} ;; + --datarootdir=*) datarootdir=${optarg} ;; + --includedir=*) includedir=${optarg} ;; + --infodir=*) infodir=${optarg} ;; + --libdir=*) libdir=${optarg} ;; + --mandir=*) mandir=${optarg} ;; + --no-create) no_create=yes ;; + --enable-shared) progname_shared=${progname}_shared ;; + + CC=*) CC=${optarg} ;; CPPFLAGS=*) CPPFLAGS=${optarg} ;; - CXXFLAGS=*) CXXFLAGS=${optarg} ;; - LDFLAGS=*) LDFLAGS=${optarg} ;; + CFLAGS=*) CFLAGS=${optarg} ;; + LDFLAGS=*) LDFLAGS=${optarg} ;; --* | *=* | *-*-*) ;; *) @@ -135,14 +120,14 @@ fi # Set srcdir to . if that's what it is. if [ "`pwd`" = "`cd ${srcdir} ; pwd`" ] ; then srcdir=. ; fi -# checking whether we are using GNU C++. -if [ -z "${CXX}" ] ; then # Let the user override the test. - if [ -x /bin/g++ ] || - [ -x /usr/bin/g++ ] || - [ -x /usr/local/bin/g++ ] ; then - CXX="g++" +# checking whether we are using GNU C. +if [ -z "${CC}" ] ; then # Let the user override the test. + if [ -x /bin/gcc ] || + [ -x /usr/bin/gcc ] || + [ -x /usr/local/bin/gcc ] ; then + CC="gcc" else - CXX="c++" + CC="cc" fi fi @@ -168,20 +153,19 @@ echo "VPATH = ${srcdir}" echo "prefix = ${prefix}" echo "exec_prefix = ${exec_prefix}" echo "bindir = ${bindir}" -echo "datadir = ${datadir}" +echo "datarootdir = ${datarootdir}" echo "includedir = ${includedir}" echo "infodir = ${infodir}" echo "libdir = ${libdir}" echo "mandir = ${mandir}" -echo "sysconfdir = ${sysconfdir}" -echo "CXX = ${CXX}" +echo "CC = ${CC}" echo "CPPFLAGS = ${CPPFLAGS}" -echo "CXXFLAGS = ${CXXFLAGS}" +echo "CFLAGS = ${CFLAGS}" echo "LDFLAGS = ${LDFLAGS}" rm -f Makefile cat > Makefile << EOF # Makefile for Lzlib - A compression library for lzip files -# Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. +# Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. # This file was generated automatically by configure. Do not edit. # # This Makefile is free software: you have unlimited permission @@ -197,15 +181,14 @@ VPATH = ${srcdir} prefix = ${prefix} exec_prefix = ${exec_prefix} bindir = ${bindir} -datadir = ${datadir} +datarootdir = ${datarootdir} includedir = ${includedir} infodir = ${infodir} libdir = ${libdir} mandir = ${mandir} -sysconfdir = ${sysconfdir} -CXX = ${CXX} +CC = ${CC} CPPFLAGS = ${CPPFLAGS} -CXXFLAGS = ${CXXFLAGS} +CFLAGS = ${CFLAGS} LDFLAGS = ${LDFLAGS} EOF cat ${srcdir}/Makefile.in >> Makefile diff --git a/decoder.c b/decoder.c new file mode 100644 index 0000000..d3ea0d6 --- /dev/null +++ b/decoder.c @@ -0,0 +1,734 @@ +/* Lzlib - A compression library for lzip files + Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. + + This library is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this library. If not, see <http://www.gnu.org/licenses/>. + + As a special exception, you may use this file as part of a free + software library without restriction. Specifically, if other files + instantiate templates or use macros or inline functions from this + file, or you compile this file and link it with other files to + produce an executable, this file does not by itself cause the + resulting executable to be covered by the GNU General Public + License. This exception does not however invalidate any other + reasons why the executable file might be covered by the GNU General + Public License. +*/ + +struct Circular_buffer + { + uint8_t * buffer; + int buffer_size; /* capacity == buffer_size - 1 */ + int get; /* buffer is empty when get == put */ + int put; + }; + +static inline void Cb_reset( struct Circular_buffer * const cb ) + { cb->get = 0; cb->put = 0; } + +static inline bool Cb_init( struct Circular_buffer * const cb, + const int buf_size ) + { + cb->buffer = (uint8_t *)malloc( buf_size + 1 ); + cb->buffer_size = buf_size + 1; + cb->get = 0; + cb->put = 0; + return ( cb->buffer != 0 ); + } + +static inline void Cb_free( struct Circular_buffer * const cb ) + { free( cb->buffer ); cb->buffer = 0; } + +static inline int Cb_used_bytes( const struct Circular_buffer * const cb ) + { return ( (cb->get <= cb->put) ? 0 : cb->buffer_size ) + cb->put - cb->get; } + +static inline int Cb_free_bytes( const struct Circular_buffer * const cb ) + { return ( (cb->get <= cb->put) ? cb->buffer_size : 0 ) - cb->put + cb->get - 1; } + +static inline uint8_t Cb_get_byte( struct Circular_buffer * const cb ) + { + const uint8_t b = cb->buffer[cb->get]; + if( ++cb->get >= cb->buffer_size ) cb->get = 0; + return b; + } + +static inline void Cb_put_byte( struct Circular_buffer * const cb, + const uint8_t b ) + { + cb->buffer[cb->put] = b; + if( ++cb->put >= cb->buffer_size ) cb->put = 0; + } + + +/* Copies up to 'out_size' bytes to 'out_buffer' and updates 'get'. + Returns the number of bytes copied. +*/ +static int Cb_read_data( struct Circular_buffer * const cb, + uint8_t * const out_buffer, const int out_size ) + { + if( out_size < 0 ) return 0; + int size = 0; + if( cb->get > cb->put ) + { + size = min( cb->buffer_size - cb->get, out_size ); + if( size > 0 ) + { + memcpy( out_buffer, cb->buffer + cb->get, size ); + cb->get += size; + if( cb->get >= cb->buffer_size ) cb->get = 0; + } + } + if( cb->get < cb->put ) + { + const int size2 = min( cb->put - cb->get, out_size - size ); + if( size2 > 0 ) + { + memcpy( out_buffer + size, cb->buffer + cb->get, size2 ); + cb->get += size2; + size += size2; + } + } + return size; + } + + +/* Copies up to 'in_size' bytes from 'in_buffer' and updates 'put'. + Returns the number of bytes copied. +*/ +static int Cb_write_data( struct Circular_buffer * const cb, + const uint8_t * const in_buffer, const int in_size ) + { + if( in_size < 0 ) return 0; + int size = 0; + if( cb->put >= cb->get ) + { + size = min( cb->buffer_size - cb->put - (cb->get == 0), in_size ); + if( size > 0 ) + { + memcpy( cb->buffer + cb->put, in_buffer, size ); + cb->put += size; + if( cb->put >= cb->buffer_size ) cb->put = 0; + } + } + if( cb->put < cb->get ) + { + const int size2 = min( cb->get - cb->put - 1, in_size - size ); + if( size2 > 0 ) + { + memcpy( cb->buffer + cb->put, in_buffer + size, size2 ); + cb->put += size2; + size += size2; + } + } + return size; + } + + +enum { rd_min_available_bytes = 8 }; + +struct Range_decoder + { + struct Circular_buffer cb; + long long member_position; + uint32_t code; + uint32_t range; + bool reload_pending; + bool at_stream_end; + }; + +static inline bool Rd_init( struct Range_decoder * const rdec ) + { + if( !Cb_init( &rdec->cb, 65536 + rd_min_available_bytes ) ) return false; + rdec->member_position = 0; + rdec->code = 0; + rdec->range = 0xFFFFFFFFU; + rdec->reload_pending = false; + rdec->at_stream_end = false; + return true; + } + +static inline void Rd_free( struct Range_decoder * const rdec ) + { Cb_free( &rdec->cb ); } + +static inline int Rd_available_bytes( const struct Range_decoder * const rdec ) + { return Cb_used_bytes( &rdec->cb ); } + +static inline void Rd_finish( struct Range_decoder * const rdec ) + { rdec->at_stream_end = true; } + +static inline bool Rd_finished( const struct Range_decoder * const rdec ) + { return rdec->at_stream_end && !Cb_used_bytes( &rdec->cb ); } + +static inline int Rd_free_bytes( const struct Range_decoder * const rdec ) + { if( rdec->at_stream_end ) return 0; return Cb_free_bytes( &rdec->cb ); } + +static inline void Rd_purge( struct Range_decoder * const rdec ) + { rdec->at_stream_end = true; Cb_reset( &rdec->cb ); } + +static inline void Rd_reset( struct Range_decoder * const rdec ) + { rdec->at_stream_end = false; Cb_reset( &rdec->cb ); } + + +/* Seeks a member header and updates 'get'. + Returns true if it finds a valid header. +*/ +static bool Rd_find_header( struct Range_decoder * const rdec ) + { + while( rdec->cb.get != rdec->cb.put ) + { + if( rdec->cb.buffer[rdec->cb.get] == magic_string[0] ) + { + int g = rdec->cb.get; + int i; + File_header header; + for( i = 0; i < Fh_size; ++i ) + { + if( g == rdec->cb.put ) return false; /* not enough data */ + header[i] = rdec->cb.buffer[g]; + if( ++g >= rdec->cb.buffer_size ) g = 0; + } + if( Fh_verify( header ) ) return true; + } + if( ++rdec->cb.get >= rdec->cb.buffer_size ) rdec->cb.get = 0; + } + return false; + } + + +/* Returns true, fills 'header', and updates 'get' if 'get' points to a + valid header. + Else returns false and leaves 'get' unmodified. +*/ +static bool Rd_read_header( struct Range_decoder * const rdec, + File_header header ) + { + int g = rdec->cb.get; + int i; + for( i = 0; i < Fh_size; ++i ) + { + if( g == rdec->cb.put ) return false; /* not enough data */ + header[i] = rdec->cb.buffer[g]; + if( ++g >= rdec->cb.buffer_size ) g = 0; + } + if( Fh_verify( header ) ) + { + rdec->cb.get = g; + rdec->member_position = Fh_size; + rdec->reload_pending = true; + return true; + } + return false; + } + + +static inline int Rd_write_data( struct Range_decoder * const rdec, + const uint8_t * const inbuf, const int size ) + { + if( rdec->at_stream_end || size <= 0 ) return 0; + return Cb_write_data( &rdec->cb, inbuf, size ); + } + +static inline uint8_t Rd_get_byte( struct Range_decoder * const rdec ) + { + ++rdec->member_position; + return Cb_get_byte( &rdec->cb ); + } + + +static bool Rd_try_reload( struct Range_decoder * const rdec, const bool force ) + { + if( force ) rdec->reload_pending = true; + if( rdec->reload_pending && Rd_available_bytes( rdec ) >= 5 ) + { + int i; + rdec->reload_pending = false; + rdec->code = 0; + rdec->range = 0xFFFFFFFFU; + for( i = 0; i < 5; ++i ) + rdec->code = (rdec->code << 8) | Rd_get_byte( rdec ); + } + return !rdec->reload_pending; + } + + +static inline int Rd_decode( struct Range_decoder * const rdec, + const int num_bits ) + { + int symbol = 0; + int i; + for( i = num_bits; i > 0; --i ) + { + symbol <<= 1; + if( rdec->range <= 0x00FFFFFFU ) + { + rdec->range <<= 7; + rdec->code = (rdec->code << 8) | Rd_get_byte( rdec ); + if( rdec->code >= rdec->range ) + { rdec->code -= rdec->range; symbol |= 1; } + } + else + { + rdec->range >>= 1; + if( rdec->code >= rdec->range ) + { rdec->code -= rdec->range; symbol |= 1; } + } + } + return symbol; + } + + +static inline void Rd_normalize( struct Range_decoder * const rdec ) + { + if( rdec->range <= 0x00FFFFFFU ) + { + rdec->range <<= 8; + rdec->code = (rdec->code << 8) | Rd_get_byte( rdec ); + } + } + + +static inline int Rd_decode_bit( struct Range_decoder * const rdec, + Bit_model * const probability ) + { + uint32_t bound; + Rd_normalize( rdec ); + bound = ( rdec->range >> bit_model_total_bits ) * *probability; + if( rdec->code < bound ) + { + rdec->range = bound; + *probability += (bit_model_total - *probability) >> bit_model_move_bits; + return 0; + } + else + { + rdec->range -= bound; + rdec->code -= bound; + *probability -= *probability >> bit_model_move_bits; + return 1; + } + } + + +static inline int Rd_decode_tree( struct Range_decoder * const rdec, + Bit_model bm[], const int num_bits ) + { + int model = 1; + int i; + for( i = num_bits; i > 0; --i ) + model = ( model << 1 ) | Rd_decode_bit( rdec, &bm[model] ); + return model - (1 << num_bits); + } + + +static inline int Rd_decode_matched( struct Range_decoder * const rdec, + Bit_model bm[], const int match_byte ) + { + Bit_model * const bm1 = bm + 0x100; + int symbol = 1; + int i; + for( i = 7; i >= 0; --i ) + { + const int match_bit = ( match_byte >> i ) & 1; + const int bit = Rd_decode_bit( rdec, &bm1[(match_bit<<8)+symbol] ); + symbol = ( symbol << 1 ) | bit; + if( match_bit != bit ) + { + while( --i >= 0 ) + symbol = ( symbol << 1 ) | Rd_decode_bit( rdec, &bm[symbol] ); + break; + } + } + return symbol & 0xFF; + } + + +static inline int Rd_decode_tree_reversed( struct Range_decoder * const rdec, + Bit_model bm[], const int num_bits ) + { + int model = 1; + int symbol = 0; + int i; + for( i = 0; i < num_bits; ++i ) + { + const int bit = Rd_decode_bit( rdec, &bm[model] ); + model <<= 1; + if( bit ) { model |= 1; symbol |= (1 << i); } + } + return symbol; + } + + +static inline bool Rd_enough_available_bytes( const struct Range_decoder * const rdec ) + { + return ( Cb_used_bytes( &rdec->cb ) >= rd_min_available_bytes || + ( rdec->at_stream_end && Cb_used_bytes( &rdec->cb ) > 0 ) ); + } + + +static inline int Rd_read_data( struct Range_decoder * const rdec, + uint8_t * const outbuf, const int size ) + { + const int sz = Cb_read_data( &rdec->cb, outbuf, size ); + if( sz > 0 ) rdec->member_position += sz; + return sz; + } + + +struct Len_decoder + { + Bit_model choice1; + Bit_model choice2; + Bit_model bm_low[pos_states][len_low_symbols]; + Bit_model bm_mid[pos_states][len_mid_symbols]; + Bit_model bm_high[len_high_symbols]; + }; + +static inline void Led_init( struct Len_decoder * const len_decoder ) + { + int i, j; + Bm_init( &len_decoder->choice1 ); + Bm_init( &len_decoder->choice2 ); + for( i = 0; i < pos_states; ++i ) + for( j = 0; j < len_low_symbols; ++j ) + Bm_init( &len_decoder->bm_low[i][j] ); + for( i = 0; i < pos_states; ++i ) + for( j = 0; j < len_mid_symbols; ++j ) + Bm_init( &len_decoder->bm_mid[i][j] ); + for( i = 0; i < len_high_symbols; ++i ) + Bm_init( &len_decoder->bm_high[i] ); + } + +static inline int Led_decode( struct Len_decoder * const len_decoder, + struct Range_decoder * const rdec, + const int pos_state ) + { + if( Rd_decode_bit( rdec, &len_decoder->choice1 ) == 0 ) + return Rd_decode_tree( rdec, len_decoder->bm_low[pos_state], len_low_bits ); + if( Rd_decode_bit( rdec, &len_decoder->choice2 ) == 0 ) + return len_low_symbols + + Rd_decode_tree( rdec, len_decoder->bm_mid[pos_state], len_mid_bits ); + return len_low_symbols + len_mid_symbols + + Rd_decode_tree( rdec, len_decoder->bm_high, len_high_bits ); + } + + +struct Literal_decoder + { + Bit_model bm_literal[1<<literal_context_bits][0x300]; + }; + +static inline void Lid_init( struct Literal_decoder * const lidec ) + { + int i, j; + for( i = 0; i < 1<<literal_context_bits; ++i ) + for( j = 0; j < 0x300; ++j ) + Bm_init( &lidec->bm_literal[i][j] ); + } + +static inline int Lid_state( const uint8_t prev_byte ) + { return ( prev_byte >> ( 8 - literal_context_bits ) ); } + +static inline uint8_t Lid_decode( struct Literal_decoder * const lidec, + struct Range_decoder * const rdec, + const uint8_t prev_byte ) + { return Rd_decode_tree( rdec, lidec->bm_literal[Lid_state(prev_byte)], 8 ); } + +static inline uint8_t Lid_decode_matched( struct Literal_decoder * const lidec, + struct Range_decoder * const rdec, + const uint8_t prev_byte, + const uint8_t match_byte ) + { return Rd_decode_matched( rdec, lidec->bm_literal[Lid_state(prev_byte)], match_byte ); } + + +enum { lzd_min_free_bytes = max_match_len }; + +struct LZ_decoder + { + struct Circular_buffer cb; + long long partial_data_pos; + int dictionary_size; + uint32_t crc; + int member_version; + bool member_finished; + bool verify_trailer_pending; + unsigned int rep0; /* rep[0-3] latest four distances */ + unsigned int rep1; /* used for efficient coding of */ + unsigned int rep2; /* repeated distances */ + unsigned int rep3; + State state; + + Bit_model bm_match[states][pos_states]; + Bit_model bm_rep[states]; + Bit_model bm_rep0[states]; + Bit_model bm_rep1[states]; + Bit_model bm_rep2[states]; + Bit_model bm_len[states][pos_states]; + Bit_model bm_dis_slot[max_dis_states][1<<dis_slot_bits]; + Bit_model bm_dis[modeled_distances-end_dis_model+1]; + Bit_model bm_align[dis_align_size]; + + struct Range_decoder * range_decoder; + struct Len_decoder len_decoder; + struct Len_decoder rep_match_len_decoder; + struct Literal_decoder literal_decoder; + }; + +static inline bool LZd_init( struct LZ_decoder * const decoder, + const File_header header, + struct Range_decoder * const rdec ) + { + int i, j; + if( !Cb_init( &decoder->cb, max( 65536, Fh_get_dictionary_size( header ) ) + lzd_min_free_bytes ) ) + return false; + decoder->partial_data_pos = 0; + decoder->dictionary_size = Fh_get_dictionary_size( header ); + decoder->crc = 0xFFFFFFFFU; + decoder->member_version = Fh_version( header ); + decoder->member_finished = false; + decoder->verify_trailer_pending = false; + decoder->rep0 = 0; + decoder->rep1 = 0; + decoder->rep2 = 0; + decoder->rep3 = 0; + decoder->state = 0; + + for( i = 0; i < states; ++i ) + { + for( j = 0; j < pos_states; ++j ) + { + Bm_init( &decoder->bm_match[i][j] ); + Bm_init( &decoder->bm_len[i][j] ); + } + Bm_init( &decoder->bm_rep[i] ); + Bm_init( &decoder->bm_rep0[i] ); + Bm_init( &decoder->bm_rep1[i] ); + Bm_init( &decoder->bm_rep2[i] ); + } + for( i = 0; i < max_dis_states; ++i ) + for( j = 0; j < 1<<dis_slot_bits; ++j ) + Bm_init( &decoder->bm_dis_slot[i][j] ); + for( i = 0; i < modeled_distances-end_dis_model+1; ++i ) + Bm_init( &decoder->bm_dis[i] ); + for( i = 0; i < dis_align_size; ++i ) + Bm_init( &decoder->bm_align[i] ); + + decoder->range_decoder = rdec; + Led_init( &decoder->len_decoder ); + Led_init( &decoder->rep_match_len_decoder ); + Lid_init( &decoder->literal_decoder ); + decoder->cb.buffer[decoder->cb.buffer_size-1] = 0; /* prev_byte of first_byte */ + return true; + } + +static inline void LZd_free( struct LZ_decoder * const decoder ) + { Cb_free( &decoder->cb ); } + +static inline bool LZd_member_finished( const struct LZ_decoder * const decoder ) + { return ( decoder->member_finished && !Cb_used_bytes( &decoder->cb ) ); } + +static inline uint32_t LZd_crc( const struct LZ_decoder * const decoder ) + { return decoder->crc ^ 0xFFFFFFFFU; } + +static inline long long LZd_data_position( const struct LZ_decoder * const decoder ) + { return decoder->partial_data_pos + decoder->cb.put; } + + +static bool LZd_verify_trailer( struct LZ_decoder * const decoder ) + { + File_trailer trailer; + const int trailer_size = Ft_versioned_size( decoder->member_version ); + const long long member_size = + decoder->range_decoder->member_position + trailer_size; + + int size = Rd_read_data( decoder->range_decoder, trailer, trailer_size ); + if( size < trailer_size ) return false; + + if( decoder->member_version == 0 ) Ft_set_member_size( trailer, member_size ); + + if( decoder->range_decoder->code != 0 || + Ft_get_data_crc( trailer ) != LZd_crc( decoder ) || + Ft_get_data_size( trailer ) != LZd_data_position( decoder ) || + Ft_get_member_size( trailer ) != member_size ) return false; + return true; + } + + +static inline void LZd_copy_block( struct LZ_decoder * const decoder, + const int distance, int len ) + { + int i = decoder->cb.put - distance - 1; + if( i < 0 ) i += decoder->cb.buffer_size; + if( len < decoder->cb.buffer_size - max( decoder->cb.put, i ) && + len <= abs( decoder->cb.put - i ) ) + { + CRC32_update_buf( &decoder->crc, decoder->cb.buffer + i, len ); + memcpy( decoder->cb.buffer + decoder->cb.put, decoder->cb.buffer + i, len ); + decoder->cb.put += len; + } + else for( ; len > 0; --len ) + { + CRC32_update_byte( &decoder->crc, decoder->cb.buffer[i] ); + decoder->cb.buffer[decoder->cb.put] = decoder->cb.buffer[i]; + if( ++decoder->cb.put >= decoder->cb.buffer_size ) + { decoder->partial_data_pos += decoder->cb.put; decoder->cb.put = 0; } + if( ++i >= decoder->cb.buffer_size ) i = 0; + } + } + + +static inline bool LZd_enough_free_bytes( const struct LZ_decoder * const decoder ) + { return Cb_free_bytes( &decoder->cb ) >= lzd_min_free_bytes; } + + +static inline uint8_t LZd_get_byte( const struct LZ_decoder * const decoder, + const int distance ) + { + int i = decoder->cb.put - distance - 1; + if( i < 0 ) i += decoder->cb.buffer_size; + return decoder->cb.buffer[i]; + } + +static inline uint8_t LZd_get_prev_byte( const struct LZ_decoder * const decoder ) + { + const int i = + ( ( decoder->cb.put > 0 ) ? decoder->cb.put : decoder->cb.buffer_size ) - 1; + return decoder->cb.buffer[i]; + } + +static inline void LZd_put_byte( struct LZ_decoder * const decoder, + const uint8_t b ) + { + CRC32_update_byte( &decoder->crc, b ); + decoder->cb.buffer[decoder->cb.put] = b; + if( ++decoder->cb.put >= decoder->cb.buffer_size ) + { decoder->partial_data_pos += decoder->cb.put; decoder->cb.put = 0; } + } + + +/* Return value: 0 = OK, 1 = decoder error, 2 = unexpected EOF, + 3 = trailer error, 4 = unknown marker found. */ +static int LZd_decode_member( struct LZ_decoder * const decoder ) + { + State * const state = &decoder->state; + if( decoder->member_finished ) return 0; + if( !Rd_try_reload( decoder->range_decoder, false ) ) return 0; + if( decoder->verify_trailer_pending ) + { + if( Rd_available_bytes( decoder->range_decoder ) < Ft_versioned_size( decoder->member_version ) && + !decoder->range_decoder->at_stream_end ) + return 0; + decoder->verify_trailer_pending = false; + decoder->member_finished = true; + if( LZd_verify_trailer( decoder ) ) return 0; else return 3; + } + + while( !Rd_finished( decoder->range_decoder ) ) + { + const int pos_state = LZd_data_position( decoder ) & pos_state_mask; + if( !Rd_enough_available_bytes( decoder->range_decoder ) || + !LZd_enough_free_bytes( decoder ) ) + return 0; + if( Rd_decode_bit( decoder->range_decoder, &decoder->bm_match[*state][pos_state] ) == 0 ) + { + const uint8_t prev_byte = LZd_get_prev_byte( decoder ); + if( St_is_char( *state ) ) + LZd_put_byte( decoder, Lid_decode( &decoder->literal_decoder, + decoder->range_decoder, prev_byte ) ); + else + LZd_put_byte( decoder, Lid_decode_matched( &decoder->literal_decoder, + decoder->range_decoder, prev_byte, LZd_get_byte( decoder, decoder->rep0 ) ) ); + St_set_char( state ); + } + else + { + int len; + if( Rd_decode_bit( decoder->range_decoder, &decoder->bm_rep[*state] ) == 1 ) + { + len = 0; + if( Rd_decode_bit( decoder->range_decoder, &decoder->bm_rep0[*state] ) == 1 ) + { + unsigned int distance; + if( Rd_decode_bit( decoder->range_decoder, &decoder->bm_rep1[*state] ) == 0 ) + distance = decoder->rep1; + else + { + if( Rd_decode_bit( decoder->range_decoder, &decoder->bm_rep2[*state] ) == 0 ) + distance = decoder->rep2; + else { distance = decoder->rep3; decoder->rep3 = decoder->rep2; } + decoder->rep2 = decoder->rep1; + } + decoder->rep1 = decoder->rep0; + decoder->rep0 = distance; + } + else + { + if( Rd_decode_bit( decoder->range_decoder, &decoder->bm_len[*state][pos_state] ) == 0 ) + { St_set_short_rep( state ); len = 1; } + } + if( len == 0 ) + { + St_set_rep( state ); + len = min_match_len + Led_decode( &decoder->rep_match_len_decoder, decoder->range_decoder, pos_state ); + } + } + else + { + int dis_slot; + const unsigned int rep0_saved = decoder->rep0; + len = min_match_len + Led_decode( &decoder->len_decoder, decoder->range_decoder, pos_state ); + dis_slot = Rd_decode_tree( decoder->range_decoder, decoder->bm_dis_slot[get_dis_state(len)], dis_slot_bits ); + if( dis_slot < start_dis_model ) decoder->rep0 = dis_slot; + else + { + const int direct_bits = ( dis_slot >> 1 ) - 1; + decoder->rep0 = ( 2 | ( dis_slot & 1 ) ) << direct_bits; + if( dis_slot < end_dis_model ) + decoder->rep0 += Rd_decode_tree_reversed( decoder->range_decoder, decoder->bm_dis + decoder->rep0 - dis_slot, direct_bits ); + else + { + decoder->rep0 += Rd_decode( decoder->range_decoder, direct_bits - dis_align_bits ) << dis_align_bits; + decoder->rep0 += Rd_decode_tree_reversed( decoder->range_decoder, decoder->bm_align, dis_align_bits ); + if( decoder->rep0 == 0xFFFFFFFFU ) /* Marker found */ + { + decoder->rep0 = rep0_saved; + Rd_normalize( decoder->range_decoder ); + if( len == min_match_len ) /* End Of Stream marker */ + { + if( Rd_available_bytes( decoder->range_decoder ) < Ft_versioned_size( decoder->member_version ) && + !decoder->range_decoder->at_stream_end ) + { decoder->verify_trailer_pending = true; return 0; } + decoder->member_finished = true; + if( LZd_verify_trailer( decoder ) ) return 0; else return 3; + } + if( len == min_match_len + 1 ) /* Sync Flush marker */ + { + if( Rd_try_reload( decoder->range_decoder, true ) ) continue; + else return 0; + } + return 4; + } + } + } + decoder->rep3 = decoder->rep2; + decoder->rep2 = decoder->rep1; decoder->rep1 = rep0_saved; + St_set_match( state ); + if( decoder->rep0 >= (unsigned int)decoder->dictionary_size || + ( decoder->rep0 >= (unsigned int)decoder->cb.put && + !decoder->partial_data_pos ) ) + return 1; + } + LZd_copy_block( decoder, decoder->rep0, len ); + } + } + return 2; + } diff --git a/decoder.cc b/decoder.cc deleted file mode 100644 index 0d8d4d5..0000000 --- a/decoder.cc +++ /dev/null @@ -1,289 +0,0 @@ -/* Lzlib - A compression library for lzip files - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. - - This library is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this library. If not, see <http://www.gnu.org/licenses/>. - - As a special exception, you may use this file as part of a free - software library without restriction. Specifically, if other files - instantiate templates or use macros or inline functions from this - file, or you compile this file and link it with other files to - produce an executable, this file does not by itself cause the - resulting executable to be covered by the GNU General Public - License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General - Public License. -*/ - -#define _FILE_OFFSET_BITS 64 - -#include <algorithm> -#include <cerrno> -#include <cstdlib> -#include <cstring> -#include <stdint.h> - -#include "lzlib.h" -#include "lzip.h" -#include "decoder.h" - - -namespace Lzlib { - -const CRC32 crc32; - -// Seeks a member header and updates `get'. -// Returns true if it finds a valid header. -bool Range_decoder::find_header() throw() - { - while( get != put ) - { - if( buffer[get] == magic_string[0] ) - { - int g = get; - File_header header; - for( int i = 0; i < File_header::size; ++i ) - { - if( g == put ) return false; // not enough data - header.data[i] = buffer[g]; - if( ++g >= buffer_size ) g = 0; - } - if( header.verify() ) return true; - } - if( ++get >= buffer_size ) get = 0; - } - return false; - } - - -// Returns true, fills `header', and updates `get' if `get' points to a -// valid header. -// Else returns false and leaves `get' unmodified. -bool Range_decoder::read_header( File_header & header ) throw() - { - int g = get; - for( int i = 0; i < File_header::size; ++i ) - { - if( g == put ) return false; // not enough data - header.data[i] = buffer[g]; - if( ++g >= buffer_size ) g = 0; - } - if( header.verify() ) - { - get = g; - member_pos = File_header::size; - reload_pending = true; - return true; - } - return false; - } - - -bool LZ_decoder::verify_trailer() - { - File_trailer trailer; - const int trailer_size = File_trailer::size( member_version ); - const long long member_size = range_decoder.member_position() + trailer_size; - bool error = false; - - for( int i = 0; i < trailer_size && !error; ++i ) - { - if( !range_decoder.finished() ) - trailer.data[i] = range_decoder.get_byte(); - else - { - error = true; - for( ; i < trailer_size; ++i ) trailer.data[i] = 0; - } - } - if( member_version == 0 ) trailer.member_size( member_size ); - if( !range_decoder.code_is_zero() ) error = true; - if( trailer.data_crc() != crc() ) error = true; - if( trailer.data_size() != data_position() ) error = true; - if( trailer.member_size() != member_size ) error = true; - return !error; - } - - - // Return value: 0 = OK, 1 = decoder error, 2 = unexpected EOF, - // 3 = trailer error, 4 = unknown marker found. -int LZ_decoder::decode_member() - { - if( member_finished_ ) return 0; - if( !range_decoder.try_reload() ) return 0; - if( verify_trailer_pending ) - { - if( range_decoder.available_bytes() < File_trailer::size( member_version ) && - !range_decoder.at_stream_end() ) - return 0; - verify_trailer_pending = false; - member_finished_ = true; - if( verify_trailer() ) return 0; else return 3; - } - while( true ) - { - if( range_decoder.finished() ) return 2; - if( !range_decoder.enough_available_bytes() || !enough_free_bytes() ) - return 0; - const int pos_state = data_position() & pos_state_mask; - if( range_decoder.decode_bit( bm_match[state()][pos_state] ) == 0 ) - { - const uint8_t prev_byte = get_prev_byte(); - if( state.is_char() ) - put_byte( literal_decoder.decode( range_decoder, prev_byte ) ); - else - put_byte( literal_decoder.decode_matched( range_decoder, prev_byte, - get_byte( rep0 ) ) ); - state.set_char(); - } - else - { - int len; - if( range_decoder.decode_bit( bm_rep[state()] ) == 1 ) - { - len = 0; - if( range_decoder.decode_bit( bm_rep0[state()] ) == 1 ) - { - unsigned int distance; - if( range_decoder.decode_bit( bm_rep1[state()] ) == 0 ) - distance = rep1; - else - { - if( range_decoder.decode_bit( bm_rep2[state()] ) == 0 ) - distance = rep2; - else { distance = rep3; rep3 = rep2; } - rep2 = rep1; - } - rep1 = rep0; - rep0 = distance; - } - else - { - if( range_decoder.decode_bit( bm_len[state()][pos_state] ) == 0 ) - { state.set_short_rep(); len = 1; } - } - if( len == 0 ) - { - state.set_rep(); - len = min_match_len + rep_match_len_decoder.decode( range_decoder, pos_state ); - } - } - else - { - const unsigned int rep0_saved = rep0; - len = min_match_len + len_decoder.decode( range_decoder, pos_state ); - const int dis_slot = range_decoder.decode_tree( bm_dis_slot[get_dis_state(len)], dis_slot_bits ); - if( dis_slot < start_dis_model ) rep0 = dis_slot; - else - { - const int direct_bits = ( dis_slot >> 1 ) - 1; - rep0 = ( 2 | ( dis_slot & 1 ) ) << direct_bits; - if( dis_slot < end_dis_model ) - rep0 += range_decoder.decode_tree_reversed( bm_dis + rep0 - dis_slot, direct_bits ); - else - { - rep0 += range_decoder.decode( direct_bits - dis_align_bits ) << dis_align_bits; - rep0 += range_decoder.decode_tree_reversed( bm_align, dis_align_bits ); - if( rep0 == 0xFFFFFFFFU ) // Marker found - { - rep0 = rep0_saved; - range_decoder.normalize(); - if( len == min_match_len ) // End Of Stream marker - { - if( range_decoder.available_bytes() < File_trailer::size( member_version ) && - !range_decoder.at_stream_end() ) - { verify_trailer_pending = true; return 0; } - member_finished_ = true; - if( verify_trailer() ) return 0; else return 3; - } - if( len == min_match_len + 1 ) // Sync Flush marker - { - if( range_decoder.try_reload( true ) ) continue; - else return 0; - } - return 4; - } - } - } - rep3 = rep2; rep2 = rep1; rep1 = rep0_saved; - state.set_match(); - if( rep0 >= (unsigned int)dictionary_size || - ( rep0 >= (unsigned int)put && !partial_data_pos ) ) - return 1; - } - copy_block( rep0, len ); - } - } - } - - -// Copies up to `out_size' bytes to `out_buffer' and updates `get'. -// Returns the number of bytes copied. -int Circular_buffer::read_data( uint8_t * const out_buffer, const int out_size ) throw() - { - if( out_size < 0 ) return 0; - int size = 0; - if( get > put ) - { - size = std::min( buffer_size - get, out_size ); - if( size > 0 ) - { - std::memcpy( out_buffer, buffer + get, size ); - get += size; - if( get >= buffer_size ) get = 0; - } - } - if( get < put ) - { - const int size2 = std::min( put - get, out_size - size ); - if( size2 > 0 ) - { - std::memcpy( out_buffer + size, buffer + get, size2 ); - get += size2; - size += size2; - } - } - return size; - } - - -// Copies up to `in_size' bytes from `in_buffer' and updates `put'. -// Returns the number of bytes copied. -int Circular_buffer::write_data( const uint8_t * const in_buffer, const int in_size ) throw() - { - if( in_size < 0 ) return 0; - int size = 0; - if( put >= get ) - { - size = std::min( buffer_size - put - (get == 0), in_size ); - if( size > 0 ) - { - std::memcpy( buffer + put, in_buffer, size ); - put += size; - if( put >= buffer_size ) put = 0; - } - } - if( put < get ) - { - const int size2 = std::min( get - put - 1, in_size - size ); - if( size2 > 0 ) - { - std::memcpy( buffer + put, in_buffer + size, size2 ); - put += size2; - size += size2; - } - } - return size; - } - -} // end namespace Lzlib diff --git a/decoder.h b/decoder.h deleted file mode 100644 index 1b89bd8..0000000 --- a/decoder.h +++ /dev/null @@ -1,323 +0,0 @@ -/* Lzlib - A compression library for lzip files - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. - - This library is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this library. If not, see <http://www.gnu.org/licenses/>. - - As a special exception, you may use this file as part of a free - software library without restriction. Specifically, if other files - instantiate templates or use macros or inline functions from this - file, or you compile this file and link it with other files to - produce an executable, this file does not by itself cause the - resulting executable to be covered by the GNU General Public - License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General - Public License. -*/ - -namespace Lzlib { - -class Range_decoder : public Circular_buffer - { - enum { min_available_bytes = 8 }; - long long member_pos; - uint32_t code; - uint32_t range; - bool reload_pending; - bool at_stream_end_; - -public: - Range_decoder() - : - Circular_buffer( 65536 + min_available_bytes ), - member_pos( 0 ), - code( 0 ), - range( 0xFFFFFFFFU ), - reload_pending( false ), - at_stream_end_( false ) {} - - bool at_stream_end() const throw() { return at_stream_end_; } - int available_bytes() const throw() { return used_bytes(); } - bool code_is_zero() const throw() { return ( code == 0 ); } - void finish() throw() { at_stream_end_ = true; } - bool finished() const throw() { return at_stream_end_ && !used_bytes(); } - int free_bytes() const throw() - { if( at_stream_end_ ) return 0; return Circular_buffer::free_bytes(); } - long long member_position() const throw() - { return member_pos; } - void purge() throw() { at_stream_end_ = true; Circular_buffer::reset(); } - void reset() throw() { at_stream_end_ = false; Circular_buffer::reset(); } - - bool find_header() throw(); - bool read_header( File_header & header ) throw(); - - bool enough_available_bytes() const throw() - { - return ( used_bytes() > 0 && - ( at_stream_end_ || used_bytes() >= min_available_bytes ) ); - } - - int write_data( const uint8_t * const in_buffer, const int in_size ) throw() - { - if( at_stream_end_ || in_size <= 0 ) return 0; - return Circular_buffer::write_data( in_buffer, in_size ); - } - - uint8_t get_byte() - { - ++member_pos; - return Circular_buffer::get_byte(); - } - - bool try_reload( const bool force = false ) throw() - { - if( force ) reload_pending = true; - if( reload_pending && available_bytes() >= 5 ) - { - reload_pending = false; - code = 0; - range = 0xFFFFFFFFU; - for( int i = 0; i < 5; ++i ) code = (code << 8) | get_byte(); - } - return !reload_pending; - } - - void normalize() - { - if( range <= 0x00FFFFFFU ) - { range <<= 8; code = (code << 8) | get_byte(); } - } - - int decode( const int num_bits ) - { - int symbol = 0; - for( int i = num_bits; i > 0; --i ) - { - symbol <<= 1; - if( range <= 0x00FFFFFFU ) - { - range <<= 7; code = (code << 8) | get_byte(); - if( code >= range ) { code -= range; symbol |= 1; } - } - else - { - range >>= 1; - if( code >= range ) { code -= range; symbol |= 1; } - } - } - return symbol; - } - - int decode_bit( Bit_model & bm ) - { - normalize(); - const uint32_t bound = ( range >> bit_model_total_bits ) * bm.probability; - if( code < bound ) - { - range = bound; - bm.probability += (bit_model_total - bm.probability) >> bit_model_move_bits; - return 0; - } - else - { - range -= bound; - code -= bound; - bm.probability -= bm.probability >> bit_model_move_bits; - return 1; - } - } - - int decode_tree( Bit_model bm[], const int num_bits ) - { - int model = 1; - for( int i = num_bits; i > 0; --i ) - model = ( model << 1 ) | decode_bit( bm[model] ); - return model - (1 << num_bits); - } - - int decode_tree_reversed( Bit_model bm[], const int num_bits ) - { - int model = 1; - int symbol = 0; - for( int i = 0; i < num_bits; ++i ) - { - const int bit = decode_bit( bm[model] ); - model <<= 1; - if( bit ) { model |= 1; symbol |= (1 << i); } - } - return symbol; - } - - int decode_matched( Bit_model bm[], const int match_byte ) - { - Bit_model * const bm1 = bm + 0x100; - int symbol = 1; - for( int i = 7; i >= 0; --i ) - { - const int match_bit = ( match_byte >> i ) & 1; - const int bit = decode_bit( bm1[(match_bit<<8)+symbol] ); - symbol = ( symbol << 1 ) | bit; - if( match_bit != bit ) - { - while( --i >= 0 ) - symbol = ( symbol << 1 ) | decode_bit( bm[symbol] ); - break; - } - } - return symbol & 0xFF; - } - }; - - -class Len_decoder - { - Bit_model choice1; - Bit_model choice2; - Bit_model bm_low[pos_states][len_low_symbols]; - Bit_model bm_mid[pos_states][len_mid_symbols]; - Bit_model bm_high[len_high_symbols]; - -public: - int decode( Range_decoder & range_decoder, const int pos_state ) - { - if( range_decoder.decode_bit( choice1 ) == 0 ) - return range_decoder.decode_tree( bm_low[pos_state], len_low_bits ); - if( range_decoder.decode_bit( choice2 ) == 0 ) - return len_low_symbols + - range_decoder.decode_tree( bm_mid[pos_state], len_mid_bits ); - return len_low_symbols + len_mid_symbols + - range_decoder.decode_tree( bm_high, len_high_bits ); - } - }; - - -class Literal_decoder - { - Bit_model bm_literal[1<<literal_context_bits][0x300]; - - int lstate( const int prev_byte ) const throw() - { return ( prev_byte >> ( 8 - literal_context_bits ) ); } - -public: - uint8_t decode( Range_decoder & range_decoder, const uint8_t prev_byte ) - { return range_decoder.decode_tree( bm_literal[lstate(prev_byte)], 8 ); } - - uint8_t decode_matched( Range_decoder & range_decoder, - const uint8_t prev_byte, const uint8_t match_byte ) - { return range_decoder.decode_matched( bm_literal[lstate(prev_byte)], - match_byte ); } - }; - - -class LZ_decoder : public Circular_buffer - { - enum { min_free_bytes = max_match_len }; - long long partial_data_pos; - const int dictionary_size; - uint32_t crc_; - const int member_version; - bool member_finished_; - bool verify_trailer_pending; - unsigned int rep0; // rep[0-3] latest four distances - unsigned int rep1; // used for efficient coding of - unsigned int rep2; // repeated distances - unsigned int rep3; - State state; - - Bit_model bm_match[State::states][pos_states]; - Bit_model bm_rep[State::states]; - Bit_model bm_rep0[State::states]; - Bit_model bm_rep1[State::states]; - Bit_model bm_rep2[State::states]; - Bit_model bm_len[State::states][pos_states]; - Bit_model bm_dis_slot[max_dis_states][1<<dis_slot_bits]; - Bit_model bm_dis[modeled_distances-end_dis_model+1]; - Bit_model bm_align[dis_align_size]; - - Range_decoder & range_decoder; - Len_decoder len_decoder; - Len_decoder rep_match_len_decoder; - Literal_decoder literal_decoder; - - bool verify_trailer(); - - uint8_t get_prev_byte() const throw() - { - const int i = ( ( put > 0 ) ? put : buffer_size ) - 1; - return buffer[i]; - } - - uint8_t get_byte( const int distance ) const throw() - { - int i = put - distance - 1; - if( i < 0 ) i += buffer_size; - return buffer[i]; - } - - void put_byte( const uint8_t b ) - { - crc32.update( crc_, b ); - buffer[put] = b; - if( ++put >= buffer_size ) { partial_data_pos += put; put = 0; } - } - - void copy_block( const int distance, int len ) - { - int i = put - distance - 1; - if( i < 0 ) i += buffer_size; - if( len < buffer_size - std::max( put, i ) && len <= std::abs( put - i ) ) - { - crc32.update( crc_, buffer + i, len ); - std::memcpy( buffer + put, buffer + i, len ); - put += len; - } - else for( ; len > 0; --len ) - { - crc32.update( crc_, buffer[i] ); - buffer[put] = buffer[i]; - if( ++put >= buffer_size ) { partial_data_pos += put; put = 0; } - if( ++i >= buffer_size ) i = 0; - } - } - -public: - LZ_decoder( const File_header & header, Range_decoder & rdec ) - : - Circular_buffer( std::max( 65536, header.dictionary_size() ) + min_free_bytes ), - partial_data_pos( 0 ), - dictionary_size( header.dictionary_size() ), - crc_( 0xFFFFFFFFU ), - member_version( header.version() ), - member_finished_( false ), - verify_trailer_pending( false ), - rep0( 0 ), - rep1( 0 ), - rep2( 0 ), - rep3( 0 ), - range_decoder( rdec ) - { buffer[buffer_size-1] = 0; } // prev_byte of first_byte - - bool enough_free_bytes() const throw() - { return free_bytes() >= min_free_bytes; } - - uint32_t crc() const throw() { return crc_ ^ 0xFFFFFFFFU; } - bool member_finished() const throw() - { return ( member_finished_ && !used_bytes() ); } - - long long data_position() const throw() - { return partial_data_pos + put; } - - int decode_member(); - }; - -} // end namespace Lzlib diff --git a/doc/lzlib.info b/doc/lzlib.info index e502f6b..bfa5a82 100644 --- a/doc/lzlib.info +++ b/doc/lzlib.info @@ -12,7 +12,7 @@ File: lzlib.info, Node: Top, Next: Introduction, Up: (dir) Lzlib Manual ************ -This manual is for Lzlib (version 1.2, 25 October 2011). +This manual is for Lzlib (version 1.3, 29 February 2012). * Menu: @@ -30,7 +30,7 @@ This manual is for Lzlib (version 1.2, 25 October 2011). * Concept Index:: Index of concepts - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. + Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. This manual is free documentation: you have unlimited permission to copy, distribute and modify it. @@ -44,11 +44,11 @@ File: lzlib.info, Node: Introduction, Next: Library Version, Prev: Top, Up: Lzlib is a data compression library providing in-memory LZMA compression and decompression functions, including integrity checking of the decompressed data. The compressed data format used by the library is the -lzip format. +lzip format. Lzlib is written in C. The functions and variables forming the interface of the compression library are declared in the file `lzlib.h'. Usage examples of the -library are given in the files `main.cc' and `bbexample.cc' from the +library are given in the files `main.c' and `bbexample.c' from the source distribution. Compression/decompression is done by repeatedly calling a couple of @@ -189,7 +189,7 @@ verified by calling `LZ_compress_errno' before using it. MEMBER_SIZE sets the member size limit in bytes. Minimum member size limit is 100kB. Small member size may degrade compression - ratio, so use it only when needed. To produce a single member data + ratio, so use it only when needed. To produce a single-member data stream, give MEMBER_SIZE a value larger than the amount of data to be produced, for example LLONG_MAX. @@ -210,7 +210,7 @@ verified by calling `LZ_compress_errno' before using it. -- Function: int LZ_compress_restart_member ( struct LZ_Encoder * const ENCODER, const long long MEMBER_SIZE ) - Use this function to start a new member, in a multimember data + Use this function to start a new member, in a multi-member data stream. Call this function only after `LZ_compress_member_finished' indicates that the current member has been fully read (with the `LZ_compress_read' function). @@ -262,8 +262,8 @@ verified by calling `LZ_compress_errno' before using it. -- Function: int LZ_compress_member_finished ( struct LZ_Encoder * const ENCODER ) - Returns 1 if the current member, in a multimember data stream, has - been fully read and `LZ_compress_restart_member' can be safely + Returns 1 if the current member, in a multi-member data stream, + has been fully read and `LZ_compress_restart_member' can be safely called. Otherwise it returns 0. -- Function: long long LZ_compress_data_position ( struct LZ_Encoder * @@ -542,7 +542,7 @@ with no additional information before, between, or after them. `Member size (8 bytes)' Total size of the member, including header and trailer. This - facilitates safe recovery of undamaged members from multimember + facilitates safe recovery of undamaged members from multi-member files. @@ -554,10 +554,15 @@ File: lzlib.info, Node: Examples, Next: Problems, Prev: Data Format, Up: Top This chapter shows the order in which the library functions should be called depending on what kind of data stream you want to compress or -decompress. See the file `bbexample.cc' in the source distribution for +decompress. See the file `bbexample.c' in the source distribution for an example of how buffer-to-buffer compression/decompression can be implemented using lzlib. + Note that lzlib's interface is symmetrical. That is, the code for +normal compression and decompression is identical except because one +calls LZ_compress* functions while the other calls LZ_decompress* +functions. + Example 1: Normal compression (MEMBER_SIZE > total output). @@ -607,7 +612,7 @@ Example 4: Decompression using LZ_decompress_write_size. 7) LZ_decompress_close -Example 5: Multimember compression (MEMBER_SIZE < total output). +Example 5: Multi-member compression (MEMBER_SIZE < total output). 1) LZ_compress_open 2) go to step 5 if LZ_compress_write_size returns 0 @@ -621,7 +626,7 @@ Example 5: Multimember compression (MEMBER_SIZE < total output). 10) LZ_compress_close -Example 6: Multimember compression (user-restarted members). +Example 6: Multi-member compression (user-restarted members). 1) LZ_compress_open 2) LZ_compress_write @@ -704,17 +709,22 @@ Concept Index Tag Table: Node: Top219 -Node: Introduction1311 -Node: Library Version3136 -Node: Buffering3781 -Node: Parameter Limits4901 -Node: Compression Functions5858 -Node: Decompression Functions11955 -Node: Error Codes18027 -Node: Error Messages19966 -Node: Data Format20545 -Node: Examples22553 -Node: Problems26419 -Node: Concept Index26991 +Node: Introduction1318 +Node: Library Version3164 +Node: Buffering3809 +Node: Parameter Limits4929 +Node: Compression Functions5886 +Node: Decompression Functions11985 +Node: Error Codes18057 +Node: Error Messages19996 +Node: Data Format20575 +Node: Examples22584 +Node: Problems26667 +Node: Concept Index27239 End Tag Table + + +Local Variables: +coding: iso-8859-15 +End: diff --git a/doc/lzlib.texinfo b/doc/lzlib.texinfo index 55e5fa6..17b78f1 100644 --- a/doc/lzlib.texinfo +++ b/doc/lzlib.texinfo @@ -1,12 +1,13 @@ \input texinfo @c -*-texinfo-*- @c %**start of header @setfilename lzlib.info +@documentencoding ISO-8859-15 @settitle Lzlib Manual @finalout @c %**end of header -@set UPDATED 25 October 2011 -@set VERSION 1.2 +@set UPDATED 29 February 2012 +@set VERSION 1.3 @dircategory Data Compression @direntry @@ -49,7 +50,7 @@ This manual is for Lzlib (version @value{VERSION}, @value{UPDATED}). @end menu @sp 1 -Copyright @copyright{} 2009, 2010, 2011 Antonio Diaz Diaz. +Copyright @copyright{} 2009, 2010, 2011, 2012 Antonio Diaz Diaz. This manual is free documentation: you have unlimited permission to copy, distribute and modify it. @@ -62,12 +63,12 @@ to copy, distribute and modify it. Lzlib is a data compression library providing in-memory LZMA compression and decompression functions, including integrity checking of the decompressed data. The compressed data format used by the library is the -lzip format. +lzip format. Lzlib is written in C. The functions and variables forming the interface of the compression library are declared in the file @samp{lzlib.h}. Usage examples of the -library are given in the files @samp{main.cc} and @samp{bbexample.cc} -from the source distribution. +library are given in the files @samp{main.c} and @samp{bbexample.c} from +the source distribution. Compression/decompression is done by repeatedly calling a couple of read/write functions until all the data has been processed by the @@ -213,7 +214,7 @@ ratios but longer compression times. @var{member_size} sets the member size limit in bytes. Minimum member size limit is 100kB. Small member size may degrade compression ratio, so -use it only when needed. To produce a single member data stream, give +use it only when needed. To produce a single-member data stream, give @var{member_size} a value larger than the amount of data to be produced, for example LLONG_MAX. @end deftypefun @@ -237,7 +238,7 @@ After all the produced compressed data has been read with @deftypefun int LZ_compress_restart_member ( struct LZ_Encoder * const @var{encoder}, const long long @var{member_size} ) -Use this function to start a new member, in a multimember data stream. +Use this function to start a new member, in a multi-member data stream. Call this function only after @samp{LZ_compress_member_finished} indicates that the current member has been fully read (with the @samp{LZ_compress_read} function). @@ -297,7 +298,7 @@ be safely called. Otherwise it returns 0. @deftypefun int LZ_compress_member_finished ( struct LZ_Encoder * const @var{encoder} ) -Returns 1 if the current member, in a multimember data stream, has been +Returns 1 if the current member, in a multi-member data stream, has been fully read and @samp{LZ_compress_restart_member} can be safely called. Otherwise it returns 0. @end deftypefun @@ -612,7 +613,7 @@ Size of the uncompressed original data. @item Member size (8 bytes) Total size of the member, including header and trailer. This facilitates -safe recovery of undamaged members from multimember files. +safe recovery of undamaged members from multi-member files. @end table @@ -623,10 +624,14 @@ safe recovery of undamaged members from multimember files. This chapter shows the order in which the library functions should be called depending on what kind of data stream you want to compress or -decompress. See the file @samp{bbexample.cc} in the source distribution +decompress. See the file @samp{bbexample.c} in the source distribution for an example of how buffer-to-buffer compression/decompression can be implemented using lzlib. +Note that lzlib's interface is symmetrical. That is, the code for normal +compression and decompression is identical except because one calls +LZ_compress* functions while the other calls LZ_decompress* functions. + @sp 1 @noindent Example 1: Normal compression (@var{member_size} > total output). @@ -693,7 +698,7 @@ Example 4: Decompression using LZ_decompress_write_size. @sp 1 @noindent -Example 5: Multimember compression (@var{member_size} < total output). +Example 5: Multi-member compression (@var{member_size} < total output). @example 1) LZ_compress_open @@ -711,7 +716,7 @@ Example 5: Multimember compression (@var{member_size} < total output). @sp 1 @noindent -Example 6: Multimember compression (user-restarted members). +Example 6: Multi-member compression (user-restarted members). @example 1) LZ_compress_open diff --git a/encoder.c b/encoder.c new file mode 100644 index 0000000..bf0dafa --- /dev/null +++ b/encoder.c @@ -0,0 +1,1334 @@ +/* Lzlib - A compression library for lzip files + Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. + + This library is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this library. If not, see <http://www.gnu.org/licenses/>. + + As a special exception, you may use this file as part of a free + software library without restriction. Specifically, if other files + instantiate templates or use macros or inline functions from this + file, or you compile this file and link it with other files to + produce an executable, this file does not by itself cause the + resulting executable to be covered by the GNU General Public + License. This exception does not however invalidate any other + reasons why the executable file might be covered by the GNU General + Public License. +*/ + +enum { max_num_trials = 1 << 12, + price_shift = 6 }; + +static inline int price0( const Bit_model probability ) + { return get_price( probability ); } + +static inline int price1( const Bit_model probability ) + { return get_price( bit_model_total-probability ); } + +static inline int price_bit( const Bit_model bm, const int bit ) + { if( bit ) return price1( bm ); else return price0( bm ); } + + +static inline int price_symbol( const Bit_model bm[], int symbol, + const int num_bits ) + { + int price = 0; + symbol |= ( 1 << num_bits ); + while( symbol > 1 ) + { + const int bit = symbol & 1; + symbol >>= 1; + price += price_bit( bm[symbol], bit ); + } + return price; + } + + +static inline int price_symbol_reversed( const Bit_model bm[], int symbol, + const int num_bits ) + { + int price = 0; + int model = 1; + int i; + for( i = num_bits; i > 0; --i ) + { + const int bit = symbol & 1; + symbol >>= 1; + price += price_bit( bm[model], bit ); + model = ( model << 1 ) | bit; + } + return price; + } + + +static inline int price_matched( const Bit_model bm[], const int symbol, + const int match_byte ) + { + int price = 0; + int model = 1; + int i; + + for( i = 7; i >= 0; --i ) + { + const int match_bit = ( match_byte >> i ) & 1; + int bit = ( symbol >> i ) & 1; + price += price_bit( bm[(match_bit<<8)+model+0x100], bit ); + model = ( model << 1 ) | bit; + if( match_bit != bit ) + { + while( --i >= 0 ) + { + bit = ( symbol >> i ) & 1; + price += price_bit( bm[model], bit ); + model = ( model << 1 ) | bit; + } + break; + } + } + return price; + } + + +enum { /* bytes to keep in buffer before dictionary */ + before_size = max_num_trials + 1, + /* bytes to keep in buffer after pos */ + after_size = max_num_trials + max_match_len, + num_prev_positions4 = 1 << 20, + num_prev_positions3 = 1 << 18, + num_prev_positions2 = 1 << 16, + num_prev_positions = num_prev_positions4 + num_prev_positions3 + + num_prev_positions2 }; + +struct Matchfinder + { + long long partial_data_pos; + uint8_t * buffer; /* input buffer */ + int32_t * prev_positions; /* last seen position of key */ + int32_t * prev_pos_tree; + int dictionary_size; /* bytes to keep in buffer before pos */ + int buffer_size; + int pos; /* current pos in buffer */ + int cyclic_pos; /* current pos in dictionary */ + int stream_pos; /* first byte not yet read from file */ + int pos_limit; /* when reached, a new block must be read */ + int match_len_limit; + int cycles; + bool at_stream_end; /* stream_pos shows real end of file */ + bool been_flushed; + }; + + +static int Mf_write_data( struct Matchfinder * const mf, + const uint8_t * const inbuf, const int size ) + { + const int sz = min( mf->buffer_size - mf->stream_pos, size ); + if( !mf->at_stream_end && sz > 0 ) + { + memcpy( mf->buffer + mf->stream_pos, inbuf, sz ); + mf->stream_pos += sz; + } + return sz; + } + + +static bool Mf_normalize_pos( struct Matchfinder * const mf ) + { + if( mf->pos > mf->stream_pos ) + { mf->pos = mf->stream_pos; return false; } + if( !mf->at_stream_end ) + { + int i; + const int offset = mf->pos - mf->dictionary_size - before_size; + const int size = mf->stream_pos - offset; + memmove( mf->buffer, mf->buffer + offset, size ); + mf->partial_data_pos += offset; + mf->pos -= offset; + mf->stream_pos -= offset; + for( i = 0; i < num_prev_positions; ++i ) + if( mf->prev_positions[i] >= 0 ) mf->prev_positions[i] -= offset; + for( i = 0; i < 2 * mf->dictionary_size; ++i ) + if( mf->prev_pos_tree[i] >= 0 ) mf->prev_pos_tree[i] -= offset; + } + return true; + } + + +static bool Mf_init( struct Matchfinder * const mf, + const int dict_size, const int len_limit ) + { + int i; + mf->partial_data_pos = 0; + mf->dictionary_size = dict_size; + mf->buffer_size = ( 2 * max( 65536, dict_size ) ) + before_size + after_size; + mf->buffer = (uint8_t *)malloc( mf->buffer_size ); + mf->prev_positions = (int32_t *)malloc( num_prev_positions * sizeof (int32_t) ); + mf->prev_pos_tree = (int32_t *)malloc( 2 * dict_size * sizeof (int32_t) ); + mf->pos = 0; + mf->cyclic_pos = 0; + mf->stream_pos = 0; + mf->pos_limit = mf->buffer_size - after_size; + mf->match_len_limit = len_limit; + mf->cycles = ( len_limit < max_match_len ) ? 16 + ( len_limit / 2 ) : 256; + mf->at_stream_end = false; + mf->been_flushed = false; + + if( !mf->buffer || !mf->prev_positions || !mf->prev_pos_tree ) + { + if( mf->prev_pos_tree ) + { free( mf->prev_pos_tree ); mf->prev_pos_tree = 0; } + if( mf->prev_positions ) + { free( mf->prev_positions ); mf->prev_positions = 0; } + if( mf->buffer ) { free( mf->buffer ); mf->buffer = 0; } + return false; + } + for( i = 0; i < num_prev_positions; ++i ) mf->prev_positions[i] = -1; + return true; + } + + +static inline void Mf_free( struct Matchfinder * const mf ) + { + free( mf->prev_pos_tree ); mf->prev_pos_tree = 0; + free( mf->prev_positions ); mf->prev_positions = 0; + free( mf->buffer ); mf->buffer = 0; + } + +static inline uint8_t Mf_peek( const struct Matchfinder * const mf, const int i ) + { return mf->buffer[mf->pos+i]; } + +static inline int Mf_available_bytes( const struct Matchfinder * const mf ) + { return mf->stream_pos - mf->pos; } + +static inline long long Mf_data_position( const struct Matchfinder * const mf ) + { return mf->partial_data_pos + mf->pos; } + +static inline bool Mf_finished( const struct Matchfinder * const mf ) + { return mf->at_stream_end && mf->pos >= mf->stream_pos; } + +static inline const uint8_t * Mf_ptr_to_current_pos( const struct Matchfinder * const mf ) + { return mf->buffer + mf->pos; } + +static inline void Mf_set_flushing( struct Matchfinder * const mf, + const bool flushing ) + { mf->at_stream_end = flushing; } + +static inline int Mf_free_bytes( const struct Matchfinder * const mf ) + { if( mf->at_stream_end ) return 0; return mf->buffer_size - mf->stream_pos; } + +static inline bool Mf_enough_available_bytes( const struct Matchfinder * const mf ) + { + return ( mf->pos + after_size <= mf->stream_pos || + ( mf->at_stream_end && mf->pos < mf->stream_pos ) ); + } + +static inline bool Mf_dec_pos( struct Matchfinder * const mf, + const int ahead ) + { + if( ahead < 0 || mf->pos < ahead ) return false; + mf->pos -= ahead; + mf->cyclic_pos -= ahead; + if( mf->cyclic_pos < 0 ) mf->cyclic_pos += mf->dictionary_size; + return true; + } + +static inline int Mf_true_match_len( const struct Matchfinder * const mf, + const int index, const int distance, + int len_limit ) + { + const uint8_t * const data = mf->buffer + mf->pos + index; + int i = 0; + + if( index + len_limit > Mf_available_bytes( mf ) ) + len_limit = Mf_available_bytes( mf ) - index; + while( i < len_limit && data[i-distance] == data[i] ) ++i; + return i; + } + +static inline bool Mf_move_pos( struct Matchfinder * const mf ) + { + if( ++mf->cyclic_pos >= mf->dictionary_size ) mf->cyclic_pos = 0; + if( ++mf->pos >= mf->pos_limit ) return Mf_normalize_pos( mf ); + return true; + } + + +static void Mf_reset( struct Matchfinder * const mf ) + { + int i; + const int size = mf->stream_pos - mf->pos; + if( size > 0 ) memmove( mf->buffer, mf->buffer + mf->pos, size ); + mf->partial_data_pos = 0; + mf->stream_pos -= mf->pos; + mf->pos = 0; + mf->cyclic_pos = 0; + mf->at_stream_end = false; + mf->been_flushed = false; + for( i = 0; i < num_prev_positions; ++i ) mf->prev_positions[i] = -1; + } + + +static int Mf_longest_match_len( struct Matchfinder * const mf, + int * const distances ) + { + int32_t * ptr0 = mf->prev_pos_tree + ( mf->cyclic_pos << 1 ); + int32_t * ptr1 = ptr0 + 1; + int32_t * newptr; + const uint8_t * newdata; + int len = 0, len0 = 0, len1 = 0; + int maxlen = min_match_len - 1; + const int min_pos = (mf->pos >= mf->dictionary_size) ? + (mf->pos - mf->dictionary_size + 1) : 0; + const uint8_t * const data = mf->buffer + mf->pos; + int count, delta, key2, key3, key4, newpos, tmp; + int len_limit = mf->match_len_limit; + + if( len_limit > Mf_available_bytes( mf ) ) + { + mf->been_flushed = true; + len_limit = Mf_available_bytes( mf ); + if( len_limit < 4 ) { *ptr0 = *ptr1 = -1; return 0; } + } + + key2 = num_prev_positions4 + num_prev_positions3 + + ( ( (int)data[0] << 8 ) | data[1] ); + tmp = crc32[data[0]] ^ data[1] ^ ( (uint32_t)data[2] << 8 ); + key3 = num_prev_positions4 + (int)( tmp & ( num_prev_positions3 - 1 ) ); + key4 = (int)( ( tmp ^ ( crc32[data[3]] << 5 ) ) & + ( num_prev_positions4 - 1 ) ); + + if( distances ) + { + int np = mf->prev_positions[key2]; + if( np >= min_pos ) + { distances[2] = mf->pos - np - 1; maxlen = 2; } + else distances[2] = 0x7FFFFFFF; + np = mf->prev_positions[key3]; + if( np >= min_pos && mf->buffer[np] == data[0] ) + { distances[3] = mf->pos - np - 1; maxlen = 3; } + else distances[3] = 0x7FFFFFFF; + distances[4] = 0x7FFFFFFF; + } + + mf->prev_positions[key2] = mf->pos; + mf->prev_positions[key3] = mf->pos; + newpos = mf->prev_positions[key4]; + mf->prev_positions[key4] = mf->pos; + + for( count = mf->cycles; ; ) + { + if( newpos < min_pos || --count < 0 ) { *ptr0 = *ptr1 = -1; break; } + newdata = mf->buffer + newpos; + if( mf->been_flushed ) len = 0; + while( len < len_limit && newdata[len] == data[len] ) ++len; + + delta = mf->pos - newpos; + if( distances ) while( maxlen < len ) distances[++maxlen] = delta - 1; + + newptr = mf->prev_pos_tree + + ( ( mf->cyclic_pos - delta + + ( ( mf->cyclic_pos >= delta ) ? 0 : mf->dictionary_size ) ) << 1 ); + + if( len < len_limit ) + { + if( newdata[len] < data[len] ) + { + *ptr0 = newpos; + ptr0 = newptr + 1; + newpos = *ptr0; + len0 = len; if( len1 < len ) len = len1; + } + else + { + *ptr1 = newpos; + ptr1 = newptr; + newpos = *ptr1; + len1 = len; if( len0 < len ) len = len0; + } + } + else + { + *ptr0 = newptr[0]; + *ptr1 = newptr[1]; + break; + } + } + if( distances ) + { + if( distances[3] > distances[4] ) distances[3] = distances[4]; + if( distances[2] > distances[3] ) distances[2] = distances[3]; + } + return maxlen; + } + + +enum { re_min_free_bytes = 2 * max_num_trials }; + +struct Range_encoder + { + struct Circular_buffer cb; + uint64_t low; + long long partial_member_pos; + uint32_t range; + int ff_count; + uint8_t cache; + }; + +static inline void Re_shift_low( struct Range_encoder * const renc ) + { + const bool carry = ( renc->low > 0xFFFFFFFFU ); + if( carry || renc->low < 0xFF000000U ) + { + Cb_put_byte( &renc->cb, renc->cache + carry ); + for( ; renc->ff_count > 0; --renc->ff_count ) + Cb_put_byte( &renc->cb, 0xFF + carry ); + renc->cache = renc->low >> 24; + } + else ++renc->ff_count; + renc->low = ( renc->low & 0x00FFFFFFU ) << 8; + } + +static inline bool Re_init( struct Range_encoder * const renc ) + { + if( !Cb_init( &renc->cb, 65536 + re_min_free_bytes ) ) return false; + renc->low = 0; + renc->partial_member_pos = 0; + renc->range = 0xFFFFFFFFU; + renc->ff_count = 0; + renc->cache = 0; + return true; + } + +static inline void Re_free( struct Range_encoder * const renc ) + { Cb_free( &renc->cb ); } + +static inline long long Re_member_position( const struct Range_encoder * const renc ) + { return renc->partial_member_pos + Cb_used_bytes( &renc->cb ) + renc->ff_count; } + +static inline bool Re_enough_free_bytes( const struct Range_encoder * const renc ) + { return Cb_free_bytes( &renc->cb ) >= re_min_free_bytes; } + +static inline int Re_read_data( struct Range_encoder * const renc, + uint8_t * const out_buffer, const int out_size ) + { + const int size = Cb_read_data( &renc->cb, out_buffer, out_size ); + if( size > 0 ) renc->partial_member_pos += size; + return size; + } + +static inline void Re_flush( struct Range_encoder * const renc ) + { + int i; for( i = 0; i < 5; ++i ) Re_shift_low( renc ); + renc->low = 0; + renc->range = 0xFFFFFFFFU; + renc->ff_count = 0; + renc->cache = 0; + } + +static inline void Re_encode( struct Range_encoder * const renc, + const int symbol, const int num_bits ) + { + int i; + for( i = num_bits - 1; i >= 0; --i ) + { + renc->range >>= 1; + if( (symbol >> i) & 1 ) renc->low += renc->range; + if( renc->range <= 0x00FFFFFFU ) + { renc->range <<= 8; Re_shift_low( renc ); } + } + } + +static inline void Re_encode_bit( struct Range_encoder * const renc, + Bit_model * const probability, const int bit ) + { + const uint32_t bound = ( renc->range >> bit_model_total_bits ) * *probability; + if( !bit ) + { + renc->range = bound; + *probability += (bit_model_total - *probability) >> bit_model_move_bits; + } + else + { + renc->low += bound; + renc->range -= bound; + *probability -= *probability >> bit_model_move_bits; + } + if( renc->range <= 0x00FFFFFFU ) + { renc->range <<= 8; Re_shift_low( renc ); } + } + +static inline void Re_encode_tree( struct Range_encoder * const renc, + Bit_model bm[], const int symbol, const int num_bits ) + { + int mask = ( 1 << ( num_bits - 1 ) ); + int model = 1; + int i; + for( i = num_bits; i > 0; --i, mask >>= 1 ) + { + const int bit = ( symbol & mask ); + Re_encode_bit( renc, &bm[model], bit ); + model <<= 1; + if( bit ) model |= 1; + } + } + +static inline void Re_encode_tree_reversed( struct Range_encoder * const renc, + Bit_model bm[], int symbol, const int num_bits ) + { + int model = 1; + int i; + for( i = num_bits; i > 0; --i ) + { + const int bit = symbol & 1; + Re_encode_bit( renc, &bm[model], bit ); + model = ( model << 1 ) | bit; + symbol >>= 1; + } + } + +static inline void Re_encode_matched( struct Range_encoder * const renc, + Bit_model bm[], int symbol, int match_byte ) + { + int model = 1; + int i; + for( i = 7; i >= 0; --i ) + { + const int match_bit = ( match_byte >> i ) & 1; + int bit = ( symbol >> i ) & 1; + Re_encode_bit( renc, &bm[(match_bit<<8)+model+0x100], bit ); + model = ( model << 1 ) | bit; + if( match_bit != bit ) + { + while( --i >= 0 ) + { + bit = ( symbol >> i ) & 1; + Re_encode_bit( renc, &bm[model], bit ); + model = ( model << 1 ) | bit; + } + break; + } + } + } + + +struct Len_encoder + { + Bit_model choice1; + Bit_model choice2; + Bit_model bm_low[pos_states][len_low_symbols]; + Bit_model bm_mid[pos_states][len_mid_symbols]; + Bit_model bm_high[len_high_symbols]; + int prices[pos_states][max_len_symbols]; + int len_symbols; + int counters[pos_states]; + }; + +static void Lee_update_prices( struct Len_encoder * const len_encoder, + const int pos_state ) + { + int * const pps = len_encoder->prices[pos_state]; + int tmp = price0( len_encoder->choice1 ); + int len = 0; + for( ; len < len_low_symbols && len < len_encoder->len_symbols; ++len ) + pps[len] = tmp + + price_symbol( len_encoder->bm_low[pos_state], len, len_low_bits ); + tmp = price1( len_encoder->choice1 ); + for( ; len < len_low_symbols + len_mid_symbols && len < len_encoder->len_symbols; ++len ) + pps[len] = tmp + price0( len_encoder->choice2 ) + + price_symbol( len_encoder->bm_mid[pos_state], len - len_low_symbols, len_mid_bits ); + for( ; len < len_encoder->len_symbols; ++len ) + /* using 4 slots per value makes "Lee_price" faster */ + len_encoder->prices[3][len] = len_encoder->prices[2][len] = + len_encoder->prices[1][len] = len_encoder->prices[0][len] = + tmp + price1( len_encoder->choice2 ) + + price_symbol( len_encoder->bm_high, len - len_low_symbols - len_mid_symbols, len_high_bits ); + len_encoder->counters[pos_state] = len_encoder->len_symbols; + } + +static void Lee_init( struct Len_encoder * const len_encoder, + const int len_limit ) + { + int i, j; + Bm_init( &len_encoder->choice1 ); + Bm_init( &len_encoder->choice2 ); + for( i = 0; i < pos_states; ++i ) + for( j = 0; j < len_low_symbols; ++j ) + Bm_init( &len_encoder->bm_low[i][j] ); + for( i = 0; i < pos_states; ++i ) + for( j = 0; j < len_mid_symbols; ++j ) + Bm_init( &len_encoder->bm_mid[i][j] ); + for( i = 0; i < len_high_symbols; ++i ) + Bm_init( &len_encoder->bm_high[i] ); + len_encoder->len_symbols = len_limit + 1 - min_match_len; + for( i = 0; i < pos_states; ++i ) Lee_update_prices( len_encoder, i ); + } + + +static void Lee_encode( struct Len_encoder * const len_encoder, + struct Range_encoder * const renc, + int symbol, const int pos_state ) + { + symbol -= min_match_len; + if( symbol < len_low_symbols ) + { + Re_encode_bit( renc, &len_encoder->choice1, 0 ); + Re_encode_tree( renc, len_encoder->bm_low[pos_state], symbol, len_low_bits ); + } + else + { + Re_encode_bit( renc, &len_encoder->choice1, 1 ); + if( symbol < len_low_symbols + len_mid_symbols ) + { + Re_encode_bit( renc, &len_encoder->choice2, 0 ); + Re_encode_tree( renc, len_encoder->bm_mid[pos_state], + symbol - len_low_symbols, len_mid_bits ); + } + else + { + Re_encode_bit( renc, &len_encoder->choice2, 1 ); + Re_encode_tree( renc, len_encoder->bm_high, + symbol - len_low_symbols - len_mid_symbols, len_high_bits ); + } + } + if( --len_encoder->counters[pos_state] <= 0 ) + Lee_update_prices( len_encoder, pos_state ); + } + + +static inline int Lee_price( const struct Len_encoder * const len_encoder, + const int symbol, const int pos_state ) + { return len_encoder->prices[pos_state][symbol - min_match_len]; } + + +struct Literal_encoder + { + Bit_model bm_literal[1<<literal_context_bits][0x300]; + }; + +static inline void Lie_init( struct Literal_encoder * const lienc ) + { + int i, j; + for( i = 0; i < 1<<literal_context_bits; ++i ) + for( j = 0; j < 0x300; ++j ) + Bm_init( &lienc->bm_literal[i][j] ); + } + +static inline int Lie_state( const uint8_t prev_byte ) + { return ( prev_byte >> ( 8 - literal_context_bits ) ); } + +static inline void Lie_encode( struct Literal_encoder * const lienc, + struct Range_encoder * const renc, + uint8_t prev_byte, uint8_t symbol ) + { Re_encode_tree( renc, lienc->bm_literal[Lie_state(prev_byte)], symbol, 8 ); } + +static inline void Lie_encode_matched( struct Literal_encoder * const lienc, + struct Range_encoder * const renc, + uint8_t prev_byte, uint8_t symbol, + uint8_t match_byte ) + { Re_encode_matched( renc, lienc->bm_literal[Lie_state(prev_byte)], + symbol, match_byte ); } + +static inline int Lie_price_symbol( const struct Literal_encoder * const lienc, + uint8_t prev_byte, uint8_t symbol ) + { return price_symbol( lienc->bm_literal[Lie_state(prev_byte)], symbol, 8 ); } + +static inline int Lie_price_matched( const struct Literal_encoder * const lienc, + uint8_t prev_byte, uint8_t symbol, + uint8_t match_byte ) + { return price_matched( lienc->bm_literal[Lie_state(prev_byte)], + symbol, match_byte ); } + + +enum { infinite_price = 0x0FFFFFFF, + max_marker_size = 16, + num_rep_distances = 4 }; /* must be 4 */ + +struct Trial + { + State state; + int dis; + int prev_index; /* index of prev trial in trials[] */ + int price; /* dual use var; cumulative price, match length */ + int reps[num_rep_distances]; + }; + +static inline void Tr_update( struct Trial * const trial, + const int d, const int p_i, const int pr ) + { + if( pr < trial->price ) + { trial->dis = d; trial->prev_index = p_i; trial->price = pr; } + } + + +struct LZ_encoder + { + long long member_size_limit; + int longest_match_found; + uint32_t crc; + + Bit_model bm_match[states][pos_states]; + Bit_model bm_rep[states]; + Bit_model bm_rep0[states]; + Bit_model bm_rep1[states]; + Bit_model bm_rep2[states]; + Bit_model bm_len[states][pos_states]; + Bit_model bm_dis_slot[max_dis_states][1<<dis_slot_bits]; + Bit_model bm_dis[modeled_distances-end_dis_model+1]; + Bit_model bm_align[dis_align_size]; + + struct Matchfinder * matchfinder; + struct Range_encoder range_encoder; + struct Len_encoder len_encoder; + struct Len_encoder rep_match_len_encoder; + struct Literal_encoder literal_encoder; + + int num_dis_slots; + int rep_distances[num_rep_distances]; + int match_distances[max_match_len+1]; + struct Trial trials[max_num_trials]; + + int dis_slot_prices[max_dis_states][2*max_dictionary_bits]; + int dis_prices[max_dis_states][modeled_distances]; + int align_prices[dis_align_size]; + int align_price_count; + int fill_counter; + State state; + bool member_finished; + }; + + +static inline bool LZe_member_finished( const struct LZ_encoder * const encoder ) + { + return ( encoder->member_finished && + !Cb_used_bytes( &encoder->range_encoder.cb ) ); + } + + +static void LZe_fill_align_prices( struct LZ_encoder * const encoder ) + { + int i; + for( i = 0; i < dis_align_size; ++i ) + encoder->align_prices[i] = + price_symbol_reversed( encoder->bm_align, i, dis_align_bits ); + encoder->align_price_count = dis_align_size; + } + + +static void LZe_fill_distance_prices( struct LZ_encoder * const encoder ) + { + int dis, dis_state; + for( dis = start_dis_model; dis < modeled_distances; ++dis ) + { + const int dis_slot = dis_slots[dis]; + const int direct_bits = ( dis_slot >> 1 ) - 1; + const int base = ( 2 | ( dis_slot & 1 ) ) << direct_bits; + const int price = + price_symbol_reversed( encoder->bm_dis + base - dis_slot, + dis - base, direct_bits ); + for( dis_state = 0; dis_state < max_dis_states; ++dis_state ) + encoder->dis_prices[dis_state][dis] = price; + } + + for( dis_state = 0; dis_state < max_dis_states; ++dis_state ) + { + int * const dsp = encoder->dis_slot_prices[dis_state]; + int * const dp = encoder->dis_prices[dis_state]; + const Bit_model * const bmds = encoder->bm_dis_slot[dis_state]; + int slot = 0; + for( ; slot < end_dis_model && slot < encoder->num_dis_slots; ++slot ) + dsp[slot] = price_symbol( bmds, slot, dis_slot_bits ); + for( ; slot < encoder->num_dis_slots; ++slot ) + dsp[slot] = price_symbol( bmds, slot, dis_slot_bits ) + + (((( slot >> 1 ) - 1 ) - dis_align_bits ) << price_shift ); + + for( dis = 0; dis < start_dis_model; ++dis ) + dp[dis] = dsp[dis]; + for( ; dis < modeled_distances; ++dis ) + dp[dis] += dsp[dis_slots[dis]]; + } + } + + +static bool LZe_init( struct LZ_encoder * const encoder, + struct Matchfinder * const mf, + const File_header header, const long long member_size ) + { + int i, j; + encoder->member_size_limit = member_size - Ft_size - max_marker_size; + encoder->longest_match_found = 0; + encoder->crc = 0xFFFFFFFFU; + + for( i = 0; i < states; ++i ) + { + for( j = 0; j < pos_states; ++j ) + { + Bm_init( &encoder->bm_match[i][j] ); + Bm_init( &encoder->bm_len[i][j] ); + } + Bm_init( &encoder->bm_rep[i] ); + Bm_init( &encoder->bm_rep0[i] ); + Bm_init( &encoder->bm_rep1[i] ); + Bm_init( &encoder->bm_rep2[i] ); + } + for( i = 0; i < max_dis_states; ++i ) + for( j = 0; j < 1<<dis_slot_bits; ++j ) + Bm_init( &encoder->bm_dis_slot[i][j] ); + for( i = 0; i < modeled_distances-end_dis_model+1; ++i ) + Bm_init( &encoder->bm_dis[i] ); + for( i = 0; i < dis_align_size; ++i ) + Bm_init( &encoder->bm_align[i] ); + + encoder->matchfinder = mf; + if( !Re_init( &encoder->range_encoder ) ) return false; + Lee_init( &encoder->len_encoder, encoder->matchfinder->match_len_limit ); + Lee_init( &encoder->rep_match_len_encoder, encoder->matchfinder->match_len_limit ); + Lie_init( &encoder->literal_encoder ); + encoder->num_dis_slots = + 2 * real_bits( encoder->matchfinder->dictionary_size - 1 ); + for( i = 0; i < num_rep_distances; ++i ) encoder->rep_distances[i] = 0; + encoder->fill_counter = 0; + encoder->state = 0; + encoder->member_finished = false; + + LZe_fill_align_prices( encoder ); + + for( i = 0; i < Fh_size; ++i ) + Cb_put_byte( &encoder->range_encoder.cb, header[i] ); + return true; + } + + +static inline void LZe_free( struct LZ_encoder * const encoder ) + { + Re_free( &encoder->range_encoder ); + } + +static inline uint32_t LZe_crc( const struct LZ_encoder * const encoder ) + { return encoder->crc ^ 0xFFFFFFFFU; } + + /* move-to-front dis in/into reps */ +static inline void LZe_mtf_reps( const int dis, int reps[num_rep_distances] ) + { + int i; + if( dis >= num_rep_distances ) + { + for( i = num_rep_distances - 1; i > 0; --i ) reps[i] = reps[i-1]; + reps[0] = dis - num_rep_distances; + } + else if( dis > 0 ) + { + const int distance = reps[dis]; + for( i = dis; i > 0; --i ) reps[i] = reps[i-1]; + reps[0] = distance; + } + } + +static inline int LZe_price_rep_len1( const struct LZ_encoder * const encoder, + const State state, const int pos_state ) + { + return price0( encoder->bm_rep0[state] ) + + price0( encoder->bm_len[state][pos_state] ); + } + +static inline int LZe_price_rep( const struct LZ_encoder * const encoder, + const int rep, + const State state, const int pos_state ) + { + int price; + if( rep == 0 ) return price0( encoder->bm_rep0[state] ) + + price1( encoder->bm_len[state][pos_state] ); + price = price1( encoder->bm_rep0[state] ); + if( rep == 1 ) + price += price0( encoder->bm_rep1[state] ); + else + { + price += price1( encoder->bm_rep1[state] ); + price += price_bit( encoder->bm_rep2[state], rep - 2 ); + } + return price; + } + +static inline int LZe_price_dis( const struct LZ_encoder * const encoder, + const int dis, const int dis_state ) + { + if( dis < modeled_distances ) + return encoder->dis_prices[dis_state][dis]; + else + return encoder->dis_slot_prices[dis_state][get_slot( dis )] + + encoder->align_prices[dis & (dis_align_size - 1)]; + } + +static inline int LZe_price_pair( const struct LZ_encoder * const encoder, + const int dis, const int len, + const int pos_state ) + { + if( len <= min_match_len && dis >= modeled_distances ) + return infinite_price; + return Lee_price( &encoder->len_encoder, len, pos_state ) + + LZe_price_dis( encoder, dis, get_dis_state( len ) ); + } + +static inline void LZe_encode_pair( struct LZ_encoder * const encoder, + const uint32_t dis, const int len, + const int pos_state ) + { + const int dis_slot = get_slot( dis ); + Lee_encode( &encoder->len_encoder, &encoder->range_encoder, len, pos_state ); + Re_encode_tree( &encoder->range_encoder, + encoder->bm_dis_slot[get_dis_state(len)], + dis_slot, dis_slot_bits ); + + if( dis_slot >= start_dis_model ) + { + const int direct_bits = ( dis_slot >> 1 ) - 1; + const uint32_t base = ( 2 | ( dis_slot & 1 ) ) << direct_bits; + const uint32_t direct_dis = dis - base; + + if( dis_slot < end_dis_model ) + Re_encode_tree_reversed( &encoder->range_encoder, + encoder->bm_dis + base - dis_slot, + direct_dis, direct_bits ); + else + { + Re_encode( &encoder->range_encoder, direct_dis >> dis_align_bits, + direct_bits - dis_align_bits ); + Re_encode_tree_reversed( &encoder->range_encoder, encoder->bm_align, + direct_dis, dis_align_bits ); + if( --encoder->align_price_count <= 0 ) LZe_fill_align_prices( encoder ); + } + } + } + + /* End Of Stream mark => (dis == 0xFFFFFFFFU, len == min_match_len) */ +static bool LZe_full_flush( struct LZ_encoder * const encoder, const State state ) + { + int i; + const int pos_state = Mf_data_position( encoder->matchfinder ) & pos_state_mask; + File_trailer trailer; + if( encoder->member_finished || + Cb_free_bytes( &encoder->range_encoder.cb ) < Ft_size + max_marker_size ) + return false; + Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[state][pos_state], 1 ); + Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep[state], 0 ); + LZe_encode_pair( encoder, 0xFFFFFFFFU, min_match_len, pos_state ); + Re_flush( &encoder->range_encoder ); + Ft_set_data_crc( trailer, LZe_crc( encoder ) ); + Ft_set_data_size( trailer, Mf_data_position( encoder->matchfinder ) ); + Ft_set_member_size( trailer, Re_member_position( &encoder->range_encoder ) + + Ft_size ); + for( i = 0; i < Ft_size; ++i ) + Cb_put_byte( &encoder->range_encoder.cb, trailer[i] ); + return true; + } + + + /* Sync Flush mark => (dis == 0xFFFFFFFFU, len == min_match_len + 1) */ +static bool LZe_sync_flush( struct LZ_encoder * const encoder ) + { + const int pos_state = Mf_data_position( encoder->matchfinder ) & pos_state_mask; + const State state = encoder->state; + if( encoder->member_finished || + Cb_free_bytes( &encoder->range_encoder.cb ) < max_marker_size ) + return false; + Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[state][pos_state], 1 ); + Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep[state], 0 ); + LZe_encode_pair( encoder, 0xFFFFFFFFU, min_match_len + 1, pos_state ); + Re_flush( &encoder->range_encoder ); + return true; + } + + +static inline int LZe_read_match_distances( struct LZ_encoder * const encoder ) + { + int len = Mf_longest_match_len( encoder->matchfinder, + encoder->match_distances ); + if( len == encoder->matchfinder->match_len_limit && len < max_match_len ) + len += Mf_true_match_len( encoder->matchfinder, len, + encoder->match_distances[len] + 1, + max_match_len - len ); + return len; + } + +static inline bool LZe_move_pos( struct LZ_encoder * const encoder, int n ) + { + if( --n >= 0 && !Mf_move_pos( encoder->matchfinder ) ) return false; + while( --n >= 0 ) + { + Mf_longest_match_len( encoder->matchfinder, 0 ); + if( !Mf_move_pos( encoder->matchfinder ) ) return false; + } + return true; + } + +static inline void LZe_backward( struct LZ_encoder * const encoder, int cur ) + { + int * const dis = &encoder->trials[cur].dis; + while( cur > 0 ) + { + const int prev_index = encoder->trials[cur].prev_index; + struct Trial * const prev_trial = &encoder->trials[prev_index]; + prev_trial->price = cur - prev_index; /* len */ + cur = *dis; *dis = prev_trial->dis; prev_trial->dis = cur; + cur = prev_index; + } + } + + +/* Return value == number of bytes advanced (ahead). + trials[0]..trials[retval-1] contain the steps to encode. + ( trials[0].dis == -1 && trials[0].price == 1 ) means literal. +*/ +static int LZe_sequence_optimizer( struct LZ_encoder * const encoder, + const int reps[num_rep_distances], + const State state ) + { + int main_len, i, rep, cur = 0, num_trials; + int replens[num_rep_distances]; + int rep_index = 0; + + if( encoder->longest_match_found > 0 ) /* from previous call */ + { + main_len = encoder->longest_match_found; + encoder->longest_match_found = 0; + } + else main_len = LZe_read_match_distances( encoder ); + + for( i = 0; i < num_rep_distances; ++i ) + { + replens[i] = + Mf_true_match_len( encoder->matchfinder, 0, reps[i] + 1, max_match_len ); + if( replens[i] > replens[rep_index] ) rep_index = i; + } + if( replens[rep_index] >= encoder->matchfinder->match_len_limit ) + { + encoder->trials[0].dis = rep_index; + encoder->trials[0].price = replens[rep_index]; + if( !LZe_move_pos( encoder, replens[rep_index] ) ) return 0; + return replens[rep_index]; + } + + if( main_len >= encoder->matchfinder->match_len_limit ) + { + encoder->trials[0].dis = + encoder->match_distances[encoder->matchfinder->match_len_limit] + + num_rep_distances; + encoder->trials[0].price = main_len; + if( !LZe_move_pos( encoder, main_len ) ) return 0; + return main_len; + } + + { + const int pos_state = Mf_data_position( encoder->matchfinder ) & pos_state_mask; + const int match_price = price1( encoder->bm_match[state][pos_state] ); + const int rep_match_price = match_price + price1( encoder->bm_rep[state] ); + const uint8_t prev_byte = Mf_peek( encoder->matchfinder, -1 ); + const uint8_t cur_byte = Mf_peek( encoder->matchfinder, 0 ); + const uint8_t match_byte = Mf_peek( encoder->matchfinder, -reps[0]-1 ); + + encoder->trials[0].state = state; + for( i = 0; i < num_rep_distances; ++i ) + encoder->trials[0].reps[i] = reps[i]; + encoder->trials[1].dis = -1; + encoder->trials[1].prev_index = 0; + encoder->trials[1].price = price0( encoder->bm_match[state][pos_state] ); + if( St_is_char( state ) ) + encoder->trials[1].price += + Lie_price_symbol( &encoder->literal_encoder, prev_byte, cur_byte ); + else + encoder->trials[1].price += + Lie_price_matched( &encoder->literal_encoder, prev_byte, cur_byte, match_byte ); + + if( match_byte == cur_byte ) + Tr_update( &encoder->trials[1], 0, 0, rep_match_price + + LZe_price_rep_len1( encoder, state, pos_state ) ); + + if( main_len < min_match_len ) + { + encoder->trials[0].dis = encoder->trials[1].dis; + encoder->trials[0].price = 1; + if( !Mf_move_pos( encoder->matchfinder ) ) return 0; + return 1; + } + + if( main_len <= replens[rep_index] ) + { + int len; + main_len = replens[rep_index]; + for( len = min_match_len; len <= main_len; ++len ) + encoder->trials[len].price = infinite_price; + } + else + { + int len; + const int normal_match_price = match_price + price0( encoder->bm_rep[state] ); + for( len = min_match_len; len <= main_len; ++len ) + { + encoder->trials[len].dis = encoder->match_distances[len] + num_rep_distances; + encoder->trials[len].prev_index = 0; + encoder->trials[len].price = normal_match_price + + LZe_price_pair( encoder, encoder->match_distances[len], len, pos_state ); + } + } + + for( rep = 0; rep < num_rep_distances; ++rep ) + { + const int price = rep_match_price + + LZe_price_rep( encoder, rep, state, pos_state ); + int len; + for( len = min_match_len; len <= replens[rep]; ++len ) + Tr_update( &encoder->trials[len], rep, 0, price + + Lee_price( &encoder->rep_match_len_encoder, len, pos_state ) ); + } + } + + num_trials = main_len; + if( !Mf_move_pos( encoder->matchfinder ) ) return 0; + + while( true ) + { + struct Trial *cur_trial, *next_trial; + int newlen, pos_state, prev_index, len_limit; + int next_price, match_price, rep_match_price; + uint8_t prev_byte, cur_byte, match_byte; + + if( ++cur >= num_trials ) /* no more initialized trials */ + { + LZe_backward( encoder, cur ); + return cur; + } + newlen = LZe_read_match_distances( encoder ); + if( newlen >= encoder->matchfinder->match_len_limit ) + { + encoder->longest_match_found = newlen; + LZe_backward( encoder, cur ); + return cur; + } + + cur_trial = &encoder->trials[cur]; + prev_index = cur_trial->prev_index; + + cur_trial->state = encoder->trials[prev_index].state; + + for( i = 0; i < num_rep_distances; ++i ) + cur_trial->reps[i] = encoder->trials[prev_index].reps[i]; + + if( prev_index == cur - 1 ) + { + if( cur_trial->dis == 0 ) St_set_short_rep( &cur_trial->state ); + else St_set_char( &cur_trial->state ); + } + else + { + if( cur_trial->dis < num_rep_distances ) St_set_rep( &cur_trial->state ); + else St_set_match( &cur_trial->state ); + LZe_mtf_reps( cur_trial->dis, cur_trial->reps ); + } + + pos_state = Mf_data_position( encoder->matchfinder ) & pos_state_mask; + prev_byte = Mf_peek( encoder->matchfinder, -1 ); + cur_byte = Mf_peek( encoder->matchfinder, 0 ); + match_byte = Mf_peek( encoder->matchfinder, -cur_trial->reps[0]-1 ); + + next_price = cur_trial->price + + price0( encoder->bm_match[cur_trial->state][pos_state] ); + if( St_is_char( cur_trial->state ) ) + next_price += Lie_price_symbol( &encoder->literal_encoder, + prev_byte, cur_byte ); + else + next_price += Lie_price_matched( &encoder->literal_encoder, + prev_byte, cur_byte, match_byte ); + if( !Mf_move_pos( encoder->matchfinder ) ) return 0; + + next_trial = &encoder->trials[cur+1]; + + Tr_update( next_trial, -1, cur, next_price ); + + match_price = cur_trial->price + price1( encoder->bm_match[cur_trial->state][pos_state] ); + rep_match_price = match_price + price1( encoder->bm_rep[cur_trial->state] ); + + if( match_byte == cur_byte && next_trial->dis != 0 ) + Tr_update( next_trial, 0, cur, rep_match_price + + LZe_price_rep_len1( encoder, cur_trial->state, pos_state ) ); + + len_limit = min( min( max_num_trials - 1 - cur, + Mf_available_bytes( encoder->matchfinder ) ), + encoder->matchfinder->match_len_limit ); + if( len_limit < min_match_len ) continue; + + for( rep = 0; rep < num_rep_distances; ++rep ) + { + const int dis = cur_trial->reps[rep] + 1; + int len = 0; + const uint8_t * const data = Mf_ptr_to_current_pos( encoder->matchfinder ) - 1; + while( len < len_limit && data[len] == data[len-dis] ) ++len; + if( len >= min_match_len ) + { + const int price = rep_match_price + + LZe_price_rep( encoder, rep, cur_trial->state, pos_state ); + while( num_trials < cur + len ) + encoder->trials[++num_trials].price = infinite_price; + for( ; len >= min_match_len; --len ) + Tr_update( &encoder->trials[cur+len], rep, cur, price + + Lee_price( &encoder->rep_match_len_encoder, len, pos_state ) ); + } + } + + if( newlen <= len_limit && + ( newlen > min_match_len || + ( newlen == min_match_len && + encoder->match_distances[min_match_len] < modeled_distances ) ) ) + { + const int normal_match_price = match_price + + price0( encoder->bm_rep[cur_trial->state] ); + int len; + int dis = encoder->match_distances[min_match_len]; + int dis_state = get_dis_state( min_match_len ); + int dis_price = infinite_price; + + while( num_trials < cur + newlen ) + encoder->trials[++num_trials].price = infinite_price; + + if( dis < modeled_distances ) + Tr_update( &encoder->trials[cur+min_match_len], dis + num_rep_distances, + cur, normal_match_price + encoder->dis_prices[dis_state][dis] + + Lee_price( &encoder->len_encoder, min_match_len, pos_state ) ); + + for( len = min_match_len + 1; len <= newlen; ++len ) + { + if( dis != encoder->match_distances[len] || + dis_state < max_dis_states - 1 ) + { + dis = encoder->match_distances[len]; + dis_state = get_dis_state( len ); + dis_price = LZe_price_dis( encoder, dis, dis_state ); + } + Tr_update( &encoder->trials[cur+len], dis + num_rep_distances, cur, + normal_match_price + dis_price + + Lee_price( &encoder->len_encoder, len, pos_state ) ); + } + } + } + } + + +static bool LZe_encode_member( struct LZ_encoder * const encoder, + const bool finish ) + { + const int fill_count = + ( encoder->matchfinder->match_len_limit > 12 ) ? 512 : 2048; + int ahead, i; + State * const state = &encoder->state; + if( encoder->member_finished ) return true; + if( Re_member_position( &encoder->range_encoder ) >= encoder->member_size_limit ) + { + if( LZe_full_flush( encoder, *state ) ) encoder->member_finished = true; + return true; + } + + /* encode first byte */ + if( Mf_data_position( encoder->matchfinder ) == 0 && + !Mf_finished( encoder->matchfinder ) ) + { + if( Mf_available_bytes( encoder->matchfinder ) < max_match_len && + !encoder->matchfinder->at_stream_end ) + return true; + const uint8_t prev_byte = 0; + const uint8_t cur_byte = Mf_peek( encoder->matchfinder, 0 ); + Re_encode_bit( &encoder->range_encoder, &encoder->bm_match[*state][0], 0 ); + Lie_encode( &encoder->literal_encoder, &encoder->range_encoder, prev_byte, cur_byte ); + CRC32_update_byte( &encoder->crc, cur_byte ); + Mf_longest_match_len( encoder->matchfinder, 0 ); + if( !Mf_move_pos( encoder->matchfinder ) ) return false; + } + + while( !Mf_finished( encoder->matchfinder ) ) + { + if( !Mf_enough_available_bytes( encoder->matchfinder ) || + !Re_enough_free_bytes( &encoder->range_encoder ) ) return true; + if( encoder->fill_counter <= 0 ) + { LZe_fill_distance_prices( encoder ); encoder->fill_counter = fill_count; } + + ahead = LZe_sequence_optimizer( encoder, encoder->rep_distances, *state ); + if( ahead <= 0 ) return false; /* can't happen */ + encoder->fill_counter -= ahead; + + for( i = 0; ; ) + { + const int pos_state = + ( Mf_data_position( encoder->matchfinder ) - ahead ) & pos_state_mask; + const int dis = encoder->trials[i].dis; + const int len = encoder->trials[i].price; + + bool bit = ( dis < 0 && len == 1 ); + Re_encode_bit( &encoder->range_encoder, + &encoder->bm_match[*state][pos_state], !bit ); + if( bit ) /* literal byte */ + { + const uint8_t prev_byte = Mf_peek( encoder->matchfinder, -ahead-1 ); + const uint8_t cur_byte = Mf_peek( encoder->matchfinder, -ahead ); + CRC32_update_byte( &encoder->crc, cur_byte ); + if( St_is_char( *state ) ) + Lie_encode( &encoder->literal_encoder, &encoder->range_encoder, + prev_byte, cur_byte ); + else + { + const uint8_t match_byte = + Mf_peek( encoder->matchfinder, -ahead-encoder->rep_distances[0]-1 ); + Lie_encode_matched( &encoder->literal_encoder, &encoder->range_encoder, + prev_byte, cur_byte, match_byte ); + } + St_set_char( state ); + } + else /* match or repeated match */ + { + CRC32_update_buf( &encoder->crc, Mf_ptr_to_current_pos( encoder->matchfinder ) - ahead, len ); + LZe_mtf_reps( dis, encoder->rep_distances ); + bit = ( dis < num_rep_distances ); + Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep[*state], bit ); + if( bit ) + { + bit = ( dis == 0 ); + Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep0[*state], !bit ); + if( bit ) + Re_encode_bit( &encoder->range_encoder, &encoder->bm_len[*state][pos_state], len > 1 ); + else + { + Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep1[*state], dis > 1 ); + if( dis > 1 ) + Re_encode_bit( &encoder->range_encoder, &encoder->bm_rep2[*state], dis > 2 ); + } + if( len == 1 ) St_set_short_rep( state ); + else + { + Lee_encode( &encoder->rep_match_len_encoder, &encoder->range_encoder, len, pos_state ); + St_set_rep( state ); + } + } + else + { + LZe_encode_pair( encoder, dis - num_rep_distances, len, pos_state ); + St_set_match( state ); + } + } + ahead -= len; i += len; + if( Re_member_position( &encoder->range_encoder ) >= encoder->member_size_limit ) + { + if( !Mf_dec_pos( encoder->matchfinder, ahead ) ) return false; + if( LZe_full_flush( encoder, *state ) ) encoder->member_finished = true; + return true; + } + if( ahead <= 0 ) break; + } + } + if( finish && LZe_full_flush( encoder, *state ) ) + encoder->member_finished = true; + return true; + } diff --git a/encoder.cc b/encoder.cc deleted file mode 100644 index fb7e65e..0000000 --- a/encoder.cc +++ /dev/null @@ -1,663 +0,0 @@ -/* Lzlib - A compression library for lzip files - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. - - This library is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this library. If not, see <http://www.gnu.org/licenses/>. - - As a special exception, you may use this file as part of a free - software library without restriction. Specifically, if other files - instantiate templates or use macros or inline functions from this - file, or you compile this file and link it with other files to - produce an executable, this file does not by itself cause the - resulting executable to be covered by the GNU General Public - License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General - Public License. -*/ - -#define _FILE_OFFSET_BITS 64 - -#include <algorithm> -#include <cerrno> -#include <cstdlib> -#include <cstring> -#include <stdint.h> - -#include "lzlib.h" -#include "lzip.h" -#include "encoder.h" - - -namespace Lzlib { - -const Dis_slots dis_slots; -const Prob_prices prob_prices; - - -int Matchfinder::write_data( const uint8_t * const in_buffer, const int in_size ) throw() - { - if( at_stream_end_ || in_size < 0 ) return 0; - const int size = std::min( buffer_size - stream_pos, in_size ); - if( size > 0 ) - { - std::memcpy( buffer + stream_pos, in_buffer, size ); - stream_pos += size; - } - return size; - } - - -Matchfinder::Matchfinder( const int dict_size, const int len_limit ) - : - partial_data_pos( 0 ), - dictionary_size_( dict_size ), - buffer_size( ( 2 * std::max( 65536, dictionary_size_ ) ) + - before_size + after_size ), - buffer( new( std::nothrow ) uint8_t[buffer_size] ), - prev_positions( new( std::nothrow ) int32_t[num_prev_positions] ), - prev_pos_tree( new( std::nothrow ) int32_t[2*dictionary_size_] ), - pos( 0 ), - cyclic_pos( 0 ), - stream_pos( 0 ), - pos_limit( buffer_size - after_size ), - match_len_limit_( len_limit ), - cycles( ( len_limit < max_match_len ) ? 16 + ( len_limit / 2 ) : 256 ), - at_stream_end_( false ), - been_flushed( false ) - { - if( !buffer || !prev_positions || !prev_pos_tree ) - { - if( prev_pos_tree ) delete[] prev_pos_tree; - if( prev_positions ) delete[] prev_positions; - if( buffer ) delete[] buffer; - throw std::bad_alloc(); - } - for( int i = 0; i < num_prev_positions; ++i ) prev_positions[i] = -1; - } - - -void Matchfinder::reset() throw() - { - const int size = stream_pos - pos; - if( size > 0 ) std::memmove( buffer, buffer + pos, size ); - partial_data_pos = 0; - stream_pos -= pos; - pos = 0; - cyclic_pos = 0; - at_stream_end_ = false; - been_flushed = false; - for( int i = 0; i < num_prev_positions; ++i ) prev_positions[i] = -1; - } - - -bool Matchfinder::move_pos() throw() - { - if( ++cyclic_pos >= dictionary_size_ ) cyclic_pos = 0; - if( ++pos >= pos_limit ) - { - if( pos > stream_pos ) - { pos = stream_pos; return false; } - else - { - const int offset = pos - dictionary_size_ - before_size; - const int size = stream_pos - offset; - std::memmove( buffer, buffer + offset, size ); - partial_data_pos += offset; - pos -= offset; - stream_pos -= offset; - for( int i = 0; i < num_prev_positions; ++i ) - if( prev_positions[i] >= 0 ) prev_positions[i] -= offset; - for( int i = 0; i < 2 * dictionary_size_; ++i ) - if( prev_pos_tree[i] >= 0 ) prev_pos_tree[i] -= offset; - } - } - return true; - } - - -int Matchfinder::longest_match_len( int * const distances ) throw() - { - int32_t * ptr0 = prev_pos_tree + ( cyclic_pos << 1 ); - int32_t * ptr1 = ptr0 + 1; - int len_limit = match_len_limit_; - if( len_limit > available_bytes() ) - { - been_flushed = true; - len_limit = available_bytes(); - if( len_limit < 4 ) { *ptr0 = *ptr1 = -1; return 0; } - } - - int maxlen = min_match_len - 1; - const int min_pos = (pos >= dictionary_size_) ? - (pos - dictionary_size_ + 1) : 0; - const uint8_t * const data = buffer + pos; - const int key2 = num_prev_positions4 + num_prev_positions3 + - ( ( (int)data[0] << 8 ) | data[1] ); - const uint32_t tmp = crc32[data[0]] ^ data[1] ^ ( (uint32_t)data[2] << 8 ); - const int key3 = num_prev_positions4 + - (int)( tmp & ( num_prev_positions3 - 1 ) ); - const int key4 = (int)( ( tmp ^ ( crc32[data[3]] << 5 ) ) & - ( num_prev_positions4 - 1 ) ); - - if( distances ) - { - int np = prev_positions[key2]; - if( np >= min_pos ) - { distances[2] = pos - np - 1; maxlen = 2; } - else distances[2] = 0x7FFFFFFF; - np = prev_positions[key3]; - if( np >= min_pos && buffer[np] == data[0] ) - { distances[3] = pos - np - 1; maxlen = 3; } - else distances[3] = 0x7FFFFFFF; - distances[4] = 0x7FFFFFFF; - } - - prev_positions[key2] = pos; - prev_positions[key3] = pos; - int newpos = prev_positions[key4]; - prev_positions[key4] = pos; - - int len = 0, len0 = 0, len1 = 0; - - for( int count = cycles; ; ) - { - if( newpos < min_pos || --count < 0 ) { *ptr0 = *ptr1 = -1; break; } - const uint8_t * const newdata = buffer + newpos; - if( been_flushed ) len = 0; - while( len < len_limit && newdata[len] == data[len] ) ++len; - - const int delta = pos - newpos; - if( distances ) while( maxlen < len ) distances[++maxlen] = delta - 1; - - int32_t * const newptr = prev_pos_tree + - ( ( cyclic_pos - delta + - ( ( cyclic_pos >= delta ) ? 0 : dictionary_size_ ) ) << 1 ); - - if( len < len_limit ) - { - if( newdata[len] < data[len] ) - { - *ptr0 = newpos; - ptr0 = newptr + 1; - newpos = *ptr0; - len0 = len; if( len1 < len ) len = len1; - } - else - { - *ptr1 = newpos; - ptr1 = newptr; - newpos = *ptr1; - len1 = len; if( len0 < len ) len = len0; - } - } - else - { - *ptr0 = newptr[0]; - *ptr1 = newptr[1]; - break; - } - } - if( distances ) - { - if( distances[3] > distances[4] ) distances[3] = distances[4]; - if( distances[2] > distances[3] ) distances[2] = distances[3]; - } - return maxlen; - } - - -void Len_encoder::encode( Range_encoder & range_encoder, int symbol, - const int pos_state ) - { - symbol -= min_match_len; - if( symbol < len_low_symbols ) - { - range_encoder.encode_bit( choice1, 0 ); - range_encoder.encode_tree( bm_low[pos_state], symbol, len_low_bits ); - } - else - { - range_encoder.encode_bit( choice1, 1 ); - if( symbol < len_low_symbols + len_mid_symbols ) - { - range_encoder.encode_bit( choice2, 0 ); - range_encoder.encode_tree( bm_mid[pos_state], symbol - len_low_symbols, len_mid_bits ); - } - else - { - range_encoder.encode_bit( choice2, 1 ); - range_encoder.encode_tree( bm_high, symbol - len_low_symbols - len_mid_symbols, len_high_bits ); - } - } - if( --counters[pos_state] <= 0 ) update_prices( pos_state ); - } - - -void LZ_encoder::fill_align_prices() throw() - { - for( int i = 0; i < dis_align_size; ++i ) - align_prices[i] = price_symbol_reversed( bm_align, i, dis_align_bits ); - align_price_count = dis_align_size; - } - - -void LZ_encoder::fill_distance_prices() throw() - { - for( int dis = start_dis_model; dis < modeled_distances; ++dis ) - { - const int dis_slot = dis_slots.table( dis ); - const int direct_bits = ( dis_slot >> 1 ) - 1; - const int base = ( 2 | ( dis_slot & 1 ) ) << direct_bits; - const int price = - price_symbol_reversed( bm_dis + base - dis_slot, dis - base, direct_bits ); - for( int dis_state = 0; dis_state < max_dis_states; ++dis_state ) - dis_prices[dis_state][dis] = price; - } - - for( int dis_state = 0; dis_state < max_dis_states; ++dis_state ) - { - int * const dsp = dis_slot_prices[dis_state]; - const Bit_model * const bmds = bm_dis_slot[dis_state]; - int slot = 0; - for( ; slot < end_dis_model && slot < num_dis_slots; ++slot ) - dsp[slot] = price_symbol( bmds, slot, dis_slot_bits ); - for( ; slot < num_dis_slots; ++slot ) - dsp[slot] = price_symbol( bmds, slot, dis_slot_bits ) + - (((( slot >> 1 ) - 1 ) - dis_align_bits ) << price_shift ); - - int * const dp = dis_prices[dis_state]; - int dis = 0; - for( ; dis < start_dis_model; ++dis ) - dp[dis] = dsp[dis]; - for( ; dis < modeled_distances; ++dis ) - dp[dis] += dsp[dis_slots.table( dis )]; - } - } - - -// Return value == number of bytes advanced (ahead). -// trials[0]..trials[retval-1] contain the steps to encode. -// ( trials[0].dis == -1 && trials[0].price == 1 ) means literal. -int LZ_encoder::sequence_optimizer( const int reps[num_rep_distances], - const State & state ) - { - int main_len; - if( longest_match_found > 0 ) // from previous call - { - main_len = longest_match_found; - longest_match_found = 0; - } - else main_len = read_match_distances(); - - int replens[num_rep_distances]; - int rep_index = 0; - for( int i = 0; i < num_rep_distances; ++i ) - { - replens[i] = matchfinder.true_match_len( 0, reps[i] + 1, max_match_len ); - if( replens[i] > replens[rep_index] ) rep_index = i; - } - if( replens[rep_index] >= matchfinder.match_len_limit() ) - { - trials[0].dis = rep_index; - trials[0].price = replens[rep_index]; - if( !move_pos( replens[rep_index], true ) ) return 0; - return replens[rep_index]; - } - - if( main_len >= matchfinder.match_len_limit() ) - { - trials[0].dis = match_distances[matchfinder.match_len_limit()] + - num_rep_distances; - trials[0].price = main_len; - if( !move_pos( main_len, true ) ) return 0; - return main_len; - } - - { - const int pos_state = matchfinder.data_position() & pos_state_mask; - const uint8_t prev_byte = matchfinder[-1]; - const uint8_t cur_byte = matchfinder[0]; - const uint8_t match_byte = matchfinder[-reps[0]-1]; - - trials[0].state = state; - for( int i = 0; i < num_rep_distances; ++i ) trials[0].reps[i] = reps[i]; - trials[1].dis = -1; - trials[1].prev_index = 0; - trials[1].price = price0( bm_match[state()][pos_state] ); - if( state.is_char() ) - trials[1].price += literal_encoder.price_symbol( prev_byte, cur_byte ); - else - trials[1].price += literal_encoder.price_matched( prev_byte, cur_byte, match_byte ); - - const int match_price = price1( bm_match[state()][pos_state] ); - const int rep_match_price = match_price + price1( bm_rep[state()] ); - - if( match_byte == cur_byte ) - trials[1].update( 0, 0, rep_match_price + price_rep_len1( state, pos_state ) ); - - if( main_len < min_match_len ) - { - trials[0].dis = trials[1].dis; - trials[0].price = 1; - if( !matchfinder.move_pos() ) return 0; - return 1; - } - - if( main_len <= replens[rep_index] ) - { - main_len = replens[rep_index]; - for( int len = min_match_len; len <= main_len; ++len ) - trials[len].price = infinite_price; - } - else - { - const int normal_match_price = match_price + price0( bm_rep[state()] ); - for( int len = min_match_len; len <= main_len; ++len ) - { - trials[len].dis = match_distances[len] + num_rep_distances; - trials[len].prev_index = 0; - trials[len].price = normal_match_price + - price_pair( match_distances[len], len, pos_state ); - } - } - - for( int rep = 0; rep < num_rep_distances; ++rep ) - { - const int price = rep_match_price + - price_rep( rep, state, pos_state ); - for( int len = min_match_len; len <= replens[rep]; ++len ) - trials[len].update( rep, 0, price + - rep_match_len_encoder.price( len, pos_state ) ); - } - } - - int cur = 0; - int num_trials = main_len; - if( !matchfinder.move_pos() ) return 0; - - while( true ) - { - if( ++cur >= num_trials ) // no more initialized trials - { - backward( cur ); - return cur; - } - const int newlen = read_match_distances(); - if( newlen >= matchfinder.match_len_limit() ) - { - longest_match_found = newlen; - backward( cur ); - return cur; - } - - Trial & cur_trial = trials[cur]; - const int prev_index = cur_trial.prev_index; - - cur_trial.state = trials[prev_index].state; - - for( int i = 0; i < num_rep_distances; ++i ) - cur_trial.reps[i] = trials[prev_index].reps[i]; - if( prev_index == cur - 1 ) - { - if( cur_trial.dis == 0 ) cur_trial.state.set_short_rep(); - else cur_trial.state.set_char(); - } - else - { - if( cur_trial.dis < num_rep_distances ) cur_trial.state.set_rep(); - else cur_trial.state.set_match(); - mtf_reps( cur_trial.dis, cur_trial.reps ); - } - - const int pos_state = matchfinder.data_position() & pos_state_mask; - const uint8_t prev_byte = matchfinder[-1]; - const uint8_t cur_byte = matchfinder[0]; - const uint8_t match_byte = matchfinder[-cur_trial.reps[0]-1]; - - int next_price = cur_trial.price + - price0( bm_match[cur_trial.state()][pos_state] ); - if( cur_trial.state.is_char() ) - next_price += literal_encoder.price_symbol( prev_byte, cur_byte ); - else - next_price += literal_encoder.price_matched( prev_byte, cur_byte, match_byte ); - if( !matchfinder.move_pos() ) return 0; - - Trial & next_trial = trials[cur+1]; - - next_trial.update( -1, cur, next_price ); - - const int match_price = cur_trial.price + price1( bm_match[cur_trial.state()][pos_state] ); - const int rep_match_price = match_price + price1( bm_rep[cur_trial.state()] ); - - if( match_byte == cur_byte && next_trial.dis != 0 ) - next_trial.update( 0, cur, rep_match_price + - price_rep_len1( cur_trial.state, pos_state ) ); - - const int len_limit = std::min( std::min( max_num_trials - 1 - cur, - matchfinder.available_bytes() ), matchfinder.match_len_limit() ); - if( len_limit < min_match_len ) continue; - - for( int rep = 0; rep < num_rep_distances; ++rep ) - { - const int dis = cur_trial.reps[rep] + 1; - int len = 0; - const uint8_t * const data = matchfinder.ptr_to_current_pos() - 1; - while( len < len_limit && data[len] == data[len-dis] ) ++len; - if( len >= min_match_len ) - { - const int price = rep_match_price + - price_rep( rep, cur_trial.state, pos_state ); - while( num_trials < cur + len ) - trials[++num_trials].price = infinite_price; - for( ; len >= min_match_len; --len ) - trials[cur+len].update( rep, cur, price + - rep_match_len_encoder.price( len, pos_state ) ); - } - } - - if( newlen <= len_limit && - ( newlen > min_match_len || - ( newlen == min_match_len && - match_distances[min_match_len] < modeled_distances ) ) ) - { - const int normal_match_price = match_price + - price0( bm_rep[cur_trial.state()] ); - while( num_trials < cur + newlen ) - trials[++num_trials].price = infinite_price; - - int dis = match_distances[min_match_len]; - int dis_state = get_dis_state( min_match_len ); - int dis_price = infinite_price; - if( dis < modeled_distances ) - trials[cur+min_match_len].update( dis + num_rep_distances, cur, - normal_match_price + dis_prices[dis_state][dis] + - len_encoder.price( min_match_len, pos_state ) ); - for( int len = min_match_len + 1; len <= newlen; ++len ) - { - if( dis != match_distances[len] || dis_state < max_dis_states - 1 ) - { - dis = match_distances[len]; - dis_state = get_dis_state( len ); - dis_price = price_dis( dis, dis_state ); - } - trials[cur+len].update( dis + num_rep_distances, cur, - normal_match_price + dis_price + - len_encoder.price( len, pos_state ) ); - } - } - } - } - - - // End Of Stream mark => (dis == 0xFFFFFFFFU, len == min_match_len) -bool LZ_encoder::full_flush( const State & state ) - { - if( member_finished_ || - range_encoder.free_bytes() < File_trailer::size() + max_marker_size ) - return false; - const int pos_state = matchfinder.data_position() & pos_state_mask; - range_encoder.encode_bit( bm_match[state()][pos_state], 1 ); - range_encoder.encode_bit( bm_rep[state()], 0 ); - encode_pair( 0xFFFFFFFFU, min_match_len, pos_state ); - range_encoder.flush(); - File_trailer trailer; - trailer.data_crc( crc() ); - trailer.data_size( matchfinder.data_position() ); - trailer.member_size( range_encoder.member_position() + File_trailer::size() ); - for( int i = 0; i < File_trailer::size(); ++i ) - range_encoder.put_byte( trailer.data[i] ); - return true; - } - - - // Sync Flush mark => (dis == 0xFFFFFFFFU, len == min_match_len + 1) -bool LZ_encoder::sync_flush() - { - if( member_finished_ || range_encoder.free_bytes() < max_marker_size ) - return false; - const State & state = main_state; - const int pos_state = matchfinder.data_position() & pos_state_mask; - range_encoder.encode_bit( bm_match[state()][pos_state], 1 ); - range_encoder.encode_bit( bm_rep[state()], 0 ); - encode_pair( 0xFFFFFFFFU, min_match_len + 1, pos_state ); - range_encoder.flush(); - return true; - } - - -LZ_encoder::LZ_encoder( Matchfinder & mf, const File_header & header, - const long long member_size ) - : - member_size_limit( member_size - File_trailer::size() - max_marker_size ), - longest_match_found( 0 ), - crc_( 0xFFFFFFFFU ), - matchfinder( mf ), - len_encoder( matchfinder.match_len_limit() ), - rep_match_len_encoder( matchfinder.match_len_limit() ), - num_dis_slots( 2 * real_bits( matchfinder.dictionary_size() - 1 ) ), - fill_counter( 0 ), - member_finished_( false ) - { - for( int i = 0; i < num_rep_distances; ++i ) rep_distances[i] = 0; - fill_align_prices(); - - for( int i = 0; i < File_header::size; ++i ) - range_encoder.put_byte( header.data[i] ); - } - - -bool LZ_encoder::encode_member( const bool finish ) - { - const int fill_count = ( matchfinder.match_len_limit() > 12 ) ? 512 : 2048; - State & state = main_state; - if( member_finished_ ) return true; - if( range_encoder.member_position() >= member_size_limit ) - { if( full_flush( state ) ) { member_finished_ = true; } return true; } - - // encode first byte - if( matchfinder.data_position() == 0 && !matchfinder.finished() ) - { - if( matchfinder.available_bytes() < 4 && !matchfinder.at_stream_end() ) - return true; - const uint8_t prev_byte = 0; - const uint8_t cur_byte = matchfinder[0]; - range_encoder.encode_bit( bm_match[state()][0], 0 ); - literal_encoder.encode( range_encoder, prev_byte, cur_byte ); - crc32.update( crc_, cur_byte ); - if( !move_pos( 1 ) ) return false; - } - - while( true ) - { - if( matchfinder.finished() ) - { - if( finish && full_flush( state ) ) member_finished_ = true; - return true; - } - if( !matchfinder.enough_available_bytes() || - !range_encoder.enough_free_bytes() ) return true; - if( fill_counter <= 0 ) - { fill_distance_prices(); fill_counter = fill_count; } - - int ahead = sequence_optimizer( rep_distances, state ); - if( ahead <= 0 ) return false; - fill_counter -= ahead; - - for( int i = 0; ; ) - { - const int pos_state = - ( matchfinder.data_position() - ahead ) & pos_state_mask; - const int dis = trials[i].dis; - const int len = trials[i].price; - - bool bit = ( dis < 0 && len == 1 ); - range_encoder.encode_bit( bm_match[state()][pos_state], !bit ); - if( bit ) // literal byte - { - const uint8_t prev_byte = matchfinder[-ahead-1]; - const uint8_t cur_byte = matchfinder[-ahead]; - crc32.update( crc_, cur_byte ); - if( state.is_char() ) - literal_encoder.encode( range_encoder, prev_byte, cur_byte ); - else - { - const uint8_t match_byte = matchfinder[-ahead-rep_distances[0]-1]; - literal_encoder.encode_matched( range_encoder, - prev_byte, cur_byte, match_byte ); - } - state.set_char(); - } - else // match or repeated match - { - crc32.update( crc_, matchfinder.ptr_to_current_pos() - ahead, len ); - mtf_reps( dis, rep_distances ); - bit = ( dis < num_rep_distances ); - range_encoder.encode_bit( bm_rep[state()], bit ); - if( bit ) - { - bit = ( dis == 0 ); - range_encoder.encode_bit( bm_rep0[state()], !bit ); - if( bit ) - range_encoder.encode_bit( bm_len[state()][pos_state], len > 1 ); - else - { - range_encoder.encode_bit( bm_rep1[state()], dis > 1 ); - if( dis > 1 ) - range_encoder.encode_bit( bm_rep2[state()], dis > 2 ); - } - if( len == 1 ) state.set_short_rep(); - else - { - rep_match_len_encoder.encode( range_encoder, len, pos_state ); - state.set_rep(); - } - } - else - { - encode_pair( dis - num_rep_distances, len, pos_state ); - state.set_match(); - } - } - ahead -= len; i += len; - if( range_encoder.member_position() >= member_size_limit ) - { - if( !matchfinder.dec_pos( ahead ) ) return false; - if( full_flush( state ) ) member_finished_ = true; - return true; - } - if( ahead <= 0 ) break; - } - } - } - -} // end namespace Lzlib diff --git a/encoder.h b/encoder.h deleted file mode 100644 index 6d9036d..0000000 --- a/encoder.h +++ /dev/null @@ -1,620 +0,0 @@ -/* Lzlib - A compression library for lzip files - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. - - This library is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this library. If not, see <http://www.gnu.org/licenses/>. - - As a special exception, you may use this file as part of a free - software library without restriction. Specifically, if other files - instantiate templates or use macros or inline functions from this - file, or you compile this file and link it with other files to - produce an executable, this file does not by itself cause the - resulting executable to be covered by the GNU General Public - License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General - Public License. -*/ - -namespace Lzlib { - -enum { max_num_trials = 1 << 12, - price_shift = 6 }; - -class Dis_slots - { - unsigned char data[1<<12]; - -public: - Dis_slots() - { - for( int slot = 0; slot < 4; ++slot ) data[slot] = slot; - for( int i = 4, size = 2, slot = 4; slot < 24; slot += 2 ) - { - std::memset( &data[i], slot, size ); - std::memset( &data[i+size], slot + 1, size ); - size <<= 1; - i += size; - } - } - - unsigned char table( const int dis ) const throw() { return data[dis]; } - - int operator[]( const uint32_t dis ) const throw() - { - if( dis < (1 << 12) ) return data[dis]; - if( dis < (1 << 23) ) return data[dis>>11] + 22; - return data[dis>>22] + 44; - } - }; - -extern const Dis_slots dis_slots; - - -class Prob_prices - { - int data[bit_model_total >> 2]; - -public: - Prob_prices() - { - const int num_bits = ( bit_model_total_bits - 2 ); - int j = 1, end = 2; - data[0] = bit_model_total_bits << price_shift; - for( int i = num_bits - 1; i >= 0; --i, end <<= 1 ) - { - for( ; j < end; ++j ) - data[j] = ( i << price_shift ) + - ( ( (end - j) << price_shift ) >> ( num_bits - i - 1 ) ); - } - } - - int operator[]( const int probability ) const throw() - { return data[probability >> 2]; } - }; - -extern const Prob_prices prob_prices; - - -inline int price0( const Bit_model & bm ) throw() - { return prob_prices[bm.probability]; } - -inline int price1( const Bit_model & bm ) throw() - { return prob_prices[bit_model_total-bm.probability]; } - -inline int price_bit( const Bit_model & bm, const int bit ) throw() - { if( bit ) return price1( bm ); else return price0( bm ); } - - -inline int price_symbol( const Bit_model bm[], int symbol, const int num_bits ) throw() - { - int price = 0; - symbol |= ( 1 << num_bits ); - while( symbol > 1 ) - { - const int bit = symbol & 1; - symbol >>= 1; - price += price_bit( bm[symbol], bit ); - } - return price; - } - - -inline int price_symbol_reversed( const Bit_model bm[], int symbol, - const int num_bits ) throw() - { - int price = 0; - int model = 1; - for( int i = num_bits; i > 0; --i ) - { - const int bit = symbol & 1; - symbol >>= 1; - price += price_bit( bm[model], bit ); - model = ( model << 1 ) | bit; - } - return price; - } - - -inline int price_matched( const Bit_model bm[], const int symbol, - const int match_byte ) throw() - { - int price = 0; - int model = 1; - - for( int i = 7; i >= 0; --i ) - { - const int match_bit = ( match_byte >> i ) & 1; - int bit = ( symbol >> i ) & 1; - price += price_bit( bm[(match_bit<<8)+model+0x100], bit ); - model = ( model << 1 ) | bit; - if( match_bit != bit ) - { - while( --i >= 0 ) - { - bit = ( symbol >> i ) & 1; - price += price_bit( bm[model], bit ); - model = ( model << 1 ) | bit; - } - break; - } - } - return price; - } - - -class Matchfinder - { - enum { // bytes to keep in buffer before dictionary - before_size = max_num_trials + 1, - // bytes to keep in buffer after pos - after_size = max_num_trials + max_match_len, - num_prev_positions4 = 1 << 20, - num_prev_positions3 = 1 << 18, - num_prev_positions2 = 1 << 16, - num_prev_positions = num_prev_positions4 + num_prev_positions3 + - num_prev_positions2 }; - - long long partial_data_pos; - const int dictionary_size_; // bytes to keep in buffer before pos - const int buffer_size; - uint8_t * const buffer; // input buffer - int32_t * const prev_positions; // last seen position of key - int32_t * const prev_pos_tree; - int pos; // current pos in buffer - int cyclic_pos; // current pos in dictionary - int stream_pos; // first byte not yet read from file - const int pos_limit; // when reached, a new block must be read - const int match_len_limit_; - const int cycles; - bool at_stream_end_; // stream_pos shows real end of file - bool been_flushed; - -public: - Matchfinder( const int dict_size, const int len_limit ); - - ~Matchfinder() - { delete[] prev_pos_tree; delete[] prev_positions; delete[] buffer; } - - uint8_t operator[]( const int i ) const throw() { return buffer[pos+i]; } - bool at_stream_end() const throw() { return at_stream_end_; } - int available_bytes() const throw() { return stream_pos - pos; } - long long data_position() const throw() { return partial_data_pos + pos; } - int dictionary_size() const throw() { return dictionary_size_; } - bool finished() const throw() { return at_stream_end_ && pos >= stream_pos; } - void flushing( const bool b ) throw() { at_stream_end_ = b; } - int free_bytes() const throw() - { if( at_stream_end_ ) return 0; return buffer_size - stream_pos; } - int match_len_limit() const throw() { return match_len_limit_; } - const uint8_t * ptr_to_current_pos() const throw() { return buffer + pos; } - - bool dec_pos( const int ahead ) throw() - { - if( ahead < 0 || pos < ahead ) return false; - pos -= ahead; - cyclic_pos -= ahead; - if( cyclic_pos < 0 ) cyclic_pos += dictionary_size_; - return true; - } - - bool enough_available_bytes() const throw() - { - return ( stream_pos > pos && - ( at_stream_end_ || stream_pos - pos >= after_size ) ); - } - - int true_match_len( const int index, const int distance, int len_limit ) const throw() - { - if( index + len_limit > available_bytes() ) - len_limit = available_bytes() - index; - const uint8_t * const data = buffer + pos + index - distance; - int i = 0; - while( i < len_limit && data[i] == data[i+distance] ) ++i; - return i; - } - - int write_data( const uint8_t * const in_buffer, const int in_size ) throw(); - void reset() throw(); - bool move_pos() throw(); - int longest_match_len( int * const distances = 0 ) throw(); - }; - - -class Range_encoder : public Circular_buffer - { - enum { min_free_bytes = 2 * max_num_trials }; - uint64_t low; - long long partial_member_pos; - uint32_t range; - int ff_count; - uint8_t cache; - - void shift_low() - { - const uint32_t carry = low >> 32; - if( low < 0xFF000000U || carry == 1 ) - { - put_byte( cache + carry ); - for( ; ff_count > 0; --ff_count ) put_byte( 0xFF + carry ); - cache = low >> 24; - } - else ++ff_count; - low = ( low & 0x00FFFFFFU ) << 8; - } - -public: - Range_encoder() - : - Circular_buffer( 65536 + min_free_bytes ), - low( 0 ), - partial_member_pos( 0 ), - range( 0xFFFFFFFFU ), - ff_count( 0 ), - cache( 0 ) {} - - long long member_position() const throw() - { return partial_member_pos + used_bytes() + ff_count; } - - bool enough_free_bytes() const throw() - { return free_bytes() >= min_free_bytes; } - - int read_data( uint8_t * const out_buffer, const int out_size ) throw() - { - const int size = Circular_buffer::read_data( out_buffer, out_size ); - if( size > 0 ) partial_member_pos += size; - return size; - } - - void flush() - { - for( int i = 0; i < 5; ++i ) shift_low(); - low = 0; - range = 0xFFFFFFFFU; - ff_count = 0; - cache = 0; - } - - void encode( const int symbol, const int num_bits ) - { - for( int i = num_bits - 1; i >= 0; --i ) - { - range >>= 1; - if( (symbol >> i) & 1 ) low += range; - if( range <= 0x00FFFFFFU ) { range <<= 8; shift_low(); } - } - } - - void encode_bit( Bit_model & bm, const int bit ) - { - const uint32_t bound = ( range >> bit_model_total_bits ) * bm.probability; - if( !bit ) - { - range = bound; - bm.probability += (bit_model_total - bm.probability) >> bit_model_move_bits; - } - else - { - low += bound; - range -= bound; - bm.probability -= bm.probability >> bit_model_move_bits; - } - if( range <= 0x00FFFFFFU ) { range <<= 8; shift_low(); } - } - - void encode_tree( Bit_model bm[], const int symbol, const int num_bits ) - { - int mask = ( 1 << ( num_bits - 1 ) ); - int model = 1; - for( int i = num_bits; i > 0; --i, mask >>= 1 ) - { - const int bit = ( symbol & mask ); - encode_bit( bm[model], bit ); - model <<= 1; - if( bit ) model |= 1; - } - } - - void encode_tree_reversed( Bit_model bm[], int symbol, const int num_bits ) - { - int model = 1; - for( int i = num_bits; i > 0; --i ) - { - const int bit = symbol & 1; - encode_bit( bm[model], bit ); - model = ( model << 1 ) | bit; - symbol >>= 1; - } - } - - void encode_matched( Bit_model bm[], int symbol, int match_byte ) - { - int model = 1; - for( int i = 7; i >= 0; --i ) - { - const int match_bit = ( match_byte >> i ) & 1; - int bit = ( symbol >> i ) & 1; - encode_bit( bm[(match_bit<<8)+model+0x100], bit ); - model = ( model << 1 ) | bit; - if( match_bit != bit ) - { - while( --i >= 0 ) - { - bit = ( symbol >> i ) & 1; - encode_bit( bm[model], bit ); - model = ( model << 1 ) | bit; - } - break; - } - } - } - }; - - -class Len_encoder - { - Bit_model choice1; - Bit_model choice2; - Bit_model bm_low[pos_states][len_low_symbols]; - Bit_model bm_mid[pos_states][len_mid_symbols]; - Bit_model bm_high[len_high_symbols]; - int prices[pos_states][max_len_symbols]; - const int len_symbols; - int counters[pos_states]; - - void update_prices( const int pos_state ) throw() - { - int * const pps = prices[pos_state]; - int tmp = price0( choice1 ); - int len = 0; - for( ; len < len_low_symbols && len < len_symbols; ++len ) - pps[len] = tmp + - price_symbol( bm_low[pos_state], len, len_low_bits ); - tmp = price1( choice1 ); - for( ; len < len_low_symbols + len_mid_symbols && len < len_symbols; ++len ) - pps[len] = tmp + price0( choice2 ) + - price_symbol( bm_mid[pos_state], len - len_low_symbols, len_mid_bits ); - for( ; len < len_symbols; ++len ) - // using 4 slots per value makes "price" faster - prices[3][len] = prices[2][len] = prices[1][len] = prices[0][len] = - tmp + price1( choice2 ) + - price_symbol( bm_high, len - len_low_symbols - len_mid_symbols, len_high_bits ); - counters[pos_state] = len_symbols; - } - -public: - Len_encoder( const int len_limit ) - : len_symbols( len_limit + 1 - min_match_len ) - { - for( int i = 0; i < pos_states; ++i ) update_prices( i ); - } - - void encode( Range_encoder & range_encoder, int symbol, - const int pos_state ); - - int price( const int symbol, const int pos_state ) const throw() - { return prices[pos_state][symbol - min_match_len]; } - }; - - -class Literal_encoder - { - Bit_model bm_literal[1<<literal_context_bits][0x300]; - - int lstate( const uint8_t prev_byte ) const throw() - { return ( prev_byte >> ( 8 - literal_context_bits ) ); } - -public: - void encode( Range_encoder & range_encoder, - uint8_t prev_byte, uint8_t symbol ) - { range_encoder.encode_tree( bm_literal[lstate(prev_byte)], symbol, 8 ); } - - void encode_matched( Range_encoder & range_encoder, - uint8_t prev_byte, uint8_t symbol, uint8_t match_byte ) - { range_encoder.encode_matched( bm_literal[lstate(prev_byte)], - symbol, match_byte ); } - - int price_symbol( uint8_t prev_byte, uint8_t symbol ) const throw() - { return Lzlib::price_symbol( bm_literal[lstate(prev_byte)], symbol, 8 ); } - - int price_matched( uint8_t prev_byte, uint8_t symbol, - uint8_t match_byte ) const throw() - { return Lzlib::price_matched( bm_literal[lstate(prev_byte)], - symbol, match_byte ); } - }; - - -class LZ_encoder - { - enum { infinite_price = 0x0FFFFFFF, - max_marker_size = 16, - num_rep_distances = 4 }; // must be 4 - - struct Trial - { - State state; - int dis; - int prev_index; // index of prev trial in trials[] - int price; // dual use var; cumulative price, match length - int reps[num_rep_distances]; - void update( const int d, const int p_i, const int pr ) throw() - { if( pr < price ) { dis = d; prev_index = p_i; price = pr; } } - }; - - const long long member_size_limit; - int longest_match_found; - uint32_t crc_; - - Bit_model bm_match[State::states][pos_states]; - Bit_model bm_rep[State::states]; - Bit_model bm_rep0[State::states]; - Bit_model bm_rep1[State::states]; - Bit_model bm_rep2[State::states]; - Bit_model bm_len[State::states][pos_states]; - Bit_model bm_dis_slot[max_dis_states][1<<dis_slot_bits]; - Bit_model bm_dis[modeled_distances-end_dis_model+1]; - Bit_model bm_align[dis_align_size]; - - Matchfinder & matchfinder; - Range_encoder range_encoder; - Len_encoder len_encoder; - Len_encoder rep_match_len_encoder; - Literal_encoder literal_encoder; - - const int num_dis_slots; - int rep_distances[num_rep_distances]; - int match_distances[max_match_len+1]; - Trial trials[max_num_trials]; - - int dis_slot_prices[max_dis_states][2*max_dictionary_bits]; - int dis_prices[max_dis_states][modeled_distances]; - int align_prices[dis_align_size]; - int align_price_count; - int fill_counter; - State main_state; - bool member_finished_; - - void fill_align_prices() throw(); - void fill_distance_prices() throw(); - - uint32_t crc() const throw() { return crc_ ^ 0xFFFFFFFFU; } - - // move-to-front dis in/into reps - void mtf_reps( const int dis, int reps[num_rep_distances] ) throw() - { - if( dis >= num_rep_distances ) - { - for( int i = num_rep_distances - 1; i > 0; --i ) reps[i] = reps[i-1]; - reps[0] = dis - num_rep_distances; - } - else if( dis > 0 ) - { - const int distance = reps[dis]; - for( int i = dis; i > 0; --i ) reps[i] = reps[i-1]; - reps[0] = distance; - } - } - - int price_rep_len1( const State & state, const int pos_state ) const throw() - { - return price0( bm_rep0[state()] ) + price0( bm_len[state()][pos_state] ); - } - - int price_rep( const int rep, const State & state, - const int pos_state ) const throw() - { - if( rep == 0 ) return price0( bm_rep0[state()] ) + - price1( bm_len[state()][pos_state] ); - int price = price1( bm_rep0[state()] ); - if( rep == 1 ) - price += price0( bm_rep1[state()] ); - else - { - price += price1( bm_rep1[state()] ); - price += price_bit( bm_rep2[state()], rep - 2 ); - } - return price; - } - - int price_dis( const int dis, const int dis_state ) const throw() - { - if( dis < modeled_distances ) - return dis_prices[dis_state][dis]; - else - return dis_slot_prices[dis_state][dis_slots[dis]] + - align_prices[dis & (dis_align_size - 1)]; - } - - int price_pair( const int dis, const int len, const int pos_state ) const throw() - { - if( len <= min_match_len && dis >= modeled_distances ) - return infinite_price; - return len_encoder.price( len, pos_state ) + - price_dis( dis, get_dis_state( len ) ); - } - - void encode_pair( const uint32_t dis, const int len, const int pos_state ) throw() - { - len_encoder.encode( range_encoder, len, pos_state ); - const int dis_slot = dis_slots[dis]; - range_encoder.encode_tree( bm_dis_slot[get_dis_state(len)], dis_slot, dis_slot_bits ); - - if( dis_slot >= start_dis_model ) - { - const int direct_bits = ( dis_slot >> 1 ) - 1; - const uint32_t base = ( 2 | ( dis_slot & 1 ) ) << direct_bits; - const uint32_t direct_dis = dis - base; - - if( dis_slot < end_dis_model ) - range_encoder.encode_tree_reversed( bm_dis + base - dis_slot, - direct_dis, direct_bits ); - else - { - range_encoder.encode( direct_dis >> dis_align_bits, direct_bits - dis_align_bits ); - range_encoder.encode_tree_reversed( bm_align, direct_dis, dis_align_bits ); - if( --align_price_count <= 0 ) fill_align_prices(); - } - } - } - - int read_match_distances() throw() - { - int len = matchfinder.longest_match_len( match_distances ); - if( len == matchfinder.match_len_limit() ) - len += matchfinder.true_match_len( len, match_distances[len] + 1, max_match_len - len ); - return len; - } - - bool move_pos( int n, bool skip = false ) throw() - { - while( --n >= 0 ) - { - if( skip ) skip = false; - else matchfinder.longest_match_len(); - if( !matchfinder.move_pos() ) return false; - } - return true; - } - - void backward( int cur ) - { - int & dis = trials[cur].dis; - while( cur > 0 ) - { - const int prev_index = trials[cur].prev_index; - Trial & prev_trial = trials[prev_index]; - prev_trial.price = cur - prev_index; // len - cur = dis; dis = prev_trial.dis; prev_trial.dis = cur; - cur = prev_index; - } - } - - int sequence_optimizer( const int reps[num_rep_distances], - const State & state ); - - bool full_flush( const State & state ); - -public: - LZ_encoder( Matchfinder & mf, const File_header & header, - const long long member_size ); - - bool encode_member( const bool finish ); - bool member_finished() const throw() - { return member_finished_ && !range_encoder.used_bytes(); } - int read_data( uint8_t * const buffer, const int size ) throw() - { return range_encoder.read_data( buffer, size ); } - bool sync_flush(); - - long long member_position() const throw() - { return range_encoder.member_position(); } - }; - -} // end namespace Lzlib @@ -1,5 +1,5 @@ /* Lzcheck - A test program for the lzlib library - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. + Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. This program is free software: you have unlimited permission to copy, distribute and modify it. @@ -14,11 +14,11 @@ #define _FILE_OFFSET_BITS 64 -#include <algorithm> -#include <cstdio> -#include <cstdlib> -#include <cstring> +#include <stdbool.h> #include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> #include <unistd.h> #include "lzlib.h" @@ -33,7 +33,11 @@ #define ULLONG_MAX 0xFFFFFFFFFFFFFFFFULL #endif -const int buffer_size = 32768; +#ifndef min + #define min(x,y) ((x) <= (y) ? (x) : (y)) +#endif + +enum { buffer_size = 32768 }; uint8_t in_buffer[buffer_size]; uint8_t mid_buffer[buffer_size]; uint8_t out_buffer[buffer_size]; @@ -43,53 +47,52 @@ int main( const int argc, const char * const argv[] ) { if( argc < 2 ) { - std::fprintf( stderr, "Usage: lzcheck filename.txt\n" ); + fprintf( stderr, "Usage: lzcheck filename.txt\n" ); return 1; } - FILE *file = std::fopen( argv[1], "rb" ); + FILE *file = fopen( argv[1], "rb" ); if( !file ) { - std::fprintf( stderr, - "lzcheck: Can't open file `%s' for reading\n", argv[1] ); + fprintf( stderr, "lzcheck: Can't open file '%s' for reading\n", argv[1] ); return 1; } -// std::fprintf( stderr, "lzcheck: Testing file `%s'\n", argv[1] ); +/* fprintf( stderr, "lzcheck: Testing file '%s'\n", argv[1] ); */ const int dictionary_size = 1 << 20; const int match_len_limit = 36; const long long member_size = LLONG_MAX; - LZ_Encoder * encoder = LZ_compress_open( dictionary_size, match_len_limit, - member_size ); + struct LZ_Encoder * const encoder = + LZ_compress_open( dictionary_size, match_len_limit, member_size ); if( !encoder || LZ_compress_errno( encoder ) != LZ_ok ) { const bool mem_error = ( LZ_compress_errno( encoder ) == LZ_mem_error ); LZ_compress_close( encoder ); if( mem_error ) { - std::fprintf( stderr, "lzcheck: Not enough memory.\n" ); + fprintf( stderr, "lzcheck: Not enough memory.\n" ); return 1; } - std::fprintf( stderr, - "lzcheck: internal error: Invalid argument to encoder.\n" ); + fprintf( stderr, "lzcheck: internal error: Invalid argument to encoder.\n" ); return 3; } - LZ_Decoder * decoder = LZ_decompress_open(); + struct LZ_Decoder * const decoder = LZ_decompress_open(); if( !decoder || LZ_decompress_errno( decoder ) != LZ_ok ) { LZ_decompress_close( decoder ); - std::fprintf( stderr, "lzcheck: Not enough memory.\n" ); + fprintf( stderr, "lzcheck: Not enough memory.\n" ); return 1; } int retval = 0; while( retval <= 1 ) { - const int read_size = std::fread( in_buffer, 1, buffer_size, file ); - if( read_size <= 0 ) break; // end of file + int i, l, r; + const int read_size = fread( in_buffer, 1, buffer_size, file ); + if( read_size <= 0 ) break; /* end of file */ - for( int l = 0, r = 1; r <= read_size; l = r, ++r ) + for( l = 0, r = 1; r <= read_size; l = r, ++r ) { while( r < read_size && in_buffer[r-1] != '\n' ) ++r; const int in_size = LZ_compress_write( encoder, in_buffer + l, r - l ); @@ -98,30 +101,30 @@ int main( const int argc, const char * const argv[] ) const int mid_size = LZ_compress_read( encoder, mid_buffer, buffer_size ); if( mid_size < 0 ) { - std::fprintf( stderr, "lzcheck: LZ_compress_read error: %s.\n", - LZ_strerror( LZ_compress_errno( encoder ) ) ); + fprintf( stderr, "lzcheck: LZ_compress_read error: %s.\n", + LZ_strerror( LZ_compress_errno( encoder ) ) ); retval = 3; break; } LZ_decompress_write( decoder, mid_buffer, mid_size ); const int out_size = LZ_decompress_read( decoder, out_buffer, buffer_size ); if( out_size < 0 ) { - std::fprintf( stderr, "lzcheck: LZ_decompress_read error: %s.\n", - LZ_strerror( LZ_decompress_errno( decoder ) ) ); + fprintf( stderr, "lzcheck: LZ_decompress_read error: %s.\n", + LZ_strerror( LZ_decompress_errno( decoder ) ) ); retval = 3; break; } - if( out_size != in_size || std::memcmp( in_buffer + l, out_buffer, out_size ) ) + if( out_size != in_size || memcmp( in_buffer + l, out_buffer, out_size ) ) { - std::fprintf( stderr, "lzcheck: Sync error at pos %d. in_size = %d, out_size = %d\n", - l, in_size, out_size ); - for( int i = 0; i < in_size; ++i ) - std::fputc( in_buffer[l+i], stderr ); + fprintf( stderr, "lzcheck: Sync error at pos %d. in_size = %d, out_size = %d\n", + l, in_size, out_size ); + for( i = 0; i < in_size; ++i ) + fputc( in_buffer[l+i], stderr ); if( in_buffer[l+in_size-1] != '\n' ) - std::fputc( '\n', stderr ); - for( int i = 0; i < out_size; ++i ) - std::fputc( out_buffer[i], stderr ); - std::fputc( '\n', stderr ); + fputc( '\n', stderr ); + for( i = 0; i < out_size; ++i ) + fputc( out_buffer[i], stderr ); + fputc( '\n', stderr ); retval = 1; } } @@ -129,39 +132,40 @@ int main( const int argc, const char * const argv[] ) if( retval <= 1 ) { - std::rewind( file ); + rewind( file ); if( LZ_compress_finish( encoder ) < 0 || LZ_decompress_write( decoder, mid_buffer, LZ_compress_read( encoder, mid_buffer, buffer_size ) ) < 0 || LZ_decompress_read( decoder, out_buffer, buffer_size ) != 0 || LZ_compress_restart_member( encoder, member_size ) < 0 ) { - std::fprintf( stderr, "lzcheck: Can't finish member: %s.\n", - LZ_strerror( LZ_decompress_errno( decoder ) ) ); + fprintf( stderr, "lzcheck: Can't finish member: %s.\n", + LZ_strerror( LZ_decompress_errno( decoder ) ) ); retval = 3; } } while( retval <= 1 ) { - const int read_size = std::fread( in_buffer, 1, buffer_size / 2, file ); - if( read_size <= 0 ) break; // end of file + int i, l, r; + const int read_size = fread( in_buffer, 1, buffer_size / 2, file ); + if( read_size <= 0 ) break; /* end of file */ - for( int l = 0, r = 1; r <= read_size; l = r, ++r ) + for( l = 0, r = 1; r <= read_size; l = r, ++r ) { while( r < read_size && in_buffer[r-1] != '\n' ) ++r; - const int leading_garbage = (l == 0) ? std::min( r, read_size / 2 ) : 0; + const int leading_garbage = (l == 0) ? min( r, read_size / 2 ) : 0; const int in_size = LZ_compress_write( encoder, in_buffer + l, r - l ); if( in_size < r - l ) r = l + in_size; LZ_compress_sync_flush( encoder ); if( leading_garbage ) - std::memset( mid_buffer, in_buffer[0], leading_garbage ); + memset( mid_buffer, in_buffer[0], leading_garbage ); const int mid_size = LZ_compress_read( encoder, mid_buffer + leading_garbage, buffer_size - leading_garbage ); if( mid_size < 0 ) { - std::fprintf( stderr, "lzcheck: LZ_compress_read error: %s.\n", - LZ_strerror( LZ_compress_errno( encoder ) ) ); + fprintf( stderr, "lzcheck: LZ_compress_read error: %s.\n", + LZ_strerror( LZ_compress_errno( encoder ) ) ); retval = 3; break; } LZ_decompress_write( decoder, mid_buffer, mid_size + leading_garbage ); @@ -171,28 +175,28 @@ int main( const int argc, const char * const argv[] ) if( LZ_decompress_errno( decoder ) == LZ_header_error || LZ_decompress_errno( decoder ) == LZ_data_error ) { - LZ_decompress_sync_to_member( decoder ); // remove leading garbage + LZ_decompress_sync_to_member( decoder ); /* remove leading garbage */ out_size = LZ_decompress_read( decoder, out_buffer, buffer_size ); } if( out_size < 0 ) { - std::fprintf( stderr, "lzcheck: LZ_decompress_read error: %s.\n", - LZ_strerror( LZ_decompress_errno( decoder ) ) ); + fprintf( stderr, "lzcheck: LZ_decompress_read error: %s.\n", + LZ_strerror( LZ_decompress_errno( decoder ) ) ); retval = 3; break; } } - if( out_size != in_size || std::memcmp( in_buffer + l, out_buffer, out_size ) ) + if( out_size != in_size || memcmp( in_buffer + l, out_buffer, out_size ) ) { - std::fprintf( stderr, "lzcheck: Sync error at pos %d. in_size = %d, out_size = %d\n", - l, in_size, out_size ); - for( int i = 0; i < in_size; ++i ) - std::fputc( in_buffer[l+i], stderr ); + fprintf( stderr, "lzcheck: Sync error at pos %d. in_size = %d, out_size = %d\n", + l, in_size, out_size ); + for( i = 0; i < in_size; ++i ) + fputc( in_buffer[l+i], stderr ); if( in_buffer[l+in_size-1] != '\n' ) - std::fputc( '\n', stderr ); - for( int i = 0; i < out_size; ++i ) - std::fputc( out_buffer[i], stderr ); - std::fputc( '\n', stderr ); + fputc( '\n', stderr ); + for( i = 0; i < out_size; ++i ) + fputc( out_buffer[i], stderr ); + fputc( '\n', stderr ); retval = 1; } } @@ -204,12 +208,12 @@ int main( const int argc, const char * const argv[] ) LZ_decompress_reset( decoder ) < 0 || LZ_compress_restart_member( encoder, member_size ) < 0 ) { - std::fprintf( stderr, "lzcheck: Can't restart member: %s.\n", - LZ_strerror( LZ_decompress_errno( decoder ) ) ); + fprintf( stderr, "lzcheck: Can't restart member: %s.\n", + LZ_strerror( LZ_decompress_errno( decoder ) ) ); retval = 3; break; } - const int size = std::min( 100, read_size ); + const int size = min( 100, read_size ); if( LZ_compress_write( encoder, in_buffer, size ) != size || LZ_compress_finish( encoder ) < 0 || LZ_decompress_write( decoder, mid_buffer, LZ_compress_read( encoder, mid_buffer, buffer_size ) ) < 0 || @@ -217,14 +221,14 @@ int main( const int argc, const char * const argv[] ) LZ_decompress_sync_to_member( decoder ) < 0 || LZ_compress_restart_member( encoder, member_size ) < 0 ) { - std::fprintf( stderr, "lzcheck: Can't seek to next member: %s.\n", - LZ_strerror( LZ_decompress_errno( decoder ) ) ); + fprintf( stderr, "lzcheck: Can't seek to next member: %s.\n", + LZ_strerror( LZ_decompress_errno( decoder ) ) ); retval = 3; break; } } LZ_decompress_close( decoder ); LZ_compress_close( encoder ); - std::fclose( file ); + fclose( file ); return retval; } @@ -1,297 +0,0 @@ -/* Lzlib - A compression library for lzip files - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. - - This library is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this library. If not, see <http://www.gnu.org/licenses/>. - - As a special exception, you may use this file as part of a free - software library without restriction. Specifically, if other files - instantiate templates or use macros or inline functions from this - file, or you compile this file and link it with other files to - produce an executable, this file does not by itself cause the - resulting executable to be covered by the GNU General Public - License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General - Public License. -*/ - -namespace Lzlib { - -class State - { - unsigned char st; - -public: - enum { states = 12 }; - State() throw() : st( 0 ) {} - unsigned char operator()() const throw() { return st; } - bool is_char() const throw() { return st < 7; } - - void set_char() throw() - { - static const unsigned char next[states] = - { 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5 }; - st = next[st]; - } - - void set_match() throw() - { - static const unsigned char next[states] = - { 7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10 }; - st = next[st]; - } - - void set_rep() throw() - { - static const unsigned char next[states] = - { 8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11 }; - st = next[st]; - } - - void set_short_rep() throw() - { - static const unsigned char next[states] = - { 9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11 }; - st = next[st]; - } - }; - - -enum { - min_dictionary_bits = 12, - min_dictionary_size = 1 << min_dictionary_bits, - max_dictionary_bits = 29, - max_dictionary_size = 1 << max_dictionary_bits, - literal_context_bits = 3, - pos_state_bits = 2, - pos_states = 1 << pos_state_bits, - pos_state_mask = pos_states - 1, - - dis_slot_bits = 6, - start_dis_model = 4, - end_dis_model = 14, - modeled_distances = 1 << (end_dis_model / 2), - dis_align_bits = 4, - dis_align_size = 1 << dis_align_bits, - - len_low_bits = 3, - len_mid_bits = 3, - len_high_bits = 8, - len_low_symbols = 1 << len_low_bits, - len_mid_symbols = 1 << len_mid_bits, - len_high_symbols = 1 << len_high_bits, - max_len_symbols = len_low_symbols + len_mid_symbols + len_high_symbols, - - min_match_len = 2, // must be 2 - max_match_len = min_match_len + max_len_symbols - 1, // 273 - min_match_len_limit = 5, - - max_dis_states = 4 }; - -inline int get_dis_state( int len ) throw() - { - len -= min_match_len; - if( len >= max_dis_states ) len = max_dis_states - 1; - return len; - } - - -enum { bit_model_move_bits = 5, - bit_model_total_bits = 11, - bit_model_total = 1 << bit_model_total_bits }; - -struct Bit_model - { - unsigned int probability; - Bit_model() throw() : probability( bit_model_total / 2 ) {} - }; - - -class CRC32 - { - uint32_t data[256]; // Table of CRCs of all 8-bit messages. - -public: - CRC32() - { - for( unsigned int n = 0; n < 256; ++n ) - { - unsigned int c = n; - for( int k = 0; k < 8; ++k ) - { if( c & 1 ) c = 0xEDB88320U ^ ( c >> 1 ); else c >>= 1; } - data[n] = c; - } - } - - uint32_t operator[]( const uint8_t byte ) const throw() { return data[byte]; } - void update( uint32_t & crc, const uint8_t byte ) const throw() - { crc = data[(crc^byte)&0xFF] ^ ( crc >> 8 ); } - void update( uint32_t & crc, const uint8_t * const buffer, const int size ) const throw() - { - for( int i = 0; i < size; ++i ) - crc = data[(crc^buffer[i])&0xFF] ^ ( crc >> 8 ); - } - }; - -extern const CRC32 crc32; - - -inline int real_bits( const int value ) throw() - { - int bits = 0; - for( int i = 1, mask = 1; mask > 0; ++i, mask <<= 1 ) - if( value & mask ) bits = i; - return bits; - } - -const uint8_t magic_string[4] = { 'L', 'Z', 'I', 'P' }; - -struct File_header - { - uint8_t data[6]; // 0-3 magic bytes - // 4 version - // 5 coded_dict_size - enum { size = 6 }; - - void set_magic() throw() - { std::memcpy( data, magic_string, 4 ); data[4] = 1; } - - bool verify_magic() const throw() - { return ( std::memcmp( data, magic_string, 4 ) == 0 ); } - - uint8_t version() const throw() { return data[4]; } - bool verify_version() const throw() { return ( data[4] <= 1 ); } - - bool verify() const throw() - { - return ( verify_magic() && verify_version() && - dictionary_size() >= min_dictionary_size && - dictionary_size() <= max_dictionary_size ); - } - - int dictionary_size() const throw() - { - int sz = ( 1 << ( data[5] & 0x1F ) ); - if( sz > min_dictionary_size && sz <= max_dictionary_size ) - sz -= ( sz / 16 ) * ( ( data[5] >> 5 ) & 0x07 ); - return sz; - } - - bool dictionary_size( const int sz ) throw() - { - if( sz >= min_dictionary_size && sz <= max_dictionary_size ) - { - data[5] = real_bits( sz - 1 ); - if( sz > min_dictionary_size ) - { - const int base_size = 1 << data[5]; - const int wedge = base_size / 16; - for( int i = 7; i >= 1; --i ) - if( base_size - ( i * wedge ) >= sz ) - { data[5] |= ( i << 5 ); break; } - } - return true; - } - return false; - } - }; - - -struct File_trailer - { - uint8_t data[20]; // 0-3 CRC32 of the uncompressed data - // 4-11 size of the uncompressed data - // 12-19 member size including header and trailer - - static int size( const int version = 1 ) - { return ( ( version >= 1 ) ? 20 : 12 ); } - - uint32_t data_crc() const throw() - { - uint32_t tmp = 0; - for( int i = 3; i >= 0; --i ) { tmp <<= 8; tmp += data[i]; } - return tmp; - } - - void data_crc( uint32_t crc ) throw() - { for( int i = 0; i <= 3; ++i ) { data[i] = (uint8_t)crc; crc >>= 8; } } - - long long data_size() const throw() - { - long long tmp = 0; - for( int i = 11; i >= 4; --i ) { tmp <<= 8; tmp += data[i]; } - return tmp; - } - - void data_size( long long sz ) throw() - { - for( int i = 4; i <= 11; ++i ) { data[i] = (uint8_t)sz; sz >>= 8; } - } - - long long member_size() const throw() - { - long long tmp = 0; - for( int i = 19; i >= 12; --i ) { tmp <<= 8; tmp += data[i]; } - return tmp; - } - - void member_size( long long sz ) throw() - { - for( int i = 12; i <= 19; ++i ) { data[i] = (uint8_t)sz; sz >>= 8; } - } - }; - - -class Circular_buffer - { -protected: - const int buffer_size; // capacity == buffer_size - 1 - uint8_t * const buffer; - int get; // buffer is empty when get == put - int put; - - void reset() throw() { get = 0; put = 0; } - -public: - Circular_buffer( const int buf_size ) - : - buffer_size( buf_size + 1 ), - buffer( new uint8_t[buffer_size] ), - get( 0 ), - put( 0 ) {} - - ~Circular_buffer() { delete[] buffer; } - - int used_bytes() const throw() - { return ( (get <= put) ? 0 : buffer_size ) + put - get; } - int free_bytes() const throw() - { return ( (get <= put) ? buffer_size : 0 ) - put + get - 1; } - - uint8_t get_byte() throw() - { - const uint8_t b = buffer[get]; - if( ++get >= buffer_size ) get = 0; - return b; - } - - void put_byte( const uint8_t b ) throw() - { - buffer[put] = b; - if( ++put >= buffer_size ) put = 0; - } - - int read_data( uint8_t * const out_buffer, const int out_size ) throw(); - int write_data( const uint8_t * const in_buffer, const int in_size ) throw(); - }; - -} // end namespace Lzlib @@ -0,0 +1,543 @@ +/* Lzlib - A compression library for lzip files + Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. + + This library is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this library. If not, see <http://www.gnu.org/licenses/>. + + As a special exception, you may use this file as part of a free + software library without restriction. Specifically, if other files + instantiate templates or use macros or inline functions from this + file, or you compile this file and link it with other files to + produce an executable, this file does not by itself cause the + resulting executable to be covered by the GNU General Public + License. This exception does not however invalidate any other + reasons why the executable file might be covered by the GNU General + Public License. +*/ + +#include <stdbool.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +#include "lzlib.h" +#include "clzip.h" +#include "tables.c" +#include "decoder.c" +#include "encoder.c" + + +struct LZ_Encoder + { + long long partial_in_size; + long long partial_out_size; + struct Matchfinder * matchfinder; + struct LZ_encoder * lz_encoder; + enum LZ_Errno lz_errno; + int flush_pending; + File_header member_header; + bool fatal; + }; + +static void LZ_Encoder_init( struct LZ_Encoder * const e, + const File_header header ) + { + int i; + e->partial_in_size = 0; + e->partial_out_size = 0; + e->matchfinder = 0; + e->lz_encoder = 0; + e->lz_errno = LZ_ok; + e->flush_pending = 0; + for( i = 0; i < Fh_size; ++i ) e->member_header[i] = header[i]; + e->fatal = false; + } + + +struct LZ_Decoder + { + long long partial_in_size; + long long partial_out_size; + struct Range_decoder * rdec; + struct LZ_decoder * lz_decoder; + enum LZ_Errno lz_errno; + File_header member_header; /* header of current member */ + bool fatal; + bool seeking; + }; + +static void LZ_Decoder_init( struct LZ_Decoder * const d ) + { + int i; + d->partial_in_size = 0; + d->partial_out_size = 0; + d->rdec = 0; + d->lz_decoder = 0; + d->lz_errno = LZ_ok; + for( i = 0; i < Fh_size; ++i ) d->member_header[i] = 0; + d->fatal = false; + d->seeking = false; + } + + +static bool verify_encoder( struct LZ_Encoder * const e ) + { + if( !e ) return false; + if( !e->matchfinder || !e->lz_encoder ) + { e->lz_errno = LZ_bad_argument; return false; } + return true; + } + + +static bool verify_decoder( struct LZ_Decoder * const d ) + { + if( !d ) return false; + if( !d->rdec ) + { d->lz_errno = LZ_bad_argument; return false; } + return true; + } + + +/*------------------------- Misc Functions -------------------------*/ + +const char * LZ_version( void ) { return LZ_version_string; } + + +const char * LZ_strerror( const enum LZ_Errno lz_errno ) + { + switch( lz_errno ) + { + case LZ_ok : return "ok"; + case LZ_bad_argument : return "bad argument"; + case LZ_mem_error : return "not enough memory"; + case LZ_sequence_error: return "sequence error"; + case LZ_header_error : return "header error"; + case LZ_unexpected_eof: return "unexpected eof"; + case LZ_data_error : return "data error"; + case LZ_library_error : return "library error"; + } + return "invalid error code"; + } + + +int LZ_min_dictionary_bits( void ) { return min_dictionary_bits; } +int LZ_min_dictionary_size( void ) { return min_dictionary_size; } +int LZ_max_dictionary_bits( void ) { return max_dictionary_bits; } +int LZ_max_dictionary_size( void ) { return max_dictionary_size; } +int LZ_min_match_len_limit( void ) { return min_match_len_limit; } +int LZ_max_match_len_limit( void ) { return max_match_len; } + + +/*---------------------- Compression Functions ----------------------*/ + +struct LZ_Encoder * LZ_compress_open( const int dictionary_size, + const int match_len_limit, + const long long member_size ) + { + File_header header; + Fh_set_magic( header ); + const bool error = ( !Fh_set_dictionary_size( header, dictionary_size ) || + match_len_limit < min_match_len_limit || + match_len_limit > max_match_len ); + + struct LZ_Encoder * const e = + (struct LZ_Encoder *)malloc( sizeof (struct LZ_Encoder) ); + if( !e ) return 0; + LZ_Encoder_init( e, header ); + if( error ) e->lz_errno = LZ_bad_argument; + else + { + e->matchfinder = (struct Matchfinder *)malloc( sizeof (struct Matchfinder) ); + e->lz_encoder = (struct LZ_encoder *)malloc( sizeof (struct LZ_encoder) ); + if( !e->matchfinder || !e->lz_encoder || + !Mf_init( e->matchfinder, + Fh_get_dictionary_size( header ), match_len_limit ) || + !LZe_init( e->lz_encoder, e->matchfinder, header, member_size ) ) + { + if( e->matchfinder ) + { Mf_free( e->matchfinder ); free( e->matchfinder ); e->matchfinder = 0; } + if( e->lz_encoder ) + { LZe_free( e->lz_encoder ); free( e->lz_encoder ); e->lz_encoder = 0; } + e->lz_errno = LZ_mem_error; + } + } + if( e->lz_errno != LZ_ok ) e->fatal = true; + return e; + } + + +int LZ_compress_close( struct LZ_Encoder * const e ) + { + if( !e ) return -1; + if( e->lz_encoder ) + { LZe_free( e->lz_encoder ); free( e->lz_encoder ); } + if( e->matchfinder ) + { Mf_free( e->matchfinder ); free( e->matchfinder ); } + free( e ); + return 0; + } + + +int LZ_compress_finish( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + Mf_set_flushing( e->matchfinder, true ); + e->flush_pending = 0; + return 0; + } + + +int LZ_compress_restart_member( struct LZ_Encoder * const e, + const long long member_size ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + if( !LZe_member_finished( e->lz_encoder ) ) + { e->lz_errno = LZ_sequence_error; return -1; } + + e->partial_in_size += Mf_data_position( e->matchfinder ); + e->partial_out_size += Re_member_position( &e->lz_encoder->range_encoder ); + Mf_reset( e->matchfinder ); + + LZe_free( e->lz_encoder ); free( e->lz_encoder ); + e->lz_encoder = (struct LZ_encoder *)malloc( sizeof (struct LZ_encoder) ); + if( !e->lz_encoder || + !LZe_init( e->lz_encoder, e->matchfinder, e->member_header, member_size ) ) + { + if( e->lz_encoder ) + { LZe_free( e->lz_encoder ); free( e->lz_encoder ); e->lz_encoder = 0; } + e->lz_errno = LZ_mem_error; e->fatal = true; + return -1; + } + e->lz_errno = LZ_ok; + return 0; + } + + +int LZ_compress_sync_flush( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + if( !e->flush_pending && !e->matchfinder->at_stream_end ) + { + e->flush_pending = 2; /* 2 consecutive markers guarantee decoding */ + Mf_set_flushing( e->matchfinder, true ); + if( !LZe_encode_member( e->lz_encoder, false ) ) + { e->lz_errno = LZ_library_error; e->fatal = true; return -1; } + while( e->flush_pending > 0 && LZe_sync_flush( e->lz_encoder ) ) + { if( --e->flush_pending <= 0 ) Mf_set_flushing( e->matchfinder, false ); } + } + return 0; + } + + +int LZ_compress_read( struct LZ_Encoder * const e, + uint8_t * const buffer, const int size ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + if( !LZe_encode_member( e->lz_encoder, !e->flush_pending ) ) + { e->lz_errno = LZ_library_error; e->fatal = true; return -1; } + while( e->flush_pending > 0 && LZe_sync_flush( e->lz_encoder ) ) + { if( --e->flush_pending <= 0 ) Mf_set_flushing( e->matchfinder, false ); } + return Re_read_data( &e->lz_encoder->range_encoder, buffer, size ); + } + + +int LZ_compress_write( struct LZ_Encoder * const e, + const uint8_t * const buffer, const int size ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + if( e->flush_pending ) return 0; + return Mf_write_data( e->matchfinder, buffer, size ); + } + + +int LZ_compress_write_size( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) || e->fatal ) return -1; + if( e->flush_pending ) return 0; + return Mf_free_bytes( e->matchfinder ); + } + + +enum LZ_Errno LZ_compress_errno( struct LZ_Encoder * const e ) + { + if( !e ) return LZ_bad_argument; + return e->lz_errno; + } + + +int LZ_compress_finished( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return -1; + return ( !e->flush_pending && Mf_finished( e->matchfinder ) && + LZe_member_finished( e->lz_encoder ) ); + } + + +int LZ_compress_member_finished( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return -1; + return LZe_member_finished( e->lz_encoder ); + } + + +long long LZ_compress_data_position( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return -1; + return Mf_data_position( e->matchfinder ); + } + + +long long LZ_compress_member_position( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return -1; + return Re_member_position( &e->lz_encoder->range_encoder ); + } + + +long long LZ_compress_total_in_size( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return -1; + return e->partial_in_size + Mf_data_position( e->matchfinder ); + } + + +long long LZ_compress_total_out_size( struct LZ_Encoder * const e ) + { + if( !verify_encoder( e ) ) return -1; + return e->partial_out_size + + Re_member_position( &e->lz_encoder->range_encoder ); + } + + +/*--------------------- Decompression Functions ---------------------*/ + +struct LZ_Decoder * LZ_decompress_open( void ) + { + struct LZ_Decoder * const d = + (struct LZ_Decoder *)malloc( sizeof (struct LZ_Decoder) ); + if( !d ) return 0; + LZ_Decoder_init( d ); + + d->rdec = (struct Range_decoder *)malloc( sizeof (struct Range_decoder) ); + if( !d->rdec || !Rd_init( d->rdec ) ) + { + if( d->rdec ) { Rd_free( d->rdec ); free( d->rdec ); d->rdec = 0; } + d->lz_errno = LZ_mem_error; d->fatal = true; + } + return d; + } + + +int LZ_decompress_close( struct LZ_Decoder * const d ) + { + if( !d ) return -1; + if( d->lz_decoder ) + { LZd_free( d->lz_decoder ); free( d->lz_decoder ); } + if( d->rdec ) { Rd_free( d->rdec ); free( d->rdec ); } + free( d ); + return 0; + } + + +int LZ_decompress_finish( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) || d->fatal ) return -1; + if( d->seeking ) { d->seeking = false; Rd_purge( d->rdec ); } + else Rd_finish( d->rdec ); + return 0; + } + + +int LZ_decompress_reset( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + if( d->lz_decoder ) + { LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0; } + d->partial_in_size = 0; + d->partial_out_size = 0; + Rd_reset( d->rdec ); + d->lz_errno = LZ_ok; + d->fatal = false; + d->seeking = false; + return 0; + } + + +int LZ_decompress_sync_to_member( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + if( d->lz_decoder ) + { LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0; } + if( Rd_find_header( d->rdec ) ) d->seeking = false; + else + { + if( !d->rdec->at_stream_end ) d->seeking = true; + else { d->seeking = false; Rd_purge( d->rdec ); } + } + d->lz_errno = LZ_ok; + d->fatal = false; + return 0; + } + + +int LZ_decompress_read( struct LZ_Decoder * const d, + uint8_t * const buffer, const int size ) + { + if( !verify_decoder( d ) || d->fatal ) return -1; + if( d->seeking ) return 0; + if( d->lz_decoder && LZd_member_finished( d->lz_decoder ) ) + { + d->partial_in_size += d->rdec->member_position ; + d->partial_out_size += LZd_data_position( d->lz_decoder ); + LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0; + } + if( !d->lz_decoder ) + { + if( Rd_available_bytes( d->rdec ) < 5 + Fh_size ) + { + if( !d->rdec->at_stream_end || Rd_finished( d->rdec ) ) return 0; + Rd_purge( d->rdec ); /* remove trailing garbage */ + d->lz_errno = LZ_header_error; + d->fatal = true; + return -1; + } + if( !Rd_read_header( d->rdec, d->member_header ) ) + { + d->lz_errno = LZ_header_error; + d->fatal = true; + return -1; + } + d->lz_decoder = (struct LZ_decoder *)malloc( sizeof (struct LZ_decoder) ); + if( !d->lz_decoder || !LZd_init( d->lz_decoder, d->member_header, d->rdec ) ) + { /* not enough free memory */ + if( d->lz_decoder ) + { LZd_free( d->lz_decoder ); free( d->lz_decoder ); d->lz_decoder = 0; } + d->lz_errno = LZ_mem_error; + d->fatal = true; + return -1; + } + } + const int result = LZd_decode_member( d->lz_decoder ); + if( result != 0 ) + { + if( result == 2 ) d->lz_errno = LZ_unexpected_eof; + else d->lz_errno = LZ_data_error; + d->fatal = true; + return -1; + } + return Cb_read_data( &d->lz_decoder->cb, buffer, size ); + } + + +int LZ_decompress_write( struct LZ_Decoder * const d, + const uint8_t * const buffer, const int size ) + { + if( !verify_decoder( d ) || d->fatal ) return -1; + int result = Rd_write_data( d->rdec, buffer, size ); + while( d->seeking ) + { + if( Rd_find_header( d->rdec ) ) d->seeking = false; + if( result >= size ) break; + const int size2 = Rd_write_data( d->rdec, buffer + result, size - result ); + if( size2 > 0 ) result += size2; + else break; + } + return result; + } + + +int LZ_decompress_write_size( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) || d->fatal ) return -1; + return Rd_free_bytes( d->rdec ); + } + + +enum LZ_Errno LZ_decompress_errno( struct LZ_Decoder * const d ) + { + if( !d ) return LZ_bad_argument; + return d->lz_errno; + } + + +int LZ_decompress_finished( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + return ( Rd_finished( d->rdec ) && + ( !d->lz_decoder || LZd_member_finished( d->lz_decoder ) ) ); + } + + +int LZ_decompress_member_finished( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + return ( d->lz_decoder && LZd_member_finished( d->lz_decoder ) ); + } + + +int LZ_decompress_member_version( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + return Fh_version( d->member_header ); + } + + +int LZ_decompress_dictionary_size( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + return Fh_get_dictionary_size( d->member_header ); + } + + +unsigned int LZ_decompress_data_crc( struct LZ_Decoder * const d ) + { + if( verify_decoder( d ) && d->lz_decoder ) + return LZd_crc( d->lz_decoder ); + else return 0; + } + + +long long LZ_decompress_data_position( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + if( d->lz_decoder ) + return LZd_data_position( d->lz_decoder ); + else return 0; + } + + +long long LZ_decompress_member_position( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + if( d->lz_decoder ) + return d->rdec->member_position ; + else return 0; + } + + +long long LZ_decompress_total_in_size( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + if( d->lz_decoder ) + return d->partial_in_size + d->rdec->member_position ; + return d->partial_in_size; + } + + +long long LZ_decompress_total_out_size( struct LZ_Decoder * const d ) + { + if( !verify_decoder( d ) ) return -1; + if( d->lz_decoder ) + return d->partial_out_size + LZd_data_position( d->lz_decoder ); + return d->partial_out_size; + } diff --git a/lzlib.cc b/lzlib.cc deleted file mode 100644 index 4a4b77b..0000000 --- a/lzlib.cc +++ /dev/null @@ -1,540 +0,0 @@ -/* Lzlib - A compression library for lzip files - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. - - This library is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this library. If not, see <http://www.gnu.org/licenses/>. - - As a special exception, you may use this file as part of a free - software library without restriction. Specifically, if other files - instantiate templates or use macros or inline functions from this - file, or you compile this file and link it with other files to - produce an executable, this file does not by itself cause the - resulting executable to be covered by the GNU General Public - License. This exception does not however invalidate any other - reasons why the executable file might be covered by the GNU General - Public License. -*/ - -#include <algorithm> -#include <cstring> -#include <stdint.h> - -#include "lzlib.h" -#include "lzip.h" -#include "decoder.h" -#include "encoder.h" - - -using namespace Lzlib; - -struct LZ_Encoder - { - long long partial_in_size; - long long partial_out_size; - Matchfinder * matchfinder; - LZ_encoder * lz_encoder; - LZ_Errno lz_errno; - int flush_pending; - const File_header member_header; - bool fatal; - - LZ_Encoder( const File_header & header ) throw() - : - partial_in_size( 0 ), - partial_out_size( 0 ), - matchfinder( 0 ), - lz_encoder( 0 ), - lz_errno( LZ_ok ), - flush_pending( 0 ), - member_header( header ), - fatal( false ) - {} - }; - - -struct LZ_Decoder - { - long long partial_in_size; - long long partial_out_size; - Range_decoder * rdec; - LZ_decoder * lz_decoder; - LZ_Errno lz_errno; - File_header member_header; // header of current member - bool fatal; - bool seeking; - - LZ_Decoder() throw() - : - partial_in_size( 0 ), - partial_out_size( 0 ), - rdec( 0 ), - lz_decoder( 0 ), - lz_errno( LZ_ok ), - fatal( false ), - seeking( false ) - { - for( int i = 0; i < File_header::size; ++i ) member_header.data[i] = 0; - } - }; - - -namespace Lzlib { - -bool verify_encoder( LZ_Encoder * const encoder ) - { - if( !encoder ) return false; - if( !encoder->matchfinder || !encoder->lz_encoder ) - { encoder->lz_errno = LZ_bad_argument; return false; } - return true; - } - - -bool verify_decoder( struct LZ_Decoder * const decoder ) - { - if( !decoder ) return false; - if( !decoder->rdec ) - { decoder->lz_errno = LZ_bad_argument; return false; } - return true; - } - -} // end namespace Lzlib - - -const char * LZ_version() { return LZ_version_string; } - - -const char * LZ_strerror( const LZ_Errno lz_errno ) - { - switch( lz_errno ) - { - case LZ_ok : return "ok"; - case LZ_bad_argument : return "bad argument"; - case LZ_mem_error : return "not enough memory"; - case LZ_sequence_error: return "sequence error"; - case LZ_header_error : return "header error"; - case LZ_unexpected_eof: return "unexpected eof"; - case LZ_data_error : return "data error"; - case LZ_library_error : return "library error"; - } - return "invalid error code"; - } - - -int LZ_min_dictionary_bits() { return min_dictionary_bits; } -int LZ_min_dictionary_size() { return min_dictionary_size; } -int LZ_max_dictionary_bits() { return max_dictionary_bits; } -int LZ_max_dictionary_size() { return max_dictionary_size; } -int LZ_min_match_len_limit() { return min_match_len_limit; } -int LZ_max_match_len_limit() { return max_match_len; } - - -/*---------------------- Compression Functions ----------------------*/ - -LZ_Encoder * LZ_compress_open( const int dictionary_size, - const int match_len_limit, - const long long member_size ) - { - File_header header; - header.set_magic(); - const bool error = ( !header.dictionary_size( dictionary_size ) || - match_len_limit < min_match_len_limit || - match_len_limit > max_match_len ); - - LZ_Encoder * encoder = new( std::nothrow ) LZ_Encoder( header ); - if( !encoder ) return 0; - LZ_Encoder & e = *encoder; - if( error ) e.lz_errno = LZ_bad_argument; - else - { - try { - e.matchfinder = new Matchfinder( header.dictionary_size(), match_len_limit ); - } - catch( std::bad_alloc ) { e.matchfinder = 0; } - if( e.matchfinder ) - { - try { - e.lz_encoder = new LZ_encoder( *e.matchfinder, header, member_size ); - } - catch( std::bad_alloc ) - { - delete e.matchfinder; - e.matchfinder = 0; - e.lz_encoder = 0; - } - } - if( !e.lz_encoder ) e.lz_errno = LZ_mem_error; - } - if( e.lz_errno != LZ_ok ) e.fatal = true; - return encoder; - } - - -int LZ_compress_close( LZ_Encoder * const encoder ) - { - if( !encoder ) return -1; - if( encoder->lz_encoder ) delete encoder->lz_encoder; - if( encoder->matchfinder ) delete encoder->matchfinder; - delete encoder; - return 0; - } - - -int LZ_compress_finish( LZ_Encoder * const encoder ) - { - if( !verify_encoder( encoder ) || encoder->fatal ) return -1; - encoder->matchfinder->flushing( true ); - encoder->flush_pending = 0; - return 0; - } - - -int LZ_compress_restart_member( LZ_Encoder * const encoder, - const long long member_size ) - { - if( !verify_encoder( encoder ) || encoder->fatal ) return -1; - LZ_Encoder & e = *encoder; - if( !e.lz_encoder->member_finished() ) - { e.lz_errno = LZ_sequence_error; return -1; } - - e.partial_in_size += e.matchfinder->data_position(); - e.partial_out_size += e.lz_encoder->member_position(); - e.matchfinder->reset(); - - delete e.lz_encoder; - try { - e.lz_encoder = new LZ_encoder( *e.matchfinder, e.member_header, member_size ); - } - catch( std::bad_alloc ) - { e.lz_encoder = 0; e.lz_errno = LZ_mem_error; e.fatal = true; return -1; } - e.lz_errno = LZ_ok; - return 0; - } - - -int LZ_compress_sync_flush( LZ_Encoder * const encoder ) - { - if( !verify_encoder( encoder ) || encoder->fatal ) return -1; - LZ_Encoder & e = *encoder; - if( !e.flush_pending && !e.matchfinder->at_stream_end() ) - { - e.flush_pending = 2; // 2 consecutive markers guarantee decoding - e.matchfinder->flushing( true ); - if( !e.lz_encoder->encode_member( false ) ) - { e.lz_errno = LZ_library_error; e.fatal = true; return -1; } - while( e.flush_pending > 0 && e.lz_encoder->sync_flush() ) - { if( --e.flush_pending <= 0 ) e.matchfinder->flushing( false ); } - } - return 0; - } - - -int LZ_compress_read( LZ_Encoder * const encoder, - uint8_t * const buffer, const int size ) - { - if( !verify_encoder( encoder ) || encoder->fatal ) return -1; - LZ_Encoder & e = *encoder; - if( !e.lz_encoder->encode_member( !e.flush_pending ) ) - { e.lz_errno = LZ_library_error; e.fatal = true; return -1; } - while( e.flush_pending > 0 && e.lz_encoder->sync_flush() ) - { if( --e.flush_pending <= 0 ) e.matchfinder->flushing( false ); } - return e.lz_encoder->read_data( buffer, size ); - } - - -int LZ_compress_write( LZ_Encoder * const encoder, - const uint8_t * const buffer, const int size ) - { - if( !verify_encoder( encoder ) || encoder->fatal ) return -1; - if( encoder->flush_pending ) return 0; - return encoder->matchfinder->write_data( buffer, size ); - } - - -int LZ_compress_write_size( LZ_Encoder * const encoder ) - { - if( !verify_encoder( encoder ) || encoder->fatal ) return -1; - if( encoder->flush_pending ) return 0; - return encoder->matchfinder->free_bytes(); - } - - -LZ_Errno LZ_compress_errno( LZ_Encoder * const encoder ) - { - if( !encoder ) return LZ_bad_argument; - return encoder->lz_errno; - } - - -int LZ_compress_finished( LZ_Encoder * const encoder ) - { - if( !verify_encoder( encoder ) ) return -1; - return ( !encoder->flush_pending && encoder->matchfinder->finished() && - encoder->lz_encoder->member_finished() ); - } - - -int LZ_compress_member_finished( LZ_Encoder * const encoder ) - { - if( !verify_encoder( encoder ) ) return -1; - return encoder->lz_encoder->member_finished(); - } - - -long long LZ_compress_data_position( LZ_Encoder * const encoder ) - { - if( !verify_encoder( encoder ) ) return -1; - return encoder->matchfinder->data_position(); - } - - -long long LZ_compress_member_position( LZ_Encoder * const encoder ) - { - if( !verify_encoder( encoder ) ) return -1; - return encoder->lz_encoder->member_position(); - } - - -long long LZ_compress_total_in_size( LZ_Encoder * const encoder ) - { - if( !verify_encoder( encoder ) ) return -1; - return encoder->partial_in_size + encoder->matchfinder->data_position(); - } - - -long long LZ_compress_total_out_size( LZ_Encoder * const encoder ) - { - if( !verify_encoder( encoder ) ) return -1; - return encoder->partial_out_size + encoder->lz_encoder->member_position(); - } - - -/*--------------------- Decompression Functions ---------------------*/ - -struct LZ_Decoder * LZ_decompress_open() - { - LZ_Decoder * decoder = new( std::nothrow ) LZ_Decoder; - if( !decoder ) return 0; - - LZ_Decoder & d = *decoder; - try { d.rdec = new Range_decoder; } - catch( std::bad_alloc ) - { d.rdec = 0; d.lz_errno = LZ_mem_error; d.fatal = true; } - return decoder; - } - - -int LZ_decompress_close( struct LZ_Decoder * const decoder ) - { - if( !decoder ) return -1; - if( decoder->lz_decoder ) delete decoder->lz_decoder; - if( decoder->rdec ) delete decoder->rdec; - delete decoder; - return 0; - } - - -int LZ_decompress_finish( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) || decoder->fatal ) return -1; - LZ_Decoder & d = *decoder; - if( d.seeking ) { d.seeking = false; d.rdec->purge(); } - else d.rdec->finish(); - return 0; - } - - -int LZ_decompress_reset( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - LZ_Decoder & d = *decoder; - if( d.lz_decoder ) { delete d.lz_decoder; d.lz_decoder = 0; } - d.partial_in_size = 0; - d.partial_out_size = 0; - d.rdec->reset(); - d.lz_errno = LZ_ok; - d.fatal = false; - d.seeking = false; - return 0; - } - - -int LZ_decompress_sync_to_member( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - LZ_Decoder & d = *decoder; - if( d.lz_decoder ) { delete d.lz_decoder; d.lz_decoder = 0; } - if( d.rdec->find_header() ) d.seeking = false; - else - { - if( !d.rdec->at_stream_end() ) d.seeking = true; - else { d.seeking = false; d.rdec->purge(); } - } - d.lz_errno = LZ_ok; - d.fatal = false; - return 0; - } - - -int LZ_decompress_read( struct LZ_Decoder * const decoder, - uint8_t * const buffer, const int size ) - { - if( !verify_decoder( decoder ) || decoder->fatal ) return -1; - LZ_Decoder & d = *decoder; - if( d.seeking ) return 0; - if( d.lz_decoder && d.lz_decoder->member_finished() ) - { - d.partial_in_size += d.rdec->member_position(); - d.partial_out_size += d.lz_decoder->data_position(); - delete d.lz_decoder; - d.lz_decoder = 0; - } - if( !d.lz_decoder ) - { - if( d.rdec->used_bytes() < 5 + File_header::size ) - { - if( !d.rdec->at_stream_end() || d.rdec->finished() ) return 0; - d.rdec->purge(); // remove trailing garbage - d.lz_errno = LZ_header_error; - d.fatal = true; - return -1; - } - if( !d.rdec->read_header( d.member_header ) ) - { - d.lz_errno = LZ_header_error; - d.fatal = true; - return -1; - } - try { d.lz_decoder = new LZ_decoder( d.member_header, *d.rdec ); } - catch( std::bad_alloc ) // not enough free memory - { - d.lz_decoder = 0; - d.lz_errno = LZ_mem_error; - d.fatal = true; - return -1; - } - } - const int result = d.lz_decoder->decode_member(); - if( result != 0 ) - { - if( result == 2 ) d.lz_errno = LZ_unexpected_eof; - else d.lz_errno = LZ_data_error; - d.fatal = true; - return -1; - } - return d.lz_decoder->read_data( buffer, size ); - } - - -int LZ_decompress_write( struct LZ_Decoder * const decoder, - const uint8_t * const buffer, const int size ) - { - if( !verify_decoder( decoder ) || decoder->fatal ) return -1; - LZ_Decoder & d = *decoder; - int result = d.rdec->write_data( buffer, size ); - while( d.seeking ) - { - if( d.rdec->find_header() ) d.seeking = false; - if( result >= size ) break; - const int size2 = d.rdec->write_data( buffer + result, size - result ); - if( size2 > 0 ) result += size2; - else break; - } - return result; - } - - -int LZ_decompress_write_size( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) || decoder->fatal ) return -1; - return decoder->rdec->free_bytes(); - } - - -LZ_Errno LZ_decompress_errno( struct LZ_Decoder * const decoder ) - { - if( !decoder ) return LZ_bad_argument; - return decoder->lz_errno; - } - - -int LZ_decompress_finished( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - return ( decoder->rdec->finished() && - ( !decoder->lz_decoder || decoder->lz_decoder->member_finished() ) ); - } - - -int LZ_decompress_member_finished( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - return ( decoder->lz_decoder && decoder->lz_decoder->member_finished() ); - } - - -int LZ_decompress_member_version( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - return decoder->member_header.version(); - } - - -int LZ_decompress_dictionary_size( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - return decoder->member_header.dictionary_size(); - } - - -unsigned int LZ_decompress_data_crc( struct LZ_Decoder * const decoder ) - { - if( verify_decoder( decoder ) && decoder->lz_decoder ) - return decoder->lz_decoder->crc(); - else return 0; - } - - -long long LZ_decompress_data_position( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - if( decoder->lz_decoder ) - return decoder->lz_decoder->data_position(); - else return 0; - } - - -long long LZ_decompress_member_position( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - if( decoder->lz_decoder ) - return decoder->rdec->member_position(); - else return 0; - } - - -long long LZ_decompress_total_in_size( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - if( decoder->lz_decoder ) - return decoder->partial_in_size + decoder->rdec->member_position(); - return decoder->partial_in_size; - } - - -long long LZ_decompress_total_out_size( struct LZ_Decoder * const decoder ) - { - if( !verify_decoder( decoder ) ) return -1; - if( decoder->lz_decoder ) - return decoder->partial_out_size + decoder->lz_decoder->data_position(); - return decoder->partial_out_size; - } @@ -1,5 +1,5 @@ /* Lzlib - A compression library for lzip files - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. + Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. This library is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ extern "C" { #endif -const char * const LZ_version_string = "1.2"; +static const char * const LZ_version_string = "1.3"; enum LZ_Errno { LZ_ok = 0, LZ_bad_argument, LZ_mem_error, LZ_sequence_error, LZ_header_error, LZ_unexpected_eof, @@ -0,0 +1,1036 @@ +/* Minilzip - A test program for the lzlib library + Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ +/* + Return values: 0 for a normal exit, 1 for environmental problems + (file not found, invalid flags, I/O errors, etc), 2 to indicate a + corrupt or invalid input file, 3 for an internal consistency error + (eg, bug) which caused minilzip to panic. +*/ + +#define _FILE_OFFSET_BITS 64 + +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <signal.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> +#include <utime.h> +#include <sys/stat.h> + +#include "carg_parser.h" +#include "lzlib.h" + +#if CHAR_BIT != 8 +#error "Environments where CHAR_BIT != 8 are not supported." +#endif + +#ifndef LLONG_MAX +#define LLONG_MAX 0x7FFFFFFFFFFFFFFFLL +#endif +#ifndef LLONG_MIN +#define LLONG_MIN (-LLONG_MAX - 1LL) +#endif +#ifndef ULLONG_MAX +#define ULLONG_MAX 0xFFFFFFFFFFFFFFFFULL +#endif + +#ifndef min + #define min(x,y) ((x) <= (y) ? (x) : (y)) +#endif + +long long int llabs( long long int number ); + + +const char * const Program_name = "Minilzip"; +const char * const program_name = "minilzip"; +const char * const program_year = "2012"; +const char * invocation_name = 0; + +#ifdef O_BINARY +const int o_binary = O_BINARY; +#else +const int o_binary = 0; +#endif + +struct { const char * from; const char * to; } const known_extensions[] = { + { ".lz", "" }, + { ".tlz", ".tar" }, + { 0, 0 } }; + +struct Lzma_options + { + int dictionary_size; /* 4KiB..512MiB */ + int match_len_limit; /* 5..273 */ + }; + +enum Mode { m_compress, m_decompress, m_test }; + +char * output_filename = 0; +int outfd = -1; +int verbosity = 0; +const mode_t usr_rw = S_IRUSR | S_IWUSR; +const mode_t all_rw = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; +mode_t outfd_mode = S_IRUSR | S_IWUSR; +bool delete_output_on_interrupt = false; + + +struct Pretty_print + { + const char * name; + const char * stdin_name; + int longest_name; + bool first_post; + }; + +static void Pp_init( struct Pretty_print * const pp, + const char * const filenames[], const int num_filenames ) + { + unsigned int stdin_name_len; + int i; + pp->name = 0; + pp->stdin_name = "(stdin)"; + pp->longest_name = 0; + pp->first_post = false; + stdin_name_len = strlen( pp->stdin_name ); + + for( i = 0; i < num_filenames; ++i ) + { + const char * const s = filenames[i]; + const int len = ( !strcmp( s, "-" ) ? stdin_name_len : strlen( s ) ); + if( len > pp->longest_name ) pp->longest_name = len; + } + if( pp->longest_name == 0 ) pp->longest_name = stdin_name_len; + } + + +static inline void Pp_set_name( struct Pretty_print * const pp, + const char * const filename ) + { + if( filename && filename[0] && strcmp( filename, "-" ) ) + pp->name = filename; + else pp->name = pp->stdin_name; + pp->first_post = true; + } + + +static inline void Pp_reset( struct Pretty_print * const pp ) + { if( pp->name && pp->name[0] ) pp->first_post = true; } + + +static void Pp_show_msg( struct Pretty_print * const pp, const char * const msg ) + { + if( verbosity >= 0 ) + { + if( pp->first_post ) + { + int i, len; + pp->first_post = false; + fprintf( stderr, " %s: ", pp->name ); + len = pp->longest_name - strlen( pp->name ); + for( i = 0; i < len; ++i ) fprintf( stderr, " " ); + if( !msg ) fflush( stderr ); + } + if( msg ) fprintf( stderr, "%s.\n", msg ); + } + } + + +static void show_help( void ) + { + printf( "%s - A test program for the lzlib library.\n", Program_name ); + printf( "\nUsage: %s [options] [files]\n", invocation_name ); + printf( "\nOptions:\n" + " -h, --help display this help and exit\n" + " -V, --version output version information and exit\n" + " -b, --member-size=<bytes> set member size limit in bytes\n" + " -c, --stdout send output to standard output\n" + " -d, --decompress decompress\n" + " -f, --force overwrite existing output files\n" + " -F, --recompress force recompression of compressed files\n" + " -k, --keep keep (don't delete) input files\n" + " -m, --match-length=<bytes> set match length limit in bytes [36]\n" + " -o, --output=<file> if reading stdin, place the output into <file>\n" + " -q, --quiet suppress all messages\n" + " -s, --dictionary-size=<bytes> set dictionary size limit in bytes [8MiB]\n" + " -S, --volume-size=<bytes> set volume size limit in bytes\n" + " -t, --test test compressed file integrity\n" + " -v, --verbose be verbose (a 2nd -v gives more)\n" + " -1 .. -9 set compression level [default 6]\n" + " --fast alias for -1\n" + " --best alias for -9\n" + "If no file names are given, minilzip compresses or decompresses\n" + "from standard input to standard output.\n" + "Numbers may be followed by a multiplier: k = kB = 10^3 = 1000,\n" + "Ki = KiB = 2^10 = 1024, M = 10^6, Mi = 2^20, G = 10^9, Gi = 2^30, etc...\n" + "The bidimensional parameter space of LZMA can't be mapped to a linear\n" + "scale optimal for all files. If your files are large, very repetitive,\n" + "etc, you may need to use the --match-length and --dictionary-size\n" + "options directly to achieve optimal performance.\n" + "\nReport bugs to lzip-bug@nongnu.org\n" + "Lzlib home page: http://www.nongnu.org/lzip/lzlib.html\n" ); + } + + +static void show_version( void ) + { + printf( "%s %s\n", Program_name, PROGVERSION ); + printf( "Copyright (C) %s Antonio Diaz Diaz.\n", program_year ); + printf( "Using Lzlib %s\n", LZ_version() ); + printf( "License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>\n" + "This is free software: you are free to change and redistribute it.\n" + "There is NO WARRANTY, to the extent permitted by law.\n" ); + } + + +void show_error( const char * const msg, const int errcode, const bool help ) + { + if( verbosity >= 0 ) + { + if( msg && msg[0] ) + { + fprintf( stderr, "%s: %s", program_name, msg ); + if( errcode > 0 ) fprintf( stderr, ": %s", strerror( errcode ) ); + fprintf( stderr, "\n" ); + } + if( help && invocation_name && invocation_name[0] ) + fprintf( stderr, "Try '%s --help' for more information.\n", + invocation_name ); + } + } + + +void internal_error( const char * const msg ) + { + if( verbosity >= 0 ) + fprintf( stderr, "%s: internal error: %s.\n", program_name, msg ); + exit( 3 ); + } + + +static const char * format_num( long long num ) + { + const char * const prefix[8] = + { "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi" }; + enum { buf_size = 16, factor = 1024 }; + static char buf[buf_size]; + const char *p = ""; + bool exact = ( num % factor == 0 ); + int i; + + for( i = 0; i < 8 && ( llabs( num ) > 9999 || + ( exact && llabs( num ) >= factor ) ); ++i ) + { num /= factor; if( num % factor != 0 ) exact = false; p = prefix[i]; } + snprintf( buf, buf_size, "%lld %s", num, p ); + return buf; + } + + +static long long getnum( const char * const ptr, + const long long llimit, const long long ulimit ) + { + long long result; + char *tail; + errno = 0; + result = strtoll( ptr, &tail, 0 ); + if( tail == ptr ) + { + show_error( "Bad or missing numerical argument.", 0, true ); + exit( 1 ); + } + + if( !errno && tail[0] ) + { + int factor = ( tail[1] == 'i' ) ? 1024 : 1000; + int exponent = 0; + int i; + bool bad_multiplier = false; + switch( tail[0] ) + { + case ' ': break; + case 'Y': exponent = 8; break; + case 'Z': exponent = 7; break; + case 'E': exponent = 6; break; + case 'P': exponent = 5; break; + case 'T': exponent = 4; break; + case 'G': exponent = 3; break; + case 'M': exponent = 2; break; + case 'K': if( factor == 1024 ) exponent = 1; else bad_multiplier = true; + break; + case 'k': if( factor == 1000 ) exponent = 1; else bad_multiplier = true; + break; + default : bad_multiplier = true; + } + if( bad_multiplier ) + { + show_error( "Bad multiplier in numerical argument.", 0, true ); + exit( 1 ); + } + for( i = 0; i < exponent; ++i ) + { + if( LLONG_MAX / factor >= llabs( result ) ) result *= factor; + else { errno = ERANGE; break; } + } + } + if( !errno && ( result < llimit || result > ulimit ) ) errno = ERANGE; + if( errno ) + { + show_error( "Numerical argument out of limits.", 0, false ); + exit( 1 ); + } + return result; + } + + +static int get_dict_size( const char * const arg ) + { + char *tail; + int bits = strtol( arg, &tail, 0 ); + if( bits >= LZ_min_dictionary_bits() && + bits <= LZ_max_dictionary_bits() && *tail == 0 ) + return ( 1 << bits ); + return getnum( arg, LZ_min_dictionary_size(), LZ_max_dictionary_size() ); + } + + +static int open_instream( const char * const name, struct stat * const in_statsp, + const enum Mode program_mode, const int eindex, + const bool recompress, const bool to_stdout ) + { + int infd = -1; + if( program_mode == m_compress && !recompress && eindex >= 0 ) + { + if( verbosity >= 0 ) + fprintf( stderr, "%s: Input file '%s' already has '%s' suffix.\n", + program_name, name, known_extensions[eindex].from ); + } + else + { + infd = open( name, O_RDONLY | o_binary ); + if( infd < 0 ) + { + if( verbosity >= 0 ) + fprintf( stderr, "%s: Can't open input file '%s': %s.\n", + program_name, name, strerror( errno ) ); + } + else + { + const int i = fstat( infd, in_statsp ); + const mode_t mode = in_statsp->st_mode; + const bool can_read = ( i == 0 && + ( S_ISBLK( mode ) || S_ISCHR( mode ) || + S_ISFIFO( mode ) || S_ISSOCK( mode ) ) ); + if( i != 0 || ( !S_ISREG( mode ) && ( !to_stdout || !can_read ) ) ) + { + if( verbosity >= 0 ) + fprintf( stderr, "%s: Input file '%s' is not a regular file%s.\n", + program_name, name, + ( can_read && !to_stdout ) ? + " and '--stdout' was not specified" : "" ); + close( infd ); + infd = -1; + } + } + } + return infd; + } + + +void cleanup_and_fail( const int retval ) + { + if( delete_output_on_interrupt ) + { + delete_output_on_interrupt = false; + if( verbosity >= 0 ) + fprintf( stderr, "%s: Deleting output file '%s', if it exists.\n", + program_name, output_filename ); + if( outfd >= 0 ) { close( outfd ); outfd = -1; } + if( remove( output_filename ) != 0 && errno != ENOENT ) + show_error( "WARNING: deletion of output file (apparently) failed.", 0, false ); + } + exit( retval ); + } + + +/* assure at least a minimum size for buffer 'buf' */ +static void * resize_buffer( void * buf, const int min_size ) + { + if( buf ) buf = realloc( buf, min_size ); + else buf = malloc( min_size ); + if( !buf ) + { + show_error( "Not enough memory.", 0, false ); + cleanup_and_fail( 1 ); + } + return buf; + } + + +static void set_c_outname( const char * const name, const bool multifile ) + { + output_filename = resize_buffer( output_filename, strlen( name ) + 5 + + strlen( known_extensions[0].from ) + 1 ); + strcpy( output_filename, name ); + if( multifile ) strcat( output_filename, "00001" ); + strcat( output_filename, known_extensions[0].from ); + } + + +static int extension_index( const char * const name ) + { + int i; + for( i = 0; known_extensions[i].from; ++i ) + { + const char * const ext = known_extensions[i].from; + if( strlen( name ) > strlen( ext ) && + strncmp( name + strlen( name ) - strlen( ext ), ext, strlen( ext ) ) == 0 ) + return i; + } + return -1; + } + + +static void set_d_outname( const char * const name, const int i ) + { + if( i >= 0 ) + { + const char * const from = known_extensions[i].from; + if( strlen( name ) > strlen( from ) ) + { + output_filename = resize_buffer( output_filename, strlen( name ) + + strlen( known_extensions[0].to ) + 1 ); + strcpy( output_filename, name ); + strcpy( output_filename + strlen( name ) - strlen( from ), + known_extensions[i].to ); + return; + } + } + output_filename = resize_buffer( output_filename, strlen( name ) + 4 + 1 ); + strcpy( output_filename, name ); + strcat( output_filename, ".out" ); + if( verbosity >= 1 ) + fprintf( stderr, "%s: Can't guess original name for '%s' -- using '%s'.\n", + program_name, name, output_filename ); + } + + +static bool open_outstream( const bool force ) + { + int flags = O_CREAT | O_WRONLY | o_binary; + if( force ) flags |= O_TRUNC; else flags |= O_EXCL; + + outfd = open( output_filename, flags, outfd_mode ); + if( outfd < 0 && verbosity >= 0 ) + { + if( errno == EEXIST ) + fprintf( stderr, "%s: Output file '%s' already exists, skipping.\n", + program_name, output_filename ); + else + fprintf( stderr, "%s: Can't create output file '%s': %s.\n", + program_name, output_filename, strerror( errno ) ); + } + return ( outfd >= 0 ); + } + + +static bool check_tty( const int infd, const enum Mode program_mode ) + { + if( program_mode == m_compress && outfd >= 0 && isatty( outfd ) ) + { + show_error( "I won't write compressed data to a terminal.", 0, true ); + return false; + } + if( ( program_mode == m_decompress || program_mode == m_test ) && + isatty( infd ) ) + { + show_error( "I won't read compressed data from a terminal.", 0, true ); + return false; + } + return true; + } + + + /* Set permissions, owner and times. */ +static void close_and_set_permissions( const struct stat * const in_statsp ) + { + bool warning = false; + if( in_statsp ) + { + /* fchown will in many cases return with EPERM, which can be safely ignored. */ + if( ( fchown( outfd, in_statsp->st_uid, in_statsp->st_gid ) != 0 && + errno != EPERM ) || + fchmod( outfd, in_statsp->st_mode ) != 0 ) warning = true; + } + if( close( outfd ) != 0 ) cleanup_and_fail( 1 ); + outfd = -1; + delete_output_on_interrupt = false; + if( in_statsp ) + { + struct utimbuf t; + t.actime = in_statsp->st_atime; + t.modtime = in_statsp->st_mtime; + if( utime( output_filename, &t ) != 0 ) warning = true; + } + if( warning && verbosity >= 1 ) + show_error( "Can't change output file attributes.", 0, false ); + } + + +/* Returns the number of bytes really read. + If (returned value < size) and (errno == 0), means EOF was reached. +*/ +int readblock( const int fd, uint8_t * const buf, const int size ) + { + int rest = size; + errno = 0; + while( rest > 0 ) + { + errno = 0; + const int n = read( fd, buf + size - rest, rest ); + if( n > 0 ) rest -= n; + else if( n == 0 ) break; + else if( errno != EINTR && errno != EAGAIN ) break; + } + return ( rest > 0 ) ? size - rest : size; + } + + +/* Returns the number of bytes really written. + If (returned value < size), it is always an error. +*/ +int writeblock( const int fd, const uint8_t * const buf, const int size ) + { + int rest = size; + errno = 0; + while( rest > 0 ) + { + errno = 0; + const int n = write( fd, buf + size - rest, rest ); + if( n > 0 ) rest -= n; + else if( n < 0 && errno != EINTR && errno != EAGAIN ) break; + } + return ( rest > 0 ) ? size - rest : size; + } + + +static bool next_filename( void ) + { + const unsigned int len = strlen( known_extensions[0].from ); + int i, j; + + if( strlen( output_filename ) >= len + 5 ) /* "*00001.lz" */ + for( i = strlen( output_filename ) - len - 1, j = 0; j < 5; --i, ++j ) + { + if( output_filename[i] < '9' ) { ++output_filename[i]; return true; } + else output_filename[i] = '0'; + } + return false; + } + + +static int do_compress( struct LZ_Encoder * const encoder, const long long member_size, + const long long volume_size, const int infd, + struct Pretty_print * const pp, + const struct stat * const in_statsp ) + { + long long partial_volume_size = 0; + enum { buffer_size = 65536 }; + uint8_t buffer[buffer_size]; + + if( verbosity >= 1 ) Pp_show_msg( pp, 0 ); + while( true ) + { + int in_size = 0; + while( LZ_compress_write_size( encoder ) > 0 ) + { + const int size = min( LZ_compress_write_size( encoder ), buffer_size ); + const int rd = readblock( infd, buffer, size ); + if( rd != size && errno ) + { + Pp_show_msg( pp, 0 ); show_error( "Read error", errno, false ); + return 1; + } + if( rd > 0 && rd != LZ_compress_write( encoder, buffer, rd ) ) + internal_error( "library error (LZ_compress_write)" ); + if( rd < size ) LZ_compress_finish( encoder ); +/* else LZ_compress_sync_flush( encoder ); */ + in_size += rd; + } + const int out_size = LZ_compress_read( encoder, buffer, buffer_size ); + if( out_size < 0 ) + { + Pp_show_msg( pp, 0 ); + if( verbosity >= 0 ) + fprintf( stderr, "%s: LZ_compress_read error: %s.\n", + program_name, LZ_strerror( LZ_compress_errno( encoder ) ) ); + return 1; + } + else if( out_size > 0 ) + { + const int wr = writeblock( outfd, buffer, out_size ); + if( wr != out_size ) + { + Pp_show_msg( pp, 0 ); show_error( "Write error", errno, false ); + return 1; + } + } + else if( in_size == 0 ) internal_error( "library error (LZ_compress_read)" ); + if( LZ_compress_member_finished( encoder ) ) + { + if( LZ_compress_finished( encoder ) == 1 ) break; + partial_volume_size += LZ_compress_member_position( encoder ); + if( partial_volume_size >= volume_size - LZ_min_dictionary_size() ) + { + partial_volume_size = 0; + if( delete_output_on_interrupt ) + { + close_and_set_permissions( in_statsp ); + if( !next_filename() ) + { Pp_show_msg( pp, "Too many volume files" ); return 1; } + if( !open_outstream( true ) ) return 1; + delete_output_on_interrupt = true; + } + } + const long long size = + min( member_size, volume_size - partial_volume_size ); + if( LZ_compress_restart_member( encoder, size ) < 0 ) + { + Pp_show_msg( pp, 0 ); + if( verbosity >= 0 ) + fprintf( stderr, "%s: LZ_compress_restart_member error: %s.\n", + program_name, LZ_strerror( LZ_compress_errno( encoder ) ) ); + return 1; + } + } + } + + if( verbosity >= 1 ) + { + const long long in_size = LZ_compress_total_in_size( encoder ); + const long long out_size = LZ_compress_total_out_size( encoder ); + if( in_size <= 0 || out_size <= 0 ) + fprintf( stderr, " no data compressed.\n" ); + else + fprintf( stderr, "%6.3f:1, %6.3f bits/byte, " + "%5.2f%% saved, %lld in, %lld out.\n", + (double)in_size / out_size, + ( 8.0 * out_size ) / in_size, + 100.0 * ( 1.0 - ( (double)out_size / in_size ) ), + in_size, out_size ); + } + return 0; + } + + +int compress( const long long member_size, const long long volume_size, + const struct Lzma_options * const encoder_options, const int infd, + struct Pretty_print * const pp, const struct stat * const in_statsp ) + { + struct LZ_Encoder * const encoder = + LZ_compress_open( encoder_options->dictionary_size, + encoder_options->match_len_limit, + min( member_size, volume_size ) ); + int retval; + + if( !encoder || LZ_compress_errno( encoder ) != LZ_ok ) + { + if( !encoder || LZ_compress_errno( encoder ) == LZ_mem_error ) + Pp_show_msg( pp, "Not enough memory. Try a smaller dictionary size" ); + else + internal_error( "invalid argument to encoder" ); + retval = 1; + } + else retval = do_compress( encoder, member_size, volume_size, + infd, pp, in_statsp ); + LZ_compress_close( encoder ); + return retval; + } + + +int do_decompress( struct LZ_Decoder * const decoder, const int infd, + struct Pretty_print * const pp, const bool testing ) + { + enum { buffer_size = 65536 }; + uint8_t buffer[buffer_size]; + bool first_member; + + for( first_member = true; ; ) + { + int in_size = min( LZ_decompress_write_size( decoder ), buffer_size ); + if( in_size > 0 ) + { + const int max_in_size = in_size; + in_size = readblock( infd, buffer, max_in_size ); + if( in_size != max_in_size && errno ) + { + Pp_show_msg( pp, 0 ); show_error( "Read error", errno, false ); + return 1; + } + if( in_size > 0 && in_size != LZ_decompress_write( decoder, buffer, in_size ) ) + internal_error( "library error (LZ_decompress_write)" ); + if( in_size < max_in_size ) LZ_decompress_finish( decoder ); + } + int out_size = 0; + while( true ) + { + const int rd = LZ_decompress_read( decoder, buffer, buffer_size ); + if( rd > 0 ) + { + out_size += rd; + if( outfd >= 0 ) + { + const int wr = writeblock( outfd, buffer, rd ); + if( wr != rd ) + { + Pp_show_msg( pp, 0 ); show_error( "Write error", errno, false ); + return 1; + } + } + } + else if( rd < 0 ) { out_size = rd; break; } + if( verbosity >= 1 && LZ_decompress_member_finished( decoder ) == 1 ) + { + const long long data_position = LZ_decompress_data_position( decoder ); + const long long member_size = LZ_decompress_member_position( decoder ); + Pp_show_msg( pp, 0 ); + if( verbosity >= 2 ) + fprintf( stderr, "version %d, dictionary size %7sB. ", + LZ_decompress_member_version( decoder ), + format_num( LZ_decompress_dictionary_size( decoder ) ) ); + if( verbosity >= 3 && data_position > 0 && member_size > 0 ) + fprintf( stderr, "%6.3f:1, %6.3f bits/byte, %5.2f%% saved. ", + (double)data_position / member_size, + ( 8.0 * member_size ) / data_position, + 100.0 * ( 1.0 - ( (double)member_size / data_position ) ) ); + if( verbosity >= 4 ) + fprintf( stderr, "data CRC %08X, data size %9lld, member size %8lld. ", + LZ_decompress_data_crc( decoder ), + data_position, member_size ); + if( testing ) fprintf( stderr, "ok\n" ); + else fprintf( stderr, "done\n" ); + } + if( LZ_decompress_member_finished( decoder ) == 1 ) + { first_member = false; Pp_reset( pp ); } + if( rd <= 0 ) break; + } + if( out_size < 0 ) + { + const enum LZ_Errno lz_errno = LZ_decompress_errno( decoder ); + if( lz_errno == LZ_header_error ) + { + if( !first_member ) break; /* trailing garbage */ + Pp_show_msg( pp, "Error reading member header" ); + return 1; + } + if( lz_errno == LZ_mem_error ) + { + Pp_show_msg( pp, "Not enough memory. Find a machine with more memory" ); + return 1; + } + Pp_show_msg( pp, 0 ); + if( lz_errno == LZ_unexpected_eof ) + { + if( verbosity >= 0 ) + fprintf( stderr, "File ends unexpectedly at pos %lld\n", + LZ_decompress_total_in_size( decoder ) ); + return 2; + } + if( verbosity >= 0 ) + fprintf( stderr, "LZ_decompress_read error: %s.\n", + LZ_strerror( LZ_decompress_errno( decoder ) ) ); + return 1; + } + if( LZ_decompress_finished( decoder ) == 1 ) break; + if( in_size == 0 && out_size == 0 ) + internal_error( "library error (LZ_decompress_read)" ); + } + return 0; + } + + +int decompress( const int infd, struct Pretty_print * const pp, + const bool testing ) + { + struct LZ_Decoder * const decoder = LZ_decompress_open(); + int retval; + + if( !decoder || LZ_decompress_errno( decoder ) != LZ_ok ) + { + Pp_show_msg( pp, "Not enough memory. Find a machine with more memory" ); + retval = 1; + } + else retval = do_decompress( decoder, infd, pp, testing ); + + LZ_decompress_close( decoder ); + return retval; + } + + +void signal_handler( int sig ) + { + sig = 0; /* keep compiler happy */ + show_error( "Control-C or similar caught, quitting.", 0, false ); + cleanup_and_fail( 1 ); + } + + +static void set_signals( void ) + { + signal( SIGHUP, signal_handler ); + signal( SIGINT, signal_handler ); + signal( SIGTERM, signal_handler ); + } + + +int main( const int argc, const char * const argv[] ) + { + /* Mapping from gzip/bzip2 style 1..9 compression modes + to the corresponding LZMA compression modes. */ + const struct Lzma_options option_mapping[] = + { + { 1 << 20, 5 }, /* -0 */ + { 1 << 20, 5 }, /* -1 */ + { 3 << 19, 6 }, /* -2 */ + { 1 << 21, 8 }, /* -3 */ + { 3 << 20, 12 }, /* -4 */ + { 1 << 22, 20 }, /* -5 */ + { 1 << 23, 36 }, /* -6 */ + { 1 << 24, 68 }, /* -7 */ + { 3 << 23, 132 }, /* -8 */ + { 1 << 25, 273 } }; /* -9 */ + struct Lzma_options encoder_options = option_mapping[6]; /* default = "-6" */ + long long member_size = LLONG_MAX; + long long volume_size = LLONG_MAX; + const char * input_filename = ""; + const char * default_output_filename = ""; + const char ** filenames = 0; + int num_filenames = 0; + int infd = -1; + enum Mode program_mode = m_compress; + int argind = 0; + int retval = 0; + int i; + bool filenames_given = false; + bool force = false; + bool keep_input_files = false; + bool recompress = false; + bool to_stdout = false; + struct Pretty_print pp; + + const struct ap_Option options[] = + { + { '0', "fast", ap_no }, + { '1', 0, ap_no }, + { '2', 0, ap_no }, + { '3', 0, ap_no }, + { '4', 0, ap_no }, + { '5', 0, ap_no }, + { '6', 0, ap_no }, + { '7', 0, ap_no }, + { '8', 0, ap_no }, + { '9', "best", ap_no }, + { 'b', "member-size", ap_yes }, + { 'c', "stdout", ap_no }, + { 'd', "decompress", ap_no }, + { 'f', "force", ap_no }, + { 'F', "recompress", ap_no }, + { 'h', "help", ap_no }, + { 'k', "keep", ap_no }, + { 'm', "match-length", ap_yes }, + { 'o', "output", ap_yes }, + { 'q', "quiet", ap_no }, + { 's', "dictionary-size", ap_yes }, + { 'S', "volume-size", ap_yes }, + { 't', "test", ap_no }, + { 'v', "verbose", ap_no }, + { 'V', "version", ap_no }, + { 0 , 0, ap_no } }; + + struct Arg_parser parser; + + invocation_name = argv[0]; + if( LZ_version()[0] != LZ_version_string[0] ) + internal_error( "bad library version" ); + if( strcmp( PROGVERSION, LZ_version_string ) ) + internal_error( "bad library version_string" ); + + if( !ap_init( &parser, argc, argv, options, 0 ) ) + { show_error( "Memory exhausted.", 0, false ); return 1; } + if( ap_error( &parser ) ) /* bad option */ + { show_error( ap_error( &parser ), 0, true ); return 1; } + + for( ; argind < ap_arguments( &parser ); ++argind ) + { + const int code = ap_code( &parser, argind ); + const char * const arg = ap_argument( &parser, argind ); + if( !code ) break; /* no more options */ + switch( code ) + { + case '0': + case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + encoder_options = option_mapping[code-'0']; break; + case 'b': member_size = getnum( arg, 100000, LLONG_MAX / 2 ); break; + case 'c': to_stdout = true; break; + case 'd': program_mode = m_decompress; break; + case 'f': force = true; break; + case 'F': recompress = true; break; + case 'h': show_help(); return 0; + case 'k': keep_input_files = true; break; + case 'm': encoder_options.match_len_limit = + getnum( arg, LZ_min_match_len_limit(), + LZ_max_match_len_limit() ); break; + case 'o': default_output_filename = arg; break; + case 'q': verbosity = -1; break; + case 's': encoder_options.dictionary_size = get_dict_size( arg ); + break; + case 'S': volume_size = getnum( arg, 100000, LLONG_MAX / 2 ); break; + case 't': program_mode = m_test; break; + case 'v': if( verbosity < 4 ) ++verbosity; break; + case 'V': show_version(); return 0; + default : internal_error( "uncaught option" ); + } + } /* end process options */ + +#if defined(__OS2__) + _fsetmode( stdin, "b" ); + _fsetmode( stdout, "b" ); +#endif + + if( program_mode == m_test ) + outfd = -1; + + for( ; argind < ap_arguments( &parser ); ++argind ) + { + if( strcmp( ap_argument( &parser, argind ), "-" ) ) + filenames_given = true; + ++num_filenames; + filenames = resize_buffer( filenames, num_filenames * sizeof (char *) ); + filenames[num_filenames-1] = ap_argument( &parser, argind ); + } + + if( num_filenames == 0 ) + { + ++num_filenames; + filenames = resize_buffer( filenames, sizeof (char *) ); + filenames[num_filenames-1] = "-"; + } + if( !to_stdout && program_mode != m_test && + ( filenames_given || default_output_filename[0] ) ) + set_signals(); + + Pp_init( &pp, filenames, num_filenames ); + + output_filename = resize_buffer( output_filename, 1 ); + for( i = 0; i < num_filenames; ++i ) + { + int tmp; + struct stat in_stats; + const struct stat * in_statsp; + output_filename[0] = 0; + + if( !filenames[i][0] || !strcmp( filenames[i], "-" ) ) + { + input_filename = ""; + infd = STDIN_FILENO; + if( program_mode != m_test ) + { + if( to_stdout || !default_output_filename[0] ) + outfd = STDOUT_FILENO; + else + { + if( program_mode == m_compress ) + set_c_outname( default_output_filename, volume_size != LLONG_MAX ); + else + { + output_filename = resize_buffer( output_filename, + strlen( default_output_filename ) + 1 ); + strcpy( output_filename, default_output_filename ); + } + outfd_mode = all_rw; + if( !open_outstream( force ) ) + { + if( outfd == -1 && retval < 1 ) retval = 1; + close( infd ); infd = -1; + continue; + } + } + } + } + else + { + const int eindex = extension_index( filenames[i] ); + input_filename = filenames[i]; + infd = open_instream( input_filename, &in_stats, program_mode, + eindex, recompress, to_stdout ); + if( infd < 0 ) { if( retval < 1 ) retval = 1; continue; } + if( program_mode != m_test ) + { + if( to_stdout ) outfd = STDOUT_FILENO; + else + { + if( program_mode == m_compress ) + set_c_outname( input_filename, volume_size != LLONG_MAX ); + else set_d_outname( input_filename, eindex ); + outfd_mode = usr_rw; + if( !open_outstream( force ) ) + { + if( outfd == -1 && retval < 1 ) retval = 1; + close( infd ); infd = -1; + continue; + } + } + } + } + + if( !check_tty( infd, program_mode ) ) return 1; + + if( output_filename[0] && !to_stdout && program_mode != m_test ) + delete_output_on_interrupt = true; + in_statsp = input_filename[0] ? &in_stats : 0; + Pp_set_name( &pp, input_filename ); + if( program_mode == m_compress ) + tmp = compress( member_size, volume_size, &encoder_options, infd, + &pp, in_statsp ); + else + tmp = decompress( infd, &pp, program_mode == m_test ); + if( tmp > retval ) retval = tmp; + if( tmp && program_mode != m_test ) cleanup_and_fail( retval ); + + if( delete_output_on_interrupt ) + close_and_set_permissions( in_statsp ); + if( input_filename[0] ) + { + close( infd ); infd = -1; + if( !keep_input_files && !to_stdout && program_mode != m_test ) + remove( input_filename ); + } + } + if( outfd >= 0 && close( outfd ) != 0 ) + { + show_error( "Can't close stdout", errno, false ); + if( retval < 1 ) retval = 1; + } + free( output_filename ); + free( filenames ); + ap_free( &parser ); + return retval; + } diff --git a/main.cc b/main.cc deleted file mode 100644 index 5c5d572..0000000 --- a/main.cc +++ /dev/null @@ -1,966 +0,0 @@ -/* Minilzip - A test program for the lzlib library - Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <http://www.gnu.org/licenses/>. -*/ -/* - Return values: 0 for a normal exit, 1 for environmental problems - (file not found, invalid flags, I/O errors, etc), 2 to indicate a - corrupt or invalid input file, 3 for an internal consistency error - (eg, bug) which caused minilzip to panic. -*/ - -#define _FILE_OFFSET_BITS 64 - -#include <algorithm> -#include <cerrno> -#include <climits> -#include <csignal> -#include <cstdio> -#include <cstdlib> -#include <cstring> -#include <string> -#include <vector> -#include <fcntl.h> -#include <stdint.h> -#include <unistd.h> -#include <utime.h> -#include <sys/stat.h> - -#include "arg_parser.h" -#include "lzlib.h" - -#if CHAR_BIT != 8 -#error "Environments where CHAR_BIT != 8 are not supported." -#endif - -#ifndef LLONG_MAX -#define LLONG_MAX 0x7FFFFFFFFFFFFFFFLL -#endif -#ifndef LLONG_MIN -#define LLONG_MIN (-LLONG_MAX - 1LL) -#endif -#ifndef ULLONG_MAX -#define ULLONG_MAX 0xFFFFFFFFFFFFFFFFULL -#endif - -void show_error( const char * const msg, const int errcode = 0, const bool help = false ) throw(); -void internal_error( const char * const msg ); -int readblock( const int fd, uint8_t * const buf, const int size ) throw(); -int writeblock( const int fd, const uint8_t * const buf, const int size ) throw(); - - -namespace { - -const char * const Program_name = "Minilzip"; -const char * const program_name = "minilzip"; -const char * const program_year = "2011"; -const char * invocation_name = 0; - -#ifdef O_BINARY -const int o_binary = O_BINARY; -#else -const int o_binary = 0; -#endif - -struct { const char * from; const char * to; } const known_extensions[] = { - { ".lz", "" }, - { ".tlz", ".tar" }, - { 0, 0 } }; - -struct Lzma_options - { - int dictionary_size; // 4KiB..512MiB - int match_len_limit; // 5..273 - }; - -enum Mode { m_compress, m_decompress, m_test }; - -std::string output_filename; -int outfd = -1; -int verbosity = 0; -mode_t outfd_mode = S_IRUSR | S_IWUSR; -bool delete_output_on_interrupt = false; - -class Pretty_print - { - const char * const stdin_name; - unsigned int longest_name; - std::string name_; - mutable bool first_post; - -public: - Pretty_print( const std::vector< std::string > & filenames ) - : stdin_name( "(stdin)" ), longest_name( 0 ), first_post( false ) - { - const unsigned int stdin_name_len = std::strlen( stdin_name ); - for( unsigned int i = 0; i < filenames.size(); ++i ) - { - const std::string & s = filenames[i]; - const unsigned int len = ( ( s == "-" ) ? stdin_name_len : s.size() ); - if( len > longest_name ) longest_name = len; - } - if( longest_name == 0 ) longest_name = stdin_name_len; - } - - void set_name( const std::string & filename ) - { - if( filename.size() && filename != "-" ) name_ = filename; - else name_ = stdin_name; - first_post = true; - } - - void reset() const throw() { if( name_.size() ) first_post = true; } - const char * name() const throw() { return name_.c_str(); } - void operator()( const char * const msg = 0 ) const throw(); - }; - - -void show_help() throw() - { - std::printf( "%s - A test program for the lzlib library.\n", Program_name ); - std::printf( "\nUsage: %s [options] [files]\n", invocation_name ); - std::printf( "\nOptions:\n" ); - std::printf( " -h, --help display this help and exit\n" ); - std::printf( " -V, --version output version information and exit\n" ); - std::printf( " -b, --member-size=<n> set member size limit in bytes\n" ); - std::printf( " -c, --stdout send output to standard output\n" ); - std::printf( " -d, --decompress decompress\n" ); - std::printf( " -f, --force overwrite existing output files\n" ); - std::printf( " -F, --recompress force recompression of compressed files\n" ); - std::printf( " -k, --keep keep (don't delete) input files\n" ); - std::printf( " -m, --match-length=<n> set match length limit in bytes [36]\n" ); - std::printf( " -o, --output=<file> if reading stdin, place the output into <file>\n" ); - std::printf( " -q, --quiet suppress all messages\n" ); - std::printf( " -s, --dictionary-size=<n> set dictionary size limit in bytes [8MiB]\n" ); - std::printf( " -S, --volume-size=<n> set volume size limit in bytes\n" ); - std::printf( " -t, --test test compressed file integrity\n" ); - std::printf( " -v, --verbose be verbose (a 2nd -v gives more)\n" ); - std::printf( " -1 .. -9 set compression level [default 6]\n" ); - std::printf( " --fast alias for -1\n" ); - std::printf( " --best alias for -9\n" ); - std::printf( "If no file names are given, %s compresses or decompresses\n", program_name ); - std::printf( "from standard input to standard output.\n" ); - std::printf( "Numbers may be followed by a multiplier: k = kB = 10^3 = 1000,\n" ); - std::printf( "Ki = KiB = 2^10 = 1024, M = 10^6, Mi = 2^20, G = 10^9, Gi = 2^30, etc...\n" ); - std::printf( "\nReport bugs to lzip-bug@nongnu.org\n" ); - std::printf( "Lzlib home page: http://www.nongnu.org/lzip/lzlib.html\n" ); - } - - -void show_version() throw() - { - std::printf( "%s %s\n", Program_name, PROGVERSION ); - std::printf( "Copyright (C) %s Antonio Diaz Diaz.\n", program_year ); - std::printf( "License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>\n" ); - std::printf( "This is free software: you are free to change and redistribute it.\n" ); - std::printf( "There is NO WARRANTY, to the extent permitted by law.\n" ); - } - - -const char * format_num( long long num ) throw() - { - const char * const prefix[8] = - { "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi" }; - enum { buf_size = 16, factor = 1024 }; - static char buf[buf_size]; - const char *p = ""; - - for( int i = 0; i < 8 && ( llabs( num ) > 9999 || - ( llabs( num ) >= factor && num % factor == 0 ) ); ++i ) - { num /= factor; p = prefix[i]; } - snprintf( buf, buf_size, "%lld %s", num, p ); - return buf; - } - - -long long getnum( const char * const ptr, - const long long llimit = LLONG_MIN + 1, - const long long ulimit = LLONG_MAX ) throw() - { - errno = 0; - char *tail; - long long result = strtoll( ptr, &tail, 0 ); - if( tail == ptr ) - { - show_error( "Bad or missing numerical argument.", 0, true ); - std::exit( 1 ); - } - - if( !errno && tail[0] ) - { - int factor = ( tail[1] == 'i' ) ? 1024 : 1000; - int exponent = 0; - bool bad_multiplier = false; - switch( tail[0] ) - { - case ' ': break; - case 'Y': exponent = 8; break; - case 'Z': exponent = 7; break; - case 'E': exponent = 6; break; - case 'P': exponent = 5; break; - case 'T': exponent = 4; break; - case 'G': exponent = 3; break; - case 'M': exponent = 2; break; - case 'K': if( factor == 1024 ) exponent = 1; else bad_multiplier = true; - break; - case 'k': if( factor == 1000 ) exponent = 1; else bad_multiplier = true; - break; - default : bad_multiplier = true; - } - if( bad_multiplier ) - { - show_error( "Bad multiplier in numerical argument.", 0, true ); - std::exit( 1 ); - } - for( int i = 0; i < exponent; ++i ) - { - if( LLONG_MAX / factor >= llabs( result ) ) result *= factor; - else { errno = ERANGE; break; } - } - } - if( !errno && ( result < llimit || result > ulimit ) ) errno = ERANGE; - if( errno ) - { - show_error( "Numerical argument out of limits." ); - std::exit( 1 ); - } - return result; - } - - -int get_dict_size( const char * const arg ) throw() - { - char *tail; - int bits = std::strtol( arg, &tail, 0 ); - if( bits >= LZ_min_dictionary_bits() && - bits <= LZ_max_dictionary_bits() && *tail == 0 ) - return ( 1 << bits ); - return getnum( arg, LZ_min_dictionary_size(), LZ_max_dictionary_size() ); - } - - -int extension_index( const std::string & name ) throw() - { - for( int i = 0; known_extensions[i].from; ++i ) - { - const std::string ext( known_extensions[i].from ); - if( name.size() > ext.size() && - name.compare( name.size() - ext.size(), ext.size(), ext ) == 0 ) - return i; - } - return -1; - } - - -int open_instream( const std::string & name, struct stat * const in_statsp, - const Mode program_mode, const int eindex, - const bool recompress, const bool to_stdout ) throw() - { - int infd = -1; - if( program_mode == m_compress && !recompress && eindex >= 0 ) - { - if( verbosity >= 0 ) - std::fprintf( stderr, "%s: Input file `%s' already has `%s' suffix.\n", - program_name, name.c_str(), - known_extensions[eindex].from ); - } - else - { - infd = open( name.c_str(), O_RDONLY | o_binary ); - if( infd < 0 ) - { - if( verbosity >= 0 ) - std::fprintf( stderr, "%s: Can't open input file `%s': %s.\n", - program_name, name.c_str(), std::strerror( errno ) ); - } - else - { - const int i = fstat( infd, in_statsp ); - const mode_t & mode = in_statsp->st_mode; - const bool can_read = ( i == 0 && - ( S_ISBLK( mode ) || S_ISCHR( mode ) || - S_ISFIFO( mode ) || S_ISSOCK( mode ) ) ); - if( i != 0 || ( !S_ISREG( mode ) && ( !to_stdout || !can_read ) ) ) - { - if( verbosity >= 0 ) - std::fprintf( stderr, "%s: Input file `%s' is not a regular file%s.\n", - program_name, name.c_str(), - ( can_read && !to_stdout ) ? - " and `--stdout' was not specified" : "" ); - close( infd ); - infd = -1; - } - } - } - return infd; - } - - -void set_c_outname( const std::string & name, const bool multifile ) throw() - { - output_filename = name; - if( multifile ) output_filename += "00001"; - output_filename += known_extensions[0].from; - } - - -void set_d_outname( const std::string & name, const int i ) throw() - { - if( i >= 0 ) - { - const std::string from( known_extensions[i].from ); - if( name.size() > from.size() ) - { - output_filename.assign( name, 0, name.size() - from.size() ); - output_filename += known_extensions[i].to; - return; - } - } - output_filename = name; output_filename += ".out"; - if( verbosity >= 1 ) - std::fprintf( stderr, "%s: Can't guess original name for `%s' -- using `%s'.\n", - program_name, name.c_str(), output_filename.c_str() ); - } - - -bool open_outstream( const bool force ) throw() - { - int flags = O_CREAT | O_WRONLY | o_binary; - if( force ) flags |= O_TRUNC; else flags |= O_EXCL; - - outfd = open( output_filename.c_str(), flags, outfd_mode ); - if( outfd < 0 && verbosity >= 0 ) - { - if( errno == EEXIST ) - std::fprintf( stderr, "%s: Output file `%s' already exists, skipping.\n", - program_name, output_filename.c_str() ); - else - std::fprintf( stderr, "%s: Can't create output file `%s': %s.\n", - program_name, output_filename.c_str(), std::strerror( errno ) ); - } - return ( outfd >= 0 ); - } - - -bool check_tty( const int infd, const Mode program_mode ) throw() - { - if( program_mode == m_compress && outfd >= 0 && isatty( outfd ) ) - { - show_error( "I won't write compressed data to a terminal.", 0, true ); - return false; - } - if( ( program_mode == m_decompress || program_mode == m_test ) && - isatty( infd ) ) - { - show_error( "I won't read compressed data from a terminal.", 0, true ); - return false; - } - return true; - } - - -void cleanup_and_fail( const int retval ) throw() - { - if( delete_output_on_interrupt ) - { - delete_output_on_interrupt = false; - if( verbosity >= 0 ) - std::fprintf( stderr, "%s: Deleting output file `%s', if it exists.\n", - program_name, output_filename.c_str() ); - if( outfd >= 0 ) { close( outfd ); outfd = -1; } - if( std::remove( output_filename.c_str() ) != 0 && errno != ENOENT ) - show_error( "WARNING: deletion of output file (apparently) failed." ); - } - std::exit( retval ); - } - - - // Set permissions, owner and times. -void close_and_set_permissions( const struct stat * const in_statsp ) - { - bool error = false; - if( in_statsp ) - { - if( ( fchown( outfd, in_statsp->st_uid, in_statsp->st_gid ) != 0 && - errno != EPERM ) || - fchmod( outfd, in_statsp->st_mode ) != 0 ) - error = true; - // fchown will in many cases return with EPERM, which can be safely ignored. - } - if( close( outfd ) == 0 ) outfd = -1; - else cleanup_and_fail( 1 ); - delete_output_on_interrupt = false; - if( !in_statsp ) return; - if( !error ) - { - struct utimbuf t; - t.actime = in_statsp->st_atime; - t.modtime = in_statsp->st_mtime; - if( utime( output_filename.c_str(), &t ) != 0 ) error = true; - } - if( error ) - { - show_error( "Can't change output file attributes." ); - cleanup_and_fail( 1 ); - } - } - - -bool next_filename() - { - const unsigned int len = std::strlen( known_extensions[0].from ); - if( output_filename.size() >= len + 5 ) // "*00001.lz" - for( int i = output_filename.size() - len - 1, j = 0; j < 5; --i, ++j ) - { - if( output_filename[i] < '9' ) { ++output_filename[i]; return true; } - else output_filename[i] = '0'; - } - return false; - } - - -int do_compress( LZ_Encoder * const encoder, const long long member_size, - const long long volume_size, const int infd, - const Pretty_print & pp, const struct stat * const in_statsp ) - { - long long partial_volume_size = 0; - const int buffer_size = 65536; - uint8_t buffer[buffer_size]; - - if( verbosity >= 1 ) pp(); - while( true ) - { - int in_size = 0; - while( LZ_compress_write_size( encoder ) > 0 ) - { - const int size = std::min( LZ_compress_write_size( encoder ), - buffer_size ); - const int rd = readblock( infd, buffer, size ); - if( rd != size && errno ) - { pp(); show_error( "Read error", errno ); return 1; } - if( rd > 0 && rd != LZ_compress_write( encoder, buffer, rd ) ) - internal_error( "library error (LZ_compress_write)" ); - if( rd < size ) LZ_compress_finish( encoder ); -// else LZ_compress_sync_flush( encoder ); - in_size += rd; - } - const int out_size = LZ_compress_read( encoder, buffer, buffer_size ); - if( out_size < 0 ) - { - pp(); - if( verbosity >= 0 ) - std::fprintf( stderr, "%s: LZ_compress_read error: %s.\n", - program_name, LZ_strerror( LZ_compress_errno( encoder ) ) ); - return 1; - } - else if( out_size > 0 ) - { - const int wr = writeblock( outfd, buffer, out_size ); - if( wr != out_size ) - { pp(); show_error( "Write error", errno ); return 1; } - } - else if( in_size == 0 ) internal_error( "library error (LZ_compress_read)" ); - if( LZ_compress_member_finished( encoder ) ) - { - if( LZ_compress_finished( encoder ) == 1 ) break; - partial_volume_size += LZ_compress_member_position( encoder ); - if( partial_volume_size >= volume_size - LZ_min_dictionary_size() ) - { - partial_volume_size = 0; - if( delete_output_on_interrupt ) - { - close_and_set_permissions( in_statsp ); - if( !next_filename() ) - { pp( "Too many volume files." ); return 1; } - if( !open_outstream( true ) ) return 1; - delete_output_on_interrupt = true; - } - } - const long long size = - std::min( member_size, volume_size - partial_volume_size ); - if( LZ_compress_restart_member( encoder, size ) < 0 ) - { - pp(); - if( verbosity >= 0 ) - std::fprintf( stderr, "%s: LZ_compress_restart_member error: %s.\n", - program_name, LZ_strerror( LZ_compress_errno( encoder ) ) ); - return 1; - } - } - } - - if( verbosity >= 1 ) - { - const long long in_size = LZ_compress_total_in_size( encoder ); - const long long out_size = LZ_compress_total_out_size( encoder ); - if( in_size <= 0 || out_size <= 0 ) - std::fprintf( stderr, "No data compressed.\n" ); - else - std::fprintf( stderr, "%6.3f:1, %6.3f bits/byte, " - "%5.2f%% saved, %lld in, %lld out.\n", - (double)in_size / out_size, - ( 8.0 * out_size ) / in_size, - 100.0 * ( 1.0 - ( (double)out_size / in_size ) ), - in_size, out_size ); - } - return 0; - } - - -int compress( const long long member_size, const long long volume_size, - const Lzma_options & encoder_options, const int infd, - const Pretty_print & pp, const struct stat * const in_statsp ) - { - LZ_Encoder * const encoder = - LZ_compress_open( encoder_options.dictionary_size, - encoder_options.match_len_limit, - std::min( member_size, volume_size ) ); - int retval; - - if( !encoder || LZ_compress_errno( encoder ) != LZ_ok ) - { - if( !encoder || LZ_compress_errno( encoder ) == LZ_mem_error ) - pp( "Not enough memory. Try a smaller dictionary size" ); - else - internal_error( "invalid argument to encoder" ); - retval = 1; - } - else retval = do_compress( encoder, member_size, volume_size, - infd, pp, in_statsp ); - LZ_compress_close( encoder ); - return retval; - } - - -int do_decompress( LZ_Decoder * const decoder, const int infd, - const Pretty_print & pp, const bool testing ) - { - const int buffer_size = 65536; - uint8_t buffer[buffer_size]; - - while( true ) - { - int in_size = std::min( LZ_decompress_write_size( decoder ), buffer_size ); - if( in_size > 0 ) - { - const int max_in_size = in_size; - in_size = readblock( infd, buffer, max_in_size ); - if( in_size != max_in_size && errno ) - { pp(); show_error( "Read error", errno ); return 1; } - if( in_size > 0 && in_size != LZ_decompress_write( decoder, buffer, in_size ) ) - internal_error( "library error (LZ_decompress_write)" ); - if( in_size < max_in_size ) LZ_decompress_finish( decoder ); - } - int out_size = 0; - while( true ) - { - const int rd = LZ_decompress_read( decoder, buffer, buffer_size ); - if( rd > 0 ) - { - out_size += rd; - if( outfd >= 0 ) - { - const int wr = writeblock( outfd, buffer, rd ); - if( wr != rd ) - { pp(); show_error( "Write error", errno ); return 1; } - } - } - else if( rd < 0 ) { out_size = rd; break; } - if( verbosity >= 1 && LZ_decompress_member_finished( decoder ) == 1 ) - { - const long long data_position = LZ_decompress_data_position( decoder ); - const long long member_size = LZ_decompress_member_position( decoder ); - pp(); - if( verbosity >= 2 ) - std::fprintf( stderr, "version %d, dictionary size %7sB. ", - LZ_decompress_member_version( decoder ), - format_num( LZ_decompress_dictionary_size( decoder ) ) ); - if( verbosity >= 3 && data_position > 0 && member_size > 0 ) - std::fprintf( stderr, "%6.3f:1, %6.3f bits/byte, %5.2f%% saved. ", - (double)data_position / member_size, - ( 8.0 * member_size ) / data_position, - 100.0 * ( 1.0 - ( (double)member_size / data_position ) ) ); - if( verbosity >= 4 ) - std::fprintf( stderr, "data CRC %08X, data size %9lld, member size %8lld. ", - LZ_decompress_data_crc( decoder ), - data_position, member_size ); - if( testing ) std::fprintf( stderr, "ok\n" ); - else std::fprintf( stderr, "done\n" ); - pp.reset(); - } - if( rd <= 0 ) break; - } - if( out_size < 0 ) - { - const LZ_Errno lz_errno = LZ_decompress_errno( decoder ); - if( lz_errno == LZ_header_error ) - { - if( LZ_decompress_total_out_size( decoder ) > 0 ) - break; // trailing garbage - pp( "Error reading member header" ); - return 1; - } - if( lz_errno == LZ_mem_error ) - { - pp( "Not enough memory. Find a machine with more memory" ); - return 1; - } - pp(); - if( lz_errno == LZ_unexpected_eof ) - { - if( verbosity >= 0 ) - std::fprintf( stderr, "File ends unexpectedly at pos %lld\n", - LZ_decompress_total_in_size( decoder ) ); - return 2; - } - if( verbosity >= 0 ) - std::fprintf( stderr, "LZ_decompress_read error: %s.\n", - LZ_strerror( LZ_decompress_errno( decoder ) ) ); - return 1; - } - if( LZ_decompress_finished( decoder ) == 1 ) break; - if( in_size == 0 && out_size == 0 ) - internal_error( "library error (LZ_decompress_read)" ); - } - return 0; - } - - -int decompress( const int infd, const Pretty_print & pp, - const bool testing ) - { - LZ_Decoder * const decoder = LZ_decompress_open(); - int retval; - - if( !decoder || LZ_decompress_errno( decoder ) != LZ_ok ) - { - pp( "Not enough memory. Find a machine with more memory" ); - retval = 1; - } - else retval = do_decompress( decoder, infd, pp, testing ); - - LZ_decompress_close( decoder ); - return retval; - } - - -extern "C" void signal_handler( int ) throw() - { - show_error( "Control-C or similar caught, quitting." ); - cleanup_and_fail( 1 ); - } - - -void set_signals() throw() - { - std::signal( SIGHUP, signal_handler ); - std::signal( SIGINT, signal_handler ); - std::signal( SIGTERM, signal_handler ); - } - -} // end namespace - - -void Pretty_print::operator()( const char * const msg ) const throw() - { - if( verbosity >= 0 ) - { - if( first_post ) - { - first_post = false; - std::fprintf( stderr, " %s: ", name_.c_str() ); - for( unsigned int i = 0; i < longest_name - name_.size(); ++i ) - std::fprintf( stderr, " " ); - if( !msg ) std::fflush( stderr ); - } - if( msg ) std::fprintf( stderr, "%s.\n", msg ); - } - } - - -void show_error( const char * const msg, const int errcode, const bool help ) throw() - { - if( verbosity >= 0 ) - { - if( msg && msg[0] ) - { - std::fprintf( stderr, "%s: %s", program_name, msg ); - if( errcode > 0 ) - std::fprintf( stderr, ": %s", std::strerror( errcode ) ); - std::fprintf( stderr, "\n" ); - } - if( help && invocation_name && invocation_name[0] ) - std::fprintf( stderr, "Try `%s --help' for more information.\n", - invocation_name ); - } - } - - -void internal_error( const char * const msg ) - { - if( verbosity >= 0 ) - std::fprintf( stderr, "%s: internal error: %s.\n", program_name, msg ); - std::exit( 3 ); - } - - -// Returns the number of bytes really read. -// If (returned value < size) and (errno == 0), means EOF was reached. -// -int readblock( const int fd, uint8_t * const buf, const int size ) throw() - { - int rest = size; - errno = 0; - while( rest > 0 ) - { - errno = 0; - const int n = read( fd, buf + size - rest, rest ); - if( n > 0 ) rest -= n; - else if( n == 0 ) break; - else if( errno != EINTR && errno != EAGAIN ) break; - } - return ( rest > 0 ) ? size - rest : size; - } - - -// Returns the number of bytes really written. -// If (returned value < size), it is always an error. -// -int writeblock( const int fd, const uint8_t * const buf, const int size ) throw() - { - int rest = size; - errno = 0; - while( rest > 0 ) - { - errno = 0; - const int n = write( fd, buf + size - rest, rest ); - if( n > 0 ) rest -= n; - else if( errno && errno != EINTR && errno != EAGAIN ) break; - } - return ( rest > 0 ) ? size - rest : size; - } - - -int main( const int argc, const char * const argv[] ) - { - // Mapping from gzip/bzip2 style 1..9 compression modes - // to the corresponding LZMA compression modes. - const Lzma_options option_mapping[] = - { - { 1 << 20, 5 }, // -0 - { 1 << 20, 5 }, // -1 - { 3 << 19, 6 }, // -2 - { 1 << 21, 8 }, // -3 - { 3 << 20, 12 }, // -4 - { 1 << 22, 20 }, // -5 - { 1 << 23, 36 }, // -6 - { 1 << 24, 68 }, // -7 - { 3 << 23, 132 }, // -8 - { 1 << 25, 273 } }; // -9 - Lzma_options encoder_options = option_mapping[6]; // default = "-6" - long long member_size = LLONG_MAX; - long long volume_size = LLONG_MAX; - int infd = -1; - Mode program_mode = m_compress; - bool force = false; - bool keep_input_files = false; - bool recompress = false; - bool to_stdout = false; - std::string input_filename; - std::string default_output_filename; - std::vector< std::string > filenames; - invocation_name = argv[0]; - - if( LZ_version()[0] != LZ_version_string[0] ) - internal_error( "bad library version" ); - - if( std::strcmp( PROGVERSION, LZ_version_string ) ) - internal_error( "bad library version_string" ); - - const Arg_parser::Option options[] = - { - { '0', "fast", Arg_parser::no }, - { '1', 0, Arg_parser::no }, - { '2', 0, Arg_parser::no }, - { '3', 0, Arg_parser::no }, - { '4', 0, Arg_parser::no }, - { '5', 0, Arg_parser::no }, - { '6', 0, Arg_parser::no }, - { '7', 0, Arg_parser::no }, - { '8', 0, Arg_parser::no }, - { '9', "best", Arg_parser::no }, - { 'b', "member-size", Arg_parser::yes }, - { 'c', "stdout", Arg_parser::no }, - { 'd', "decompress", Arg_parser::no }, - { 'e', "extreme", Arg_parser::no }, - { 'f', "force", Arg_parser::no }, - { 'F', "recompress", Arg_parser::no }, - { 'h', "help", Arg_parser::no }, - { 'k', "keep", Arg_parser::no }, - { 'm', "match-length", Arg_parser::yes }, - { 'o', "output", Arg_parser::yes }, - { 'q', "quiet", Arg_parser::no }, - { 's', "dictionary-size", Arg_parser::yes }, - { 'S', "volume-size", Arg_parser::yes }, - { 't', "test", Arg_parser::no }, - { 'v', "verbose", Arg_parser::no }, - { 'V', "version", Arg_parser::no }, - { 0 , 0, Arg_parser::no } }; - - const Arg_parser parser( argc, argv, options ); - if( parser.error().size() ) // bad option - { show_error( parser.error().c_str(), 0, true ); return 1; } - - int argind = 0; - for( ; argind < parser.arguments(); ++argind ) - { - const int code = parser.code( argind ); - if( !code ) break; // no more options - const char * const arg = parser.argument( argind ).c_str(); - switch( code ) - { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - encoder_options = option_mapping[code-'0']; break; - case 'b': member_size = getnum( arg, 100000, LLONG_MAX / 2 ); break; - case 'c': to_stdout = true; break; - case 'd': program_mode = m_decompress; break; - case 'e': break; // ignored by now - case 'f': force = true; break; - case 'F': recompress = true; break; - case 'h': show_help(); return 0; - case 'k': keep_input_files = true; break; - case 'm': encoder_options.match_len_limit = - getnum( arg, LZ_min_match_len_limit(), - LZ_max_match_len_limit() ); break; - case 'o': default_output_filename = arg; break; - case 'q': verbosity = -1; break; - case 's': encoder_options.dictionary_size = get_dict_size( arg ); - break; - case 'S': volume_size = getnum( arg, 100000, LLONG_MAX / 2 ); break; - case 't': program_mode = m_test; break; - case 'v': if( verbosity < 4 ) ++verbosity; break; - case 'V': show_version(); return 0; - default : internal_error( "uncaught option" ); - } - } // end process options - -#if defined(__OS2__) - _fsetmode( stdin, "b" ); - _fsetmode( stdout, "b" ); -#endif - - bool filenames_given = false; - for( ; argind < parser.arguments(); ++argind ) - { - if( parser.argument( argind ) != "-" ) filenames_given = true; - filenames.push_back( parser.argument( argind ) ); - } - - if( filenames.empty() ) filenames.push_back("-"); - if( !to_stdout && program_mode != m_test && - ( filenames_given || default_output_filename.size() ) ) - set_signals(); - - Pretty_print pp( filenames ); - if( program_mode == m_test ) - outfd = -1; - - int retval = 0; - for( unsigned int i = 0; i < filenames.size(); ++i ) - { - struct stat in_stats; - output_filename.clear(); - - if( !filenames[i].size() || filenames[i] == "-" ) - { - input_filename.clear(); - infd = STDIN_FILENO; - if( program_mode != m_test ) - { - if( to_stdout || !default_output_filename.size() ) - outfd = STDOUT_FILENO; - else - { - if( program_mode == m_compress ) - set_c_outname( default_output_filename, volume_size != LLONG_MAX ); - else output_filename = default_output_filename; - outfd_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH; - if( !open_outstream( force ) ) - { - if( outfd == -1 && retval < 1 ) retval = 1; - close( infd ); infd = -1; - continue; - } - } - } - } - else - { - input_filename = filenames[i]; - const int eindex = extension_index( input_filename ); - infd = open_instream( input_filename, &in_stats, program_mode, - eindex, recompress, to_stdout ); - if( infd < 0 ) { if( retval < 1 ) retval = 1; continue; } - if( program_mode != m_test ) - { - if( to_stdout ) outfd = STDOUT_FILENO; - else - { - if( program_mode == m_compress ) - set_c_outname( input_filename, volume_size != LLONG_MAX ); - else set_d_outname( input_filename, eindex ); - outfd_mode = S_IRUSR | S_IWUSR; - if( !open_outstream( force ) ) - { - if( outfd == -1 && retval < 1 ) retval = 1; - close( infd ); infd = -1; - continue; - } - } - } - } - - if( !check_tty( infd, program_mode ) ) return 1; - - if( output_filename.size() && !to_stdout && program_mode != m_test ) - delete_output_on_interrupt = true; - const struct stat * const in_statsp = input_filename.size() ? &in_stats : 0; - pp.set_name( input_filename ); - int tmp = 0; - if( program_mode == m_compress ) - tmp = compress( member_size, volume_size, encoder_options, infd, - pp, in_statsp ); - else - tmp = decompress( infd, pp, program_mode == m_test ); - if( tmp > retval ) retval = tmp; - if( tmp && program_mode != m_test ) cleanup_and_fail( retval ); - - if( delete_output_on_interrupt ) - close_and_set_permissions( in_statsp ); - if( input_filename.size() ) - { - close( infd ); infd = -1; - if( !keep_input_files && !to_stdout && program_mode != m_test ) - std::remove( input_filename.c_str() ); - } - } - if( outfd >= 0 && close( outfd ) != 0 ) - { - show_error( "Can't close stdout", errno ); - if( retval < 1 ) retval = 1; - } - return retval; - } diff --git a/tables.c b/tables.c new file mode 100644 index 0000000..c088706 --- /dev/null +++ b/tables.c @@ -0,0 +1,395 @@ +/* Lzlib - A compression library for lzip files + Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. + + This library is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this library. If not, see <http://www.gnu.org/licenses/>. + + As a special exception, you may use this file as part of a free + software library without restriction. Specifically, if other files + instantiate templates or use macros or inline functions from this + file, or you compile this file and link it with other files to + produce an executable, this file does not by itself cause the + resulting executable to be covered by the GNU General Public + License. This exception does not however invalidate any other + reasons why the executable file might be covered by the GNU General + Public License. +*/ + +/* Table of CRCs of all 8-bit messages. */ +static const uint32_t crc32[256] = + { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, + 0xE963A535, 0x9E6495A3, 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, + 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, 0x1DB71064, 0x6AB020F2, + 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, + 0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, + 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, 0x35B5A8FA, 0x42B2986C, + 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, + 0xCFBA9599, 0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, + 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, 0x76DC4190, 0x01DB7106, + 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, + 0x91646C97, 0xE6635C01, 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, + 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950, + 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, + 0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, + 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA, + 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, + 0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, + 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, 0xE3630B12, 0x94643B84, + 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, + 0x196C3671, 0x6E6B06E7, 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, + 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, 0xD6D6A3E8, 0xA1D1937E, + 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, + 0x316E8EEF, 0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, + 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE, 0xB2BD0B28, + 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, + 0x72076785, 0x05005713, 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, + 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242, + 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, + 0x616BFFD3, 0x166CCF45, 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, + 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC, + 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, + 0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, + 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D }; + + +static inline void CRC32_update_byte( uint32_t * crc, const uint8_t byte ) + { *crc = crc32[(*crc^byte)&0xFF] ^ ( *crc >> 8 ); } + +static inline void CRC32_update_buf( uint32_t * crc, const uint8_t * const buffer, + const int size ) + { + int i; + for( i = 0; i < size; ++i ) + *crc = crc32[(*crc^buffer[i])&0xFF] ^ ( *crc >> 8 ); + } + + +static const uint8_t dis_slots[1<<12] = + { + 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, + 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, + 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, + 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23 }; + + +static inline int get_slot( const uint32_t dis ) + { + if( dis < (1 << 12) ) return dis_slots[dis]; + if( dis < (1 << 23) ) return dis_slots[dis>>11] + 22; + return dis_slots[dis>>22] + 44; + } + + +static const int prob_prices[bit_model_total >> 2] = +{ +704, 576, 512, 480, 448, 432, 416, 400, 384, 376, 368, 360, 352, 344, 336, 328, +320, 316, 312, 308, 304, 300, 296, 292, 288, 284, 280, 276, 272, 268, 264, 260, +256, 254, 252, 250, 248, 246, 244, 242, 240, 238, 236, 234, 232, 230, 228, 226, +224, 222, 220, 218, 216, 214, 212, 210, 208, 206, 204, 202, 200, 198, 196, 194, +192, 191, 190, 189, 188, 187, 186, 185, 184, 183, 182, 181, 180, 179, 178, 177, +176, 175, 174, 173, 172, 171, 170, 169, 168, 167, 166, 165, 164, 163, 162, 161, +160, 159, 158, 157, 156, 155, 154, 153, 152, 151, 150, 149, 148, 147, 146, 145, +144, 143, 142, 141, 140, 139, 138, 137, 136, 135, 134, 133, 132, 131, 130, 129, +128, 127, 127, 126, 126, 125, 125, 124, 124, 123, 123, 122, 122, 121, 121, 120, +120, 119, 119, 118, 118, 117, 117, 116, 116, 115, 115, 114, 114, 113, 113, 112, +112, 111, 111, 110, 110, 109, 109, 108, 108, 107, 107, 106, 106, 105, 105, 104, +104, 103, 103, 102, 102, 101, 101, 100, 100, 99, 99, 98, 98, 97, 97, 96, + 96, 95, 95, 94, 94, 93, 93, 92, 92, 91, 91, 90, 90, 89, 89, 88, + 88, 87, 87, 86, 86, 85, 85, 84, 84, 83, 83, 82, 82, 81, 81, 80, + 80, 79, 79, 78, 78, 77, 77, 76, 76, 75, 75, 74, 74, 73, 73, 72, + 72, 71, 71, 70, 70, 69, 69, 68, 68, 67, 67, 66, 66, 65, 65, 64, + 64, 63, 63, 63, 63, 62, 62, 62, 62, 61, 61, 61, 61, 60, 60, 60, + 60, 59, 59, 59, 59, 58, 58, 58, 58, 57, 57, 57, 57, 56, 56, 56, + 56, 55, 55, 55, 55, 54, 54, 54, 54, 53, 53, 53, 53, 52, 52, 52, + 52, 51, 51, 51, 51, 50, 50, 50, 50, 49, 49, 49, 49, 48, 48, 48, + 48, 47, 47, 47, 47, 46, 46, 46, 46, 45, 45, 45, 45, 44, 44, 44, + 44, 43, 43, 43, 43, 42, 42, 42, 42, 41, 41, 41, 41, 40, 40, 40, + 40, 39, 39, 39, 39, 38, 38, 38, 38, 37, 37, 37, 37, 36, 36, 36, + 36, 35, 35, 35, 35, 34, 34, 34, 34, 33, 33, 33, 33, 32, 32, 32, + 32, 31, 31, 31, 31, 30, 30, 30, 30, 29, 29, 29, 29, 28, 28, 28, + 28, 27, 27, 27, 27, 26, 26, 26, 26, 25, 25, 25, 25, 24, 24, 24, + 24, 23, 23, 23, 23, 22, 22, 22, 22, 21, 21, 21, 21, 20, 20, 20, + 20, 19, 19, 19, 19, 18, 18, 18, 18, 17, 17, 17, 17, 16, 16, 16, + 16, 15, 15, 15, 15, 14, 14, 14, 14, 13, 13, 13, 13, 12, 12, 12, + 12, 11, 11, 11, 11, 10, 10, 10, 10, 9, 9, 9, 9, 8, 8, 8, + 8, 7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, + 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0 }; + + +static inline int get_price( const int probability ) + { + return prob_prices[probability >> 2]; + } diff --git a/testsuite/check.sh b/testsuite/check.sh index b74e550..371fcec 100755 --- a/testsuite/check.sh +++ b/testsuite/check.sh @@ -1,6 +1,6 @@ #! /bin/sh # check script for Lzlib - A compression library for lzip files -# Copyright (C) 2009, 2010, 2011 Antonio Diaz Diaz. +# Copyright (C) 2009, 2010, 2011, 2012 Antonio Diaz Diaz. # # This script is free software: you have unlimited permission # to copy, distribute and modify it. @@ -21,12 +21,13 @@ fi if [ -d tmp ] ; then rm -rf tmp ; fi mkdir tmp -printf "testing lzlib-%s..." "$2" cd "${objdir}"/tmp cat "${testdir}"/test.txt > in || framework_failure fail=0 +printf "testing lzlib-%s..." "$2" + "${LZIP}" -t "${testdir}"/test_v0.lz || fail=1 printf . "${LZIP}" -cd "${testdir}"/test_v0.lz > copy || fail=1 @@ -45,7 +46,7 @@ printf . cmp in copy || fail=1 printf . -"${LZIP}" -cf "${testdir}"/test_v1.lz > out 2>/dev/null +"${LZIP}" -cfq "${testdir}"/test_v1.lz > out if [ $? != 1 ] ; then fail=1 ; printf - ; else printf . ; fi "${LZIP}" -cF "${testdir}"/test_v1.lz > out || fail=1 "${LZIP}" -cd out | "${LZIP}" -d > copy || fail=1 @@ -83,7 +84,7 @@ for i in s4Ki 0 1 2 3 4 5 6 7 8s16 9s16 ; do printf . done -"${LZIP}" -$i < in > anyothername || fail=1 +"${LZIP}" < in > anyothername || fail=1 "${LZIP}" -d anyothername || fail=1 cmp in anyothername.out || fail=1 printf . |