diff options
| -rw-r--r-- | COPYING | 3 | ||||
| -rw-r--r-- | ChangeLog | 85 | ||||
| -rw-r--r-- | Makefile.in | 12 | ||||
| -rw-r--r-- | NEWS | 23 | ||||
| -rw-r--r-- | README | 43 | ||||
| -rw-r--r-- | arg_parser.cc | 15 | ||||
| -rw-r--r-- | arg_parser.h | 10 | ||||
| -rwxr-xr-x | configure | 4 | ||||
| -rw-r--r-- | decoder.cc | 51 | ||||
| -rw-r--r-- | decoder.h | 20 | ||||
| -rw-r--r-- | doc/lzip.1 | 30 | ||||
| -rw-r--r-- | doc/lzip.info | 758 | ||||
| -rw-r--r-- | doc/lzip.texi | 750 | ||||
| -rw-r--r-- | encoder.cc | 8 | ||||
| -rw-r--r-- | encoder_base.cc | 2 | ||||
| -rw-r--r-- | encoder_base.h | 6 | ||||
| -rw-r--r-- | list.cc | 13 | ||||
| -rw-r--r-- | lzip.h | 9 | ||||
| -rw-r--r-- | lzip_index.cc | 35 | ||||
| -rw-r--r-- | lzip_index.h | 11 | ||||
| -rw-r--r-- | main.cc | 139 | ||||
| -rwxr-xr-x | testsuite/check.sh | 163 | ||||
| -rw-r--r-- | testsuite/fox6.lz | bin | 480 -> 0 bytes | |||
| -rw-r--r-- | testsuite/fox6_mark.lz | bin | 480 -> 0 bytes | |||
| -rw-r--r-- | testsuite/fox_nz.lz | bin | 0 -> 80 bytes | |||
| -rw-r--r-- | testsuite/test.txt | 6 | ||||
| -rw-r--r-- | testsuite/test.txt.lz | bin | 7376 -> 7341 bytes | |||
| -rw-r--r-- | testsuite/test_em.txt.lz | bin | 14024 -> 0 bytes |
28 files changed, 1131 insertions, 1065 deletions
@@ -1,8 +1,7 @@ GNU GENERAL PUBLIC LICENSE Version 2, June 1991 - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Copyright (C) 1989, 1991 Free Software Foundation, Inc. <http://fsf.org/> Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. @@ -1,7 +1,16 @@ +2024-11-26 Antonio Diaz Diaz <antonio@gnu.org> + + * Version 1.25-rc1 released. + * Remove options '--empty-error' and '--marking-error'. + * decoder.cc (decode_member): Remove support for Sync Flush marker. + * lzip.texi: New chapter 'Syntax of command-line arguments'. + * check.sh: Use 'cp' instead of 'cat'. + * testsuite: Add fox_nz.lz. Remove fox6.lz,fox6_mark.lz,test_em.txt.lz. + 2024-03-01 Antonio Diaz Diaz <antonio@gnu.org> * Version 1.24.1 released. - * main.cc: Fix compilation failure on MinGW because of mkdir. + * main.cc: Fix compilation failure on MinGW caused by mkdir. 2024-01-26 Antonio Diaz Diaz <antonio@gnu.org> @@ -31,13 +40,13 @@ Make '-o' behave like '-c', but writing to file instead of stdout. Make '-c' and '-o' check whether the output is a terminal only once. Do not open output if input is a terminal. + Set a valid invocation_name even if argc == 0. * configure: Build, check, and install without 'make'. * Replace 'decompressed', 'compressed' with 'out', 'in' in output. * lzip_index.cc: Improve messages for corruption in last header. - * main.cc: Set a valid invocation_name even if argc == 0. * Document extraction from tar.lz in manual, '--help', and man page. * lzip.texi (Introduction): Mention plzip and tarlz as alternatives. - * lzip.texi: Several fixes and improvements. + Several fixes and improvements. * testsuite: Add 9 new test files. 2019-01-03 Antonio Diaz Diaz <antonio@gnu.org> @@ -47,9 +56,9 @@ * lzip.h (Lzip_trailer): New function 'verify_consistency'. * lzip_index.cc: Detect some kinds of corrupt trailers. * main.cc (main): Check return value of close( infd ). - * main.cc: Compile on DOS with DJGPP. - * Fix a GCC warning about catching std::bad_alloc by value. - * lzip.texi: Improve description of '-0..-9', '-m', and '-s'. + Compile on DOS with DJGPP. (Reported by Robert Riebisch). + Fix a GCC warning about catching std::bad_alloc by value. + * lzip.texi: Improve descriptions of '-0..-9', '-m', and '-s'. * configure: Accept appending to CXXFLAGS; 'CXXFLAGS+=OPTIONS'. * INSTALL: Document use of CXXFLAGS+='-D __USE_MINGW_ANSI_STDIO'. @@ -59,13 +68,13 @@ * New option '--loose-trailing'. * Improve corrupt header detection to HD=3. * main.cc: Show corrupt or truncated header in multimember file. - * main.cc (main): Option '-S, --volume-size' now keeps input files. + (main): Make option '-S, --volume-size' keep input files. + (main): Show final diagnostic when testing multiple files. + (set_c_outname): Do not add a second '.lz' to the arg of '-o'. * encoder_base.*: Adjust dictionary size for each member. * Replace 'bits/byte' with inverse compression ratio in output. * Show progress of decompression at verbosity level 2 (-vv). * Show progress of (de)compression only if stderr is a terminal. - * main.cc: Show final diagnostic when testing multiple files. - * main.cc: Do not add a second extension '.lz' to the arg of '-o'. * decoder.cc (verify_trailer): Show stored sizes also in hex. Show dictionary size at verbosity level 4 (-vvvv). * lzip.texi: New chapter 'Meaning of lzip's output'. @@ -78,7 +87,7 @@ * Compression time of option '-0' has been slightly reduced. * Decompression time has been reduced by 2%. * main.cc: Continue testing if any input file is a terminal. - * main.cc: Show trailing data in both hexadecimal and ASCII. + Show trailing data in both hexadecimal and ASCII. * encoder.cc (Matchfinder_base): Check the size passed to new. * lzip_index.cc: Improve detection of bad dict and trailing data. * lzip.h: Unify messages for bad magic, trailing data, etc. @@ -88,15 +97,14 @@ * Version 1.18 released. * New option '-a, --trailing-error'. * Decompression time has been reduced by 2%. - * decoder.cc (verify_trailer): Remove test of final code. * main.cc (main): Delete '--output' file if infd is a terminal. - * main.cc (main): Don't use stdin more than once. + (main): Don't use stdin more than once. * Remove decompression support for version 0 files. + * decoder.cc (verify_trailer): Remove test of final code. * lzip.texi: New chapter 'Trailing data'. * configure: Avoid warning on some shells when testing for g++. * Makefile.in: Detect the existence of install-info. - * check.sh: A POSIX shell is required to run the tests. - * check.sh: Don't check error messages. + * check.sh: Require a POSIX shell. Don't check error messages. 2015-07-12 Antonio Diaz Diaz <antonio@gnu.org> @@ -134,8 +142,8 @@ * Compression time has been reduced by 5%. * Decompression time has been reduced by 12%. * Makefile.in: New target 'install-bin'. - * main.cc: Use 'setmode' instead of '_setmode' on Windows and OS/2. * main.cc: Define 'strtoull' to 'std::strtoul' on Windows. + (main): Use 'setmode' instead of '_setmode' on Windows and OS/2. 2012-02-24 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -163,14 +171,12 @@ * Compression time of option '-0' has been reduced by 2%. * main.cc (decompress): Print only one status line for each multimember file when only one '-v' is specified. - * main.cc (decompress): Print up to 6 bytes of trailing data when - '-vvvv' is specified. - * main.cc (open_instream): Don't show the message + (decompress): Print 6 bytes of trailing data at verbosity level 4. + (open_instream): Don't show the message " and '--stdout' was not specified" for directories, etc. * lziprecover.cc: If '-v' is not specified show errors only. * unzcrash.cc: Use Arg_parser. - * unzcrash.cc: New options '-b, --bits', '-p, --position', and - '-s, --size'. + New options '-b, --bits', '-p, --position', and '-s, --size'. 2010-09-16 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -183,28 +189,26 @@ compress less but faster. (-1 now takes 43% less time for only 20% larger compressed size). Exit with status 1 if any output file exists and is skipped. + Fix warning about fchown's return value being ignored. + (main): Set stdin/stdout in binary mode on MSVC and OS2. * Compression ratio of option '-9' has been slightly increased. - * lziprecover.cc: New option '-m, --merge', which tries to produce a - correct file by merging the good parts of two or more damaged copies. - * lziprecover.cc: New option '-R, --repair' for repairing a - 1-byte error in single-member files. * decoder.cc (decode_member): Detect file errors earlier to improve efficiency of lziprecover's new repair capability. This change also prevents (harmless) access to uninitialized memory when decompressing a corrupt file. - * lziprecover.cc: New options '-f, --force' and '-o, --output'. - * lziprecover.cc: New option '-s, --split' to select the until now - only operation of splitting multimember files. - * lziprecover.cc: If no operation is specified, warn the user and do - nothing. - * main.cc: Fix warning about fchown's return value being ignored. - * decoder.cc: '-tvvvv' now also shows compression ratio. - * main.cc: Set stdin/stdout in binary mode on MSVC and OS2. + (verify_trailer): '-tvvvv' now also shows compression ratio. + * lziprecover.cc: New option '-m, --merge', which tries to produce a + correct file by merging the good parts of two or more damaged copies. + New option '-R, --repair' for repairing a 1-byte error in + single-member files. + New options '-f, --force' and '-o, --output'. + New option '-s, --split' to select the until now only operation of + splitting multimember files. + If no operation is specified, warn the user and do nothing. * lzip.texinfo: New examples. * testsuite: Rename 'test1' to 'test.txt'. New tests. * Matchfinder types HC4 (4 bytes hash-chain) and HT4 (4 bytes - hash-table) have been tested and found no better than the current - BT4. + hash-table) have been tested and found no better than current BT4. 2010-04-05 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -212,9 +216,9 @@ * decoder.h: Input_buffer integrated in Range_decoder. * main.cc: File specified with option '-o' is now created with mode 0666 if umask allows it, deleted if interrupted by user. - * main.cc: New constant 'o_binary'. - * main.cc: Dictionary size for options -2, -3, -4 and -8 has been - changed to improve linearity of compressed sizes. + New constant 'o_binary'. + (main): Change dictionary size for options -2, -3, -4 and -8 to + improve linearity of compressed sizes. * lzip.h: Fix warnings produced by over-optimization (-O3). * Makefile.in: Add quotes to directory names. @@ -241,7 +245,7 @@ * Version 1.6 released. * Decompression time has been reduced by 17%. - * Add decompression support for Sync Flush marker. + * decoder.cc (decode_member): Add support for Sync Flush marker. * Add support for the extension '.tbz' to lzdiff and lzgrep. * Add man pages for lzdiff, lzgrep and lziprecover. * encoder.cc (Matchfinder): Reduce memory use to 9x if input file is @@ -265,8 +269,7 @@ * Compression speed has been improved. * Implement bt4 type matchfinder. * lzip.texinfo: New chapter 'Algorithm'. - * Lzdiff and lzgrep now accept '-h' for '--help' and - '-V' for '--version'. + * lzdiff, lzgrep: Accept '-h' for '--help' and '-V' for '--version'. * Makefile.in: Man page is now installed by default. * check.sh: Check that files are opened in binary mode. @@ -330,7 +333,7 @@ * Version 0.3 released. * encoder.cc: Compression is now 15% faster, 1% worse. * main.cc (main): Make option '-t' override '-c'. - * main.cc (decompress): Show 'done' instead of 'ok' when not testing. + (decompress): Show 'done' instead of 'ok' when not testing. * encoder.h: Use trials[] to return the list of pairs. 2008-09-09 Antonio Diaz Diaz <ant_diaz@teleline.es> diff --git a/Makefile.in b/Makefile.in index 2b7f23c..ad0715a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -2,8 +2,8 @@ DISTNAME = $(pkgname)-$(pkgversion) INSTALL = install INSTALL_PROGRAM = $(INSTALL) -m 755 -INSTALL_DATA = $(INSTALL) -m 644 INSTALL_DIR = $(INSTALL) -d -m 755 +INSTALL_DATA = $(INSTALL) -m 644 SHELL = /bin/sh CAN_RUN_INSTALLINFO = $(SHELL) -c "install-info --version" > /dev/null 2>&1 @@ -28,6 +28,11 @@ main.o : main.cc %.o : %.cc $(CXX) $(CPPFLAGS) $(CXXFLAGS) -c -o $@ $< +# prevent 'make' from trying to remake source files +$(VPATH)/configure $(VPATH)/Makefile.in $(VPATH)/doc/$(pkgname).texi : ; +MAKEFLAGS += -r +.SUFFIXES : + $(objs) : Makefile arg_parser.o : arg_parser.h decoder.o : lzip.h decoder.h @@ -121,10 +126,7 @@ dist : doc $(DISTNAME)/testsuite/test.txt \ $(DISTNAME)/testsuite/fox.lz \ $(DISTNAME)/testsuite/fox_*.lz \ - $(DISTNAME)/testsuite/fox6.lz \ - $(DISTNAME)/testsuite/fox6_mark.lz \ - $(DISTNAME)/testsuite/test.txt.lz \ - $(DISTNAME)/testsuite/test_em.txt.lz + $(DISTNAME)/testsuite/test.txt.lz rm -f $(DISTNAME) lzip -v -9 $(DISTNAME).tar @@ -1,20 +1,11 @@ -Changes in version 1.24: +Changes in version 1.25: -The option '--empty-error', which forces exit status 2 if any empty member -is found, has been added. +lzip now exits with error status 2 if any empty member is found in a +multimember file. -The option '--marking-error', which forces exit status 2 if the first LZMA -byte is non-zero in any member, has been added. +lzip now exits with error status 2 if the first byte of the LZMA stream is +not 0. -File diagnostics have been reformatted as 'PROGRAM: FILE: MESSAGE'. +Options '--empty-error' and '--marking-error' have been removed. -Diagnostics caused by invalid arguments to command-line options now show the -argument and the name of the option. - -The option '-o, --output' now preserves dates, permissions, and ownership of -the file when (de)compressing exactly one file. - -The option '-o, --output' now creates missing intermediate directories when -writing to a file. - -The variable MAKEINFO has been added to configure and Makefile.in. +The chapter 'Syntax of command-line arguments' has been added to the manual. @@ -1,16 +1,18 @@ +See the file INSTALL for compilation and installation instructions. + Description Lzip is a lossless data compressor with a user interface similar to the one -of gzip or bzip2. Lzip uses a simplified form of the 'Lempel-Ziv-Markov -chain-Algorithm' (LZMA) stream format to maximize interoperability. The -maximum dictionary size is 512 MiB so that any lzip file can be decompressed -on 32-bit machines. Lzip provides accurate and robust 3-factor integrity -checking. Lzip can compress about as fast as gzip (lzip -0) or compress most -files more than bzip2 (lzip -9). Decompression speed is intermediate between -gzip and bzip2. Lzip is better than gzip and bzip2 from a data recovery -perspective. Lzip has been designed, written, and tested with great care to -replace gzip and bzip2 as the standard general-purpose compressed format for -Unix-like systems. +of gzip or bzip2. Lzip uses a simplified form of LZMA (Lempel-Ziv-Markov +chain-Algorithm) designed to achieve complete interoperability between +implementations. The maximum dictionary size is 512 MiB so that any lzip +file can be decompressed on 32-bit machines. Lzip provides accurate and +robust 3-factor integrity checking. 'lzip -0' compresses about as fast as +gzip, while 'lzip -9' compresses most files more than bzip2. Decompression +speed is intermediate between gzip and bzip2. Lzip provides better data +recovery capabilities than gzip and bzip2. Lzip has been designed, written, +and tested with great care to replace gzip and bzip2 as general-purpose +compressed format for Unix-like systems. For compressing/decompressing large files on multiprocessor machines plzip can be much faster than lzip at the cost of a slightly reduced compression @@ -23,11 +25,10 @@ alignment between tar members and lzip members. The lzip file format is designed for data sharing and long-term archiving, taking into account both data integrity and decoder availability: - * The lzip format provides very safe integrity checking and some data - recovery means. The program lziprecover can repair bit flip errors - (one of the most common forms of data corruption) in lzip files, and - provides data recovery capabilities, including error-checked merging - of damaged copies of a file. + * The program lziprecover can repair bit flip errors (one of the most + common forms of data corruption) in lzip files, and provides data + recovery capabilities, including error-checked merging of damaged + copies of a file. * The lzip format is as simple as possible (but not simpler). The lzip manual provides the source code of a simple decompressor along with a @@ -91,7 +92,6 @@ also supported. Lzip can produce multimember files, and lziprecover can safely recover the undamaged members in case of file damage. Lzip can also split the compressed output in volumes of a given size, even when reading from standard input. -This allows the direct creation of multivolume compressed tar archives. Lzip is able to compress and decompress streams of unlimited size by automatically creating multimember output. The members so created are large, @@ -101,10 +101,10 @@ In spite of its name (Lempel-Ziv-Markov chain-Algorithm), LZMA is not a concrete algorithm; it is more like "any algorithm using the LZMA coding scheme". For example, the option '-0' of lzip uses the scheme in almost the simplest way possible; issuing the longest match it can find, or a literal -byte if it can't find a match. Inversely, a much more elaborated way of -finding coding sequences of minimum size than the one currently used by lzip -could be developed, and the resulting sequence could also be coded using the -LZMA coding scheme. +byte if it can't find a match. Inversely, a more elaborate way of finding +coding sequences of minimum size than the one currently used by lzip could +be developed, and the resulting sequence could also be coded using the LZMA +coding scheme. Lzip currently implements two variants of the LZMA algorithm: fast (used by option '-0') and normal (used by all other compression levels). @@ -121,6 +121,9 @@ definition of Markov chains), G.N.N. Martin (for the definition of range encoding), Igor Pavlov (for putting all the above together in LZMA), and Julian Seward (for bzip2's CLI). +Lzip uses Arg_parser for command-line argument parsing: +http://www.nongnu.org/arg-parser/arg_parser.html + LANGUAGE NOTE: Uncompressed = not compressed = plain data; it may never have been compressed. Decompressed is used to refer to data which have undergone the process of decompression. diff --git a/arg_parser.cc b/arg_parser.cc index 0c04d8e..0c528b2 100644 --- a/arg_parser.cc +++ b/arg_parser.cc @@ -75,19 +75,19 @@ bool Arg_parser::parse_long_option( const char * const opt, const char * const a error_ += "' requires an argument"; return false; } - data.back().argument = &opt[len+3]; + data.back().argument = &opt[len+3]; // argument may be empty return true; } - if( options[index].has_arg == yes ) + if( options[index].has_arg == yes || options[index].has_arg == yme ) { - if( !arg || !arg[0] ) + if( !arg || ( options[index].has_arg == yes && !arg[0] ) ) { error_ = "option '--"; error_ += options[index].long_name; error_ += "' requires an argument"; return false; } - ++argind; data.back().argument = arg; + ++argind; data.back().argument = arg; // argument may be empty return true; } @@ -123,15 +123,16 @@ bool Arg_parser::parse_short_option( const char * const opt, const char * const { data.back().argument = &opt[cind]; ++argind; cind = 0; } - else if( options[index].has_arg == yes ) + else if( options[index].has_arg == yes || options[index].has_arg == yme ) { - if( !arg || !arg[0] ) + if( !arg || ( options[index].has_arg == yes && !arg[0] ) ) { error_ = "option requires an argument -- '"; error_ += c; error_ += '\''; return false; } - data.back().argument = arg; ++argind; cind = 0; + ++argind; cind = 0; + data.back().argument = arg; // argument may be empty } } return true; diff --git a/arg_parser.h b/arg_parser.h index 1eeec9a..ab77fc5 100644 --- a/arg_parser.h +++ b/arg_parser.h @@ -36,14 +36,18 @@ The argument '--' terminates all options; any following arguments are treated as non-option arguments, even if they begin with a hyphen. - The syntax for optional option arguments is '-<short_option><argument>' - (without whitespace), or '--<long_option>=<argument>'. + The syntax of options with an optional argument is + '-<short_option><argument>' (without whitespace), or + '--<long_option>=<argument>'. + + The syntax of options with an empty argument is '-<short_option> ""', + '--<long_option> ""', or '--<long_option>=""'. */ class Arg_parser { public: - enum Has_arg { no, yes, maybe }; + enum Has_arg { no, yes, maybe, yme }; // yme = yes but maybe empty struct Option { @@ -6,7 +6,7 @@ # to copy, distribute, and modify it. pkgname=lzip -pkgversion=1.24.1 +pkgversion=1.25-rc1 progname=lzip srctrigger=doc/${pkgname}.texi @@ -119,7 +119,7 @@ while [ $# != 0 ] ; do exit 1 ;; esac - # Check if the option took a separate argument + # Check whether the option took a separate argument if [ "${arg2}" = yes ] ; then if [ $# != 0 ] ; then args="${args} \"$1\"" ; shift else echo "configure: Missing argument to '${option}'" 1>&2 @@ -74,7 +74,7 @@ bool Range_decoder::read_block() { stream_pos = readblock( infd, buffer, buffer_size ); if( stream_pos != buffer_size && errno ) throw Error( "Read error" ); - at_stream_end = ( stream_pos < buffer_size ); + at_stream_end = stream_pos < buffer_size; partial_member_pos += pos; pos = 0; show_dprogress(); @@ -90,7 +90,7 @@ void LZ_decoder::flush_data() const int size = pos - stream_pos; crc32.update_buf( crc_, buffer + stream_pos, size ); if( outfd >= 0 && writeblock( outfd, buffer + stream_pos, size ) != size ) - throw Error( "Write error" ); + throw Error( write_error_msg ); if( pos >= dictionary_size ) { partial_data_pos += pos; pos = 0; pos_wrapped = true; } stream_pos = pos; @@ -98,8 +98,7 @@ void LZ_decoder::flush_data() } -int LZ_decoder::check_trailer( const Pretty_print & pp, - const bool ignore_empty ) const +bool LZ_decoder::check_trailer( const Pretty_print & pp ) const { Lzip_trailer trailer; int size = rdec.read_data( trailer.data, trailer.size ); @@ -144,8 +143,7 @@ int LZ_decoder::check_trailer( const Pretty_print & pp, std::fprintf( stderr, "Member size mismatch; stored %llu (0x%llX), computed %llu (0x%llX)\n", tm_size, tm_size, member_size, member_size ); } } - if( error ) return 3; - if( !ignore_empty && data_size == 0 ) return 5; + if( error ) return false; if( verbosity >= 2 ) { if( verbosity >= 4 ) show_header( dictionary_size ); @@ -160,15 +158,14 @@ int LZ_decoder::check_trailer( const Pretty_print & pp, if( verbosity >= 3 ) std::fprintf( stderr, "%9llu out, %8llu in. ", data_size, member_size ); } - return 0; + return true; } /* Return value: 0 = OK, 1 = decoder error, 2 = unexpected EOF, 3 = trailer error, 4 = unknown marker found, - 5 = empty member found, 6 = marked member found. */ -int LZ_decoder::decode_member( const Cl_options & cl_opts, - const Pretty_print & pp ) + 5 = nonzero first LZMA byte found. */ +int LZ_decoder::decode_member( const Pretty_print & pp ) { Bit_model bm_literal[1<<literal_context_bits][0x300]; Bit_model bm_match[State::states][pos_states]; @@ -188,7 +185,7 @@ int LZ_decoder::decode_member( const Cl_options & cl_opts, unsigned rep3 = 0; State state; - if( !rdec.load( cl_opts.ignore_marking ) ) return 6; + if( !rdec.load() ) return 5; while( !rdec.finished() ) { const int pos_state = data_position() & pos_state_mask; @@ -232,39 +229,33 @@ int LZ_decoder::decode_member( const Cl_options & cl_opts, } else // match { + rep3 = rep2; rep2 = rep1; rep1 = rep0; len = rdec.decode_len( match_len_model, pos_state ); - unsigned distance = rdec.decode_tree6( bm_dis_slot[get_len_state(len)] ); - if( distance >= start_dis_model ) + rep0 = rdec.decode_tree6( bm_dis_slot[get_len_state(len)] ); + if( rep0 >= start_dis_model ) { - const unsigned dis_slot = distance; + const unsigned dis_slot = rep0; const int direct_bits = ( dis_slot >> 1 ) - 1; - distance = ( 2 | ( dis_slot & 1 ) ) << direct_bits; + rep0 = ( 2 | ( dis_slot & 1 ) ) << direct_bits; if( dis_slot < end_dis_model ) - distance += rdec.decode_tree_reversed( - bm_dis + ( distance - dis_slot ), direct_bits ); + rep0 += rdec.decode_tree_reversed( bm_dis + ( rep0 - dis_slot ), + direct_bits ); else { - distance += - rdec.decode( direct_bits - dis_align_bits ) << dis_align_bits; - distance += rdec.decode_tree_reversed4( bm_align ); - if( distance == 0xFFFFFFFFU ) // marker found + rep0 += rdec.decode( direct_bits - dis_align_bits ) << dis_align_bits; + rep0 += rdec.decode_tree_reversed4( bm_align ); + if( rep0 == 0xFFFFFFFFU ) // marker found { rdec.normalize(); flush_data(); if( len == min_match_len ) // End Of Stream marker - return check_trailer( pp, cl_opts.ignore_empty ); - if( len == min_match_len + 1 ) // Sync Flush marker - { rdec.load(); continue; } - if( verbosity >= 0 ) - { - pp(); - std::fprintf( stderr, "Unsupported marker code '%d'\n", len ); - } + { if( check_trailer( pp ) ) return 0; else return 3; } + if( verbosity >= 0 ) { pp(); + std::fprintf( stderr, "Unsupported marker code '%d'\n", len ); } return 4; } } } - rep3 = rep2; rep2 = rep1; rep1 = rep0; rep0 = distance; state.set_match(); if( rep0 >= dictionary_size || ( rep0 >= pos && !pos_wrapped ) ) { flush_data(); return 1; } @@ -75,12 +75,12 @@ public: return sz; } - bool load( const bool ignore_marking = true ) + bool load() { code = 0; range = 0xFFFFFFFFU; - // check and discard first byte of the LZMA stream - if( get_byte() != 0 && !ignore_marking ) return false; + // check first byte of the LZMA stream + if( get_byte() != 0 ) return false; for( int i = 0; i < 4; ++i ) code = ( code << 8 ) | get_byte(); return true; } @@ -100,7 +100,7 @@ public: range >>= 1; // symbol <<= 1; // if( code >= range ) { code -= range; symbol |= 1; } - const bool bit = ( code >= range ); + const bool bit = code >= range; symbol <<= 1; symbol += bit; code -= range & ( 0U - bit ); } @@ -270,7 +270,7 @@ class LZ_decoder bool pos_wrapped; void flush_data(); - int check_trailer( const Pretty_print & pp, const bool ignore_empty ) const; + bool check_trailer( const Pretty_print & pp ) const; uint8_t peek_prev() const { return buffer[((pos > 0) ? pos : dictionary_size)-1]; } @@ -294,14 +294,14 @@ class LZ_decoder bool fast, fast2; if( lpos > distance ) { - fast = ( len < dictionary_size - lpos ); - fast2 = ( fast && len <= lpos - i ); + fast = len < dictionary_size - lpos; + fast2 = fast && len <= lpos - i; } else { i += dictionary_size; - fast = ( len < dictionary_size - i ); // (i == pos) may happen - fast2 = ( fast && len <= i - lpos ); + fast = len < dictionary_size - i; // (i == pos) may happen + fast2 = fast && len <= i - lpos; } if( fast ) // no wrap { @@ -342,5 +342,5 @@ public: unsigned crc() const { return crc_ ^ 0xFFFFFFFFU; } unsigned long long data_position() const { return partial_data_pos + pos; } - int decode_member( const Cl_options & cl_opts, const Pretty_print & pp ); + int decode_member( const Pretty_print & pp ); }; @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.49.2. -.TH LZIP "1" "March 2024" "lzip 1.24.1" "User Commands" +.TH LZIP "1" "November 2024" "lzip 1.25-rc1" "User Commands" .SH NAME lzip \- reduces the size of files .SH SYNOPSIS @@ -7,16 +7,16 @@ lzip \- reduces the size of files [\fI\,options\/\fR] [\fI\,files\/\fR] .SH DESCRIPTION Lzip is a lossless data compressor with a user interface similar to the one -of gzip or bzip2. Lzip uses a simplified form of the 'Lempel\-Ziv\-Markov -chain\-Algorithm' (LZMA) stream format to maximize interoperability. The -maximum dictionary size is 512 MiB so that any lzip file can be decompressed -on 32\-bit machines. Lzip provides accurate and robust 3\-factor integrity -checking. Lzip can compress about as fast as gzip (lzip \fB\-0\fR) or compress most -files more than bzip2 (lzip \fB\-9\fR). Decompression speed is intermediate between -gzip and bzip2. Lzip is better than gzip and bzip2 from a data recovery -perspective. Lzip has been designed, written, and tested with great care to -replace gzip and bzip2 as the standard general\-purpose compressed format for -Unix\-like systems. +of gzip or bzip2. Lzip uses a simplified form of LZMA (Lempel\-Ziv\-Markov +chain\-Algorithm) designed to achieve complete interoperability between +implementations. The maximum dictionary size is 512 MiB so that any lzip +file can be decompressed on 32\-bit machines. Lzip provides accurate and +robust 3\-factor integrity checking. 'lzip \fB\-0\fR' compresses about as fast as +gzip, while 'lzip \fB\-9\fR' compresses most files more than bzip2. Decompression +speed is intermediate between gzip and bzip2. Lzip provides better data +recovery capabilities than gzip and bzip2. Lzip has been designed, written, +and tested with great care to replace gzip and bzip2 as general\-purpose +compressed format for Unix\-like systems. .SH OPTIONS .TP \fB\-h\fR, \fB\-\-help\fR @@ -29,7 +29,7 @@ output version information and exit exit with error status if trailing data .TP \fB\-b\fR, \fB\-\-member\-size=\fR<bytes> -set member size limit in bytes +set member size limit of multimember files .TP \fB\-c\fR, \fB\-\-stdout\fR write to standard output, keep input files @@ -79,12 +79,6 @@ alias for \fB\-0\fR \fB\-\-best\fR alias for \fB\-9\fR .TP -\fB\-\-empty\-error\fR -exit with error status if empty member in file -.TP -\fB\-\-marking\-error\fR -exit with error status if 1st LZMA byte not 0 -.TP \fB\-\-loose\-trailing\fR allow trailing data seeming corrupt header .PP diff --git a/doc/lzip.info b/doc/lzip.info index 04f81c4..95587d8 100644 --- a/doc/lzip.info +++ b/doc/lzip.info @@ -11,17 +11,18 @@ File: lzip.info, Node: Top, Next: Introduction, Up: (dir) Lzip Manual *********** -This manual is for Lzip (version 1.24.1, 1 March 2024). +This manual is for Lzip (version 1.25-rc1, 26 November 2024). * Menu: * Introduction:: Purpose and features of lzip * Output:: Meaning of lzip's output * Invoking lzip:: Command-line interface -* Quality assurance:: Design, development, and testing of lzip -* Algorithm:: How lzip compresses the data +* Argument syntax:: By convention, options start with a hyphen * File format:: Detailed format of the compressed file * Stream format:: Format of the LZMA stream in lzip files +* Quality assurance:: Design, development, and testing of lzip +* Algorithm:: How lzip compresses the data * Trailing data:: Extra data appended to the file * Examples:: A small tutorial with examples * Problems:: Reporting bugs @@ -41,16 +42,16 @@ File: lzip.info, Node: Introduction, Next: Output, Prev: Top, Up: Top ************** Lzip is a lossless data compressor with a user interface similar to the one -of gzip or bzip2. Lzip uses a simplified form of the 'Lempel-Ziv-Markov -chain-Algorithm' (LZMA) stream format to maximize interoperability. The -maximum dictionary size is 512 MiB so that any lzip file can be decompressed -on 32-bit machines. Lzip provides accurate and robust 3-factor integrity -checking. Lzip can compress about as fast as gzip (lzip -0) or compress most -files more than bzip2 (lzip -9). Decompression speed is intermediate between -gzip and bzip2. Lzip is better than gzip and bzip2 from a data recovery -perspective. Lzip has been designed, written, and tested with great care to -replace gzip and bzip2 as the standard general-purpose compressed format for -Unix-like systems. +of gzip or bzip2. Lzip uses a simplified form of LZMA (Lempel-Ziv-Markov +chain-Algorithm) designed to achieve complete interoperability between +implementations. The maximum dictionary size is 512 MiB so that any lzip +file can be decompressed on 32-bit machines. Lzip provides accurate and +robust 3-factor integrity checking. 'lzip -0' compresses about as fast as +gzip, while 'lzip -9' compresses most files more than bzip2. Decompression +speed is intermediate between gzip and bzip2. Lzip provides better data +recovery capabilities than gzip and bzip2. Lzip has been designed, written, +and tested with great care to replace gzip and bzip2 as general-purpose +compressed format for Unix-like systems. For compressing/decompressing large files on multiprocessor machines plzip can be much faster than lzip at the cost of a slightly reduced @@ -64,12 +65,10 @@ alignment between tar members and lzip members. *Note tarlz manual: The lzip file format is designed for data sharing and long-term archiving, taking into account both data integrity and decoder availability: - * The lzip format provides very safe integrity checking and some data - recovery means. The program lziprecover can repair bit flip errors - (one of the most common forms of data corruption) in lzip files, and - provides data recovery capabilities, including error-checked merging - of damaged copies of a file. *Note Data safety: (lziprecover)Data - safety. + * The program lziprecover can repair bit flip errors (one of the most + common forms of data corruption) in lzip files, and provides data + recovery capabilities, including error-checked merging of damaged + copies of a file. *Note Data safety: (lziprecover)Data safety. * The lzip format is as simple as possible (but not simpler). The lzip manual provides the source code of a simple decompressor along with a @@ -88,7 +87,7 @@ byte near the beginning is a thing of the past. The member trailer stores the 32-bit CRC of the original data, the size of the original data, and the size of the member. These values, together -with the "End Of Stream" marker, provide a 3-factor integrity checking which +with the 'End Of Stream' marker, provide a 3-factor integrity checking which guarantees that the decompressed version of the data is identical to the original. This guards against corruption of the compressed data, and against undetected bugs in lzip (hopefully very unlikely). The chances of data @@ -101,9 +100,9 @@ makes it safer than compressors returning ambiguous warning values (like gzip) when it is used as a back end for other programs like tar or zutils. Lzip automatically uses for each file the largest dictionary size that -does not exceed neither the file size nor the limit given. Keep in mind -that the decompression memory requirement is affected at compression time -by the choice of dictionary size limit. +does not exceed neither the file size nor the limit given. The dictionary +size used for decompression is the same dictionary size used for +compression. The amount of memory required for compression is about 1 or 2 times the dictionary size limit (1 if input file size is less than dictionary size @@ -142,8 +141,7 @@ also supported. Lzip can produce multimember files, and lziprecover can safely recover the undamaged members in case of file damage. Lzip can also split the compressed output in volumes of a given size, even when reading from -standard input. This allows the direct creation of multivolume compressed -tar archives. +standard input. Lzip is able to compress and decompress streams of unlimited size by automatically creating multimember output. The members so created are large, @@ -197,7 +195,7 @@ have been compressed. Decompressed is used to refer to data which have undergone the process of decompression. -File: lzip.info, Node: Invoking lzip, Next: Quality assurance, Prev: Output, Up: Top +File: lzip.info, Node: Invoking lzip, Next: Argument syntax, Prev: Output, Up: Top 3 Invoking lzip *************** @@ -212,8 +210,7 @@ means standard input. It can be mixed with other FILES and is read just once, the first time it appears in the command line. Remember to prepend './' to any file name beginning with a hyphen, or use '--'. - lzip supports the following options: *Note Argument syntax: -(arg_parser)Argument syntax. +lzip supports the following options: *Note Argument syntax::. '-h' '--help' @@ -232,11 +229,12 @@ once, the first time it appears in the command line. Remember to prepend '-b BYTES' '--member-size=BYTES' - When compressing, set the member size limit to BYTES. It is advisable - to keep members smaller than RAM size so that they can be repaired with - lziprecover in case of corruption. A small member size may degrade - compression ratio, so use it only when needed. Valid values range from - 100 kB to 2 PiB. Defaults to 2 PiB. + When compressing, set the member size limit to BYTES. If BYTES is + smaller than the compressed size, a multimember file is produced. It is + advisable to keep members smaller than RAM size so that they can be + repaired with lziprecover in case of corruption. A small member size + may degrade compression ratio, so use it only when needed. Valid + values range from 100 kB to 2 PiB. Defaults to 2 PiB. '-c' '--stdout' @@ -257,7 +255,8 @@ once, the first time it appears in the command line. Remember to prepend status 1. If a file fails to decompress, or is a terminal, lzip exits immediately with error status 2 without decompressing the rest of the files. A terminal is considered an uncompressed file, and therefore - invalid. + invalid. A multimember file with one or more empty members is accepted + if redirected to standard input. '-f' '--force' @@ -281,7 +280,8 @@ once, the first time it appears in the command line. Remember to prepend '-v', the dictionary size, the number of members in the file, and the amount of trailing data (if any) are also printed. With '-vv', the positions and sizes of each member in multimember files are also - printed. + printed. A multimember file with one or more empty members is accepted + if redirected to standard input. If any file is damaged, does not exist, can't be opened, or is not regular, the final exit status is > 0. '-lq' can be used to check @@ -356,7 +356,8 @@ once, the first time it appears in the command line. Remember to prepend fails the test, does not exist, can't be opened, or is a terminal, lzip continues testing the rest of the files. A final diagnostic is shown at verbosity level 1 or higher if any file fails the test when testing - multiple files. + multiple files. A multimember file with one or more empty members is + accepted if redirected to standard input. '-v' '--verbose' @@ -366,8 +367,8 @@ once, the first time it appears in the command line. Remember to prepend When decompressing or testing, further -v's (up to 4) increase the verbosity level, showing status, compression ratio, dictionary size, trailer contents (CRC, data size, member size), and up to 6 bytes of - trailing data (if any) both in hexadecimal and as a string of printable - ASCII characters. + trailing data (if any) both in hexadecimal and as a string of + printable ASCII characters. Two or more '-v' options show the progress of (de)compression. '-0 .. -9' @@ -387,6 +388,7 @@ once, the first time it appears in the command line. Remember to prepend '-s64MiB -m273' Level Dictionary size (-s) Match length limit (-m) + ------------------------------------------------------ -0 64 KiB 16 bytes -1 1 MiB 5 bytes -2 1.5 MiB 6 bytes @@ -402,21 +404,11 @@ once, the first time it appears in the command line. Remember to prepend '--best' Aliases for GNU gzip compatibility. -'--empty-error' - Exit with error status 2 if any empty member is found in the input - files. - -'--marking-error' - Exit with error status 2 if the first LZMA byte is non-zero in any - member of the input files. This may be caused by data corruption or by - deliberate insertion of tracking information in the file. Use - 'lziprecover --clear-marking' to clear any such non-zero bytes. - '--loose-trailing' When decompressing, testing, or listing, allow trailing data whose first bytes are so similar to the magic bytes of a lzip header that they can be confused with a corrupt header. Use this option if a file - triggers a "corrupt header" error and the cause is not indeed a + triggers a 'corrupt header' error and the cause is not indeed a corrupt header. @@ -427,6 +419,7 @@ and may be followed by a multiplier and an optional 'B' for "byte". Table of SI and binary prefixes (unit multipliers): Prefix Value | Prefix Value +---------------------------------------------------------------------- k kilobyte (10^3 = 1000) | Ki kibibyte (2^10 = 1024) M megabyte (10^6) | Mi mebibyte (2^20) G gigabyte (10^9) | Gi gibibyte (2^30) @@ -445,278 +438,58 @@ corrupt or invalid input file, 3 for an internal consistency error (e.g., bug) which caused lzip to panic. -File: lzip.info, Node: Quality assurance, Next: Algorithm, Prev: Invoking lzip, Up: Top - -4 Design, development, and testing of lzip -****************************************** - -There are two ways of constructing a software design: One way is to make it -so simple that there are obviously no deficiencies and the other way is to -make it so complicated that there are no obvious deficiencies. The first -method is far more difficult. --- C.A.R. Hoare - - Lzip has been designed, written, and tested with great care to replace -gzip and bzip2 as the standard general-purpose compressed format for -Unix-like systems. This chapter describes the lessons learned from these -previous formats, and their application to the design of lzip. The lzip -format specification has been reviewed carefully and is believed to be free -from design errors. - - -4.1 Format design -================= - -When gzip was designed in 1992, computers and operating systems were much -less capable than they are today. The designers of gzip tried to work around -some of those limitations, like 8.3 file names, with additional fields in -the file format. - - Today those limitations have mostly disappeared, and the format of gzip -has proved to be unnecessarily complicated. It includes fields that were -never used, others that have lost their usefulness, and finally others that -have become too limited. - - Bzip2 was designed 5 years later, and its format is simpler than the one -of gzip. - - Probably the worst defect of the gzip format from the point of view of -data safety is the variable size of its header. If the byte at offset 3 -(flags) of a gzip member gets corrupted, it may become difficult to recover -the data, even if the compressed blocks are intact, because it can't be -known with certainty where the compressed blocks begin. - - By contrast, the header of a lzip member has a fixed length of 6. The -LZMA stream in a lzip member always starts at offset 6, making it trivial to -recover the data even if the whole header becomes corrupt. - - Bzip2 also provides a header of fixed length and marks the begin and end -of each compressed block with six magic bytes, making it possible to find -the compressed blocks even in case of file damage. But bzip2 does not store -the size of each compressed block, as lzip does. - - Lziprecover is able to provide unique data recovery capabilities because -the lzip format is extraordinarily safe. The simple and safe design of the -file format complements the embedded error detection provided by the LZMA -data stream. Any distance larger than the dictionary size acts as a -forbidden symbol, allowing the decompressor to detect the approximate -position of errors, and leaving very little work for the check sequence -(CRC and data sizes) in the detection of errors. Lzip is usually able to -detect all possible bit flips in the compressed data without resorting to -the check sequence. It would be difficult to write an automatic recovery -tool like lziprecover for the gzip format. And, as far as I know, it has -never been written. - - Lzip, like gzip and bzip2, uses a CRC32 to check the integrity of the -decompressed data because it provides optimal accuracy in the detection of -errors up to a compressed size of about 16 GiB, a size larger than that of -most files. In the case of lzip, the additional detection capability of the -decompressor reduces the probability of undetected errors several million -times more, resulting in a combined integrity checking optimally accurate -for any member size produced by lzip. Preliminary results suggest that the -lzip format is safe enough to be used in critical safety avionics systems. - - The lzip format is designed for long-term archiving. Therefore it -excludes any unneeded features that may interfere with the future -extraction of the decompressed data. - - -4.1.1 Gzip format (mis)features not present in lzip ---------------------------------------------------- - -'Multiple algorithms' - Gzip provides a CM (Compression Method) field that has never been used - because it is a bad idea to begin with. New compression methods may - require additional fields, making it impossible to implement new - methods and, at the same time, keep the same format. This field does - not solve the problem of format proliferation; it just makes the - problem less obvious. - -'Optional fields in header' - Unless special precautions are taken, optional fields are generally a - bad idea because they produce a header of variable size. The gzip - header has 2 fields that, in addition to being optional, are - zero-terminated. This means that if any byte inside the field gets - zeroed, or if the terminating zero gets altered, gzip won't be able to - find neither the header CRC nor the compressed blocks. - -'Optional CRC for the header' - Using an optional CRC for the header is not only a bad idea, it is an - error; it circumvents the Hamming distance (HD) of the CRC and may - prevent the extraction of perfectly good data. For example, if the CRC - is used and the bit enabling it is reset by a bit flip, then the - header seems to be intact (in spite of being corrupt) while the - compressed blocks seem to be totally unrecoverable (in spite of being - intact). Very misleading indeed. - -'Metadata' - The gzip format stores some metadata, like the modification time of the - original file or the operating system on which compression took place. - This complicates reproducible compression (obtaining identical - compressed output from identical input). - - -4.1.2 Lzip format improvements over gzip and bzip2 --------------------------------------------------- - -'64-bit size field' - Probably the most frequently reported shortcoming of the gzip format - is that it only stores the least significant 32 bits of the - uncompressed size. The size of any file larger or equal than 4 GiB - gets truncated. - - Bzip2 does not store the uncompressed size of the file. - - The lzip format provides a 64-bit field for the uncompressed size. - Additionally, lzip produces multimember output automatically when the - size is too large for a single member, allowing for an unlimited - uncompressed size. - -'Distributed index' - The lzip format provides a distributed index that, among other things, - helps plzip to decompress several times faster than pigz and helps - lziprecover do its job. Neither the gzip format nor the bzip2 format - do provide an index. - - A distributed index is safer and more scalable than a monolithic - index. The monolithic index introduces a single point of failure in - the compressed file and may limit the number of members or the total - uncompressed size. - - -4.2 Quality of implementation -============================= - -Our civilization depends critically on software; it had better be quality -software. --- Bjarne Stroustrup - -'Accurate and robust error detection' - The lzip format provides 3-factor integrity checking, and the - decompressors report mismatches in each factor separately. This method - detects most false positives for corruption. If just one byte in one - factor fails but the other two factors match the data, it probably - means that the data are intact and the corruption just affects the - mismatching factor (CRC, data size, or member size) in the member - trailer. - -'Multiple implementations' - Just like the lzip format provides 3-factor protection against - undetected data corruption, the development methodology of the lzip - family of compressors provides 3-factor protection against undetected - programming errors. - - Three related but independent compressor implementations, lzip, clzip, - and minilzip/lzlib, are developed concurrently. Every stable release - of any of them is tested to check that it produces identical output to - the other two. This guarantees that all three implement the same - algorithm, and makes it unlikely that any of them may contain serious - undiscovered errors. In fact, no errors have been discovered in lzip - since 2009. - - Additionally, the three implementations have been extensively tested - with unzcrash, valgrind, and 'american fuzzy lop' without finding a - single vulnerability or false negative. *Note Unzcrash: - (lziprecover)Unzcrash. - -'Dictionary size' - Lzip automatically adapts the dictionary size to the size of each file. - In addition to reducing the amount of memory required for - decompression, this feature also minimizes the probability of being - affected by RAM errors during compression. - -'Exit status' - Returning a warning status of 2 is a design flaw of compress that - leaked into the design of gzip. Both bzip2 and lzip are free from this - flaw. - - - -File: lzip.info, Node: Algorithm, Next: File format, Prev: Quality assurance, Up: Top +File: lzip.info, Node: Argument syntax, Next: File format, Prev: Invoking lzip, Up: Top -5 Algorithm -*********** +4 Syntax of command-line arguments +********************************** -In spite of its name (Lempel-Ziv-Markov chain-Algorithm), LZMA is not a -concrete algorithm; it is more like "any algorithm using the LZMA coding -scheme". LZMA compression consists in describing the uncompressed data as a -succession of coding sequences from the set shown in Section 'What is -coded' (*note what-is-coded::), and then encoding them using a range -encoder. For example, the option '-0' of lzip uses the scheme in almost the -simplest way possible; issuing the longest match it can find, or a literal -byte if it can't find a match. Inversely, a much more elaborated way of -finding coding sequences of minimum size than the one currently used by -lzip could be developed, and the resulting sequence could also be coded -using the LZMA coding scheme. +POSIX recommends these conventions for command-line arguments. - Lzip currently implements two variants of the LZMA algorithm: fast (used -by option '-0') and normal (used by all other compression levels). - - The high compression of LZMA comes from combining two basic, well-proven -compression ideas: sliding dictionaries (LZ77) and Markov models (the thing -used by every compression algorithm that uses a range encoder or similar -order-0 entropy coder as its last stage) with segregation of contexts -according to what the bits are used for. - - Lzip is a two stage compressor. The first stage is a Lempel-Ziv coder, -which reduces redundancy by translating chunks of data to their -corresponding distance-length pairs. The second stage is a range encoder -that uses a different probability model for each type of data: distances, -lengths, literal bytes, etc. - - Here is how it works, step by step: + * A command-line argument is an option if it begins with a hyphen ('-'). - 1) The member header is written to the output stream. - - 2) The first byte is coded literally, because there are no previous -bytes to which the match finder can refer to. + * Option names are single alphanumeric characters. - 3) The main encoder advances to the next byte in the input data and -calls the match finder. + * Certain options require an argument. - 4) The match finder fills an array with the minimum distances before the -current byte where a match of a given length can be found. - - 5) Go back to step 3 until a sequence (formed of pairs, repeated -distances, and literal bytes) of minimum price has been formed. Where the -price represents the number of output bits produced. - - 6) The range encoder encodes the sequence produced by the main encoder -and sends the bytes produced to the output stream. + * An option and its argument may or may not appear as separate tokens. + (In other words, the whitespace separating them is optional). Thus, + '-o foo' and '-ofoo' are equivalent. - 7) Go back to step 3 until the input data are finished or until the -member or volume size limits are reached. + * One or more options without arguments, followed by at most one option + that takes an argument, may follow a hyphen in a single token. Thus, + '-abc' is equivalent to '-a -b -c'. - 8) The range encoder is flushed. + * Options typically precede other non-option arguments. - 9) The member trailer is written to the output stream. + * The argument '--' terminates all options; any following arguments are + treated as non-option arguments, even if they begin with a hyphen. - 10) If there are more data to compress, go back to step 1. + * A token consisting of a single hyphen character is interpreted as an + ordinary non-option argument. By convention, it is used to specify + standard input, standard output, or a file named '-'. +GNU adds "long options" to these conventions: - During compression, lzip reads data in large blocks (one dictionary size -at a time). Therefore it may block for up to tens of seconds any process -feeding data to it through a pipe. This is normal. The blocking intervals -get longer with higher compression levels because dictionary size increases -(and compression speed decreases) with compression level. + * A long option consists of two hyphens ('--') followed by a name made + of alphanumeric characters and hyphens. Option names are typically one + to three words long, with hyphens to separate words. Abbreviations can + be used for the long option names as long as the abbreviations are + unique. -The ideas embodied in lzip are due to (at least) the following people: -Abraham Lempel and Jacob Ziv (for the LZ algorithm), Andrei Markov (for the -definition of Markov chains), G.N.N. Martin (for the definition of range -encoding), Igor Pavlov (for putting all the above together in LZMA), and -Julian Seward (for bzip2's CLI). + * A long option and its argument may or may not appear as separate + tokens. In the latter case they must be separated by an equal sign '='. + Thus, '--foo bar' and '--foo=bar' are equivalent. -File: lzip.info, Node: File format, Next: Stream format, Prev: Algorithm, Up: Top +File: lzip.info, Node: File format, Next: Stream format, Prev: Argument syntax, Up: Top -6 File format +5 File format ************* Perfection is reached, not when there is no longer anything to add, but when there is no longer anything to take away. -- Antoine de Saint-Exupery - In the diagram below, a box like this: +---+ @@ -731,12 +504,12 @@ when there is no longer anything to take away. represents a variable number of bytes. - - A lzip file consists of one or more independent "members" (compressed -data sets). The members simply appear one after another in the file, with no +A lzip file consists of one or more independent "members" (compressed data +sets). The members simply appear one after another in the file, with no additional information before, between, or after them. Each member can encode in compressed form up to 16 EiB - 1 byte of uncompressed data. The -size of a multimember file is unlimited. +size of a multimember file is unlimited. Empty members (data size = 0) are +not allowed in multimember files. Each member has the following structure: @@ -764,7 +537,7 @@ size of a multimember file is unlimited. Valid values for dictionary size range from 4 KiB to 512 MiB. 'LZMA stream' - The LZMA stream, finished by an "End Of Stream" marker. Uses default + The LZMA stream, terminated by an 'End Of Stream' marker. Uses default values for encoder properties. *Note Stream format::, for a complete description. @@ -781,14 +554,13 @@ size of a multimember file is unlimited. files. Lzip limits the member size to 2 PiB to prevent the data size field from overflowing. - -File: lzip.info, Node: Stream format, Next: Trailing data, Prev: File format, Up: Top +File: lzip.info, Node: Stream format, Next: Quality assurance, Prev: File format, Up: Top -7 Format of the LZMA stream in lzip files +6 Format of the LZMA stream in lzip files ***************************************** -The LZMA algorithm has three parameters, called "special LZMA properties", +The LZMA algorithm has three parameters, called 'special LZMA properties', to adjust it for some kinds of binary data. These parameters are: 'literal_context_bits' (with a default value of 3), 'literal_pos_state_bits' (with a default value of 0), and 'pos_state_bits' @@ -797,14 +569,17 @@ uses the default values for these parameters. In particular 'literal_pos_state_bits' has been optimized away and does not even appear in the code. - Lzip finishes the LZMA stream with an "End Of Stream" (EOS) marker (the + The first byte of the LZMA stream is set to zero to help tools like grep +recognize lzip files as binary files. + + The LZMA stream is terminated by an 'End Of Stream' (EOS) marker (the distance-length pair 0xFFFFFFFFU, 2), which in conjunction with the 'member size' field in the member trailer allows the checking of stream integrity. The EOS marker is the only LZMA marker allowed in lzip files. The LZMA stream in lzip files always has these two features (default properties and EOS marker) and is referred to in this document as LZMA-302eos. This simplified and marker-terminated form of the LZMA stream format has been -chosen to maximize interoperability and safety. +chosen to achieve complete interoperability and robust safety. The second stage of LZMA is a range encoder that uses a different probability model for each type of symbol: distances, lengths, literal @@ -821,13 +596,12 @@ the range decoder that need to be described accurately, the source code of a real decompressor seems the only appropriate reference to use. What follows is a description of the decoding algorithm for LZMA-302eos -streams using as reference the source code of "lzd", an educational +streams using as reference the source code of lzd, an educational decompressor for lzip files, included in appendix A. *Note Reference source code::. Lzd is written in C++11 and can be downloaded from the lzip download directory. - -7.1 What is coded +6.1 What is coded ================= The LZMA stream includes literals, matches, and repeated matches (matches @@ -901,7 +675,7 @@ slot + direct_bits distances from 4 to 127 slot + (direct_bits - 4) + 4 bits distances from 128 to 2^32 - 1 -7.2 The coding contexts +6.2 The coding contexts ======================= These contexts ('Bit_model' in the source), are integers or arrays of @@ -991,8 +765,7 @@ corresponding bit in 'match_byte'. After the first difference is found, the rest of the byte is decoded using the normal bit tree context. (See 'decode_matched' in the source). - -7.3 The range decoder +6.3 The range decoder ===================== The LZMA stream is consumed one byte at a time by the range decoder. (See @@ -1005,29 +778,284 @@ of decoded bits, depending on how well these bits agree with their context. decoded) and 'code' (representing the current point within 'range'). 'range' is initialized to 2^32 - 1, and 'code' is initialized to 0. - The range encoder produces a first 0 byte that must be ignored by the -range decoder. (See the 'Range_decoder' constructor in the source). - - -7.4 Decoding and checking the LZMA stream +6.4 Decoding and checking the LZMA stream ========================================= After decoding the member header and obtaining the dictionary size, the range decoder is initialized and then the LZMA decoder enters a loop (see 'decode_member' in the source) where it invokes the range decoder with the appropriate contexts to decode the different coding sequences (matches, -repeated matches, and literal bytes), until the "End Of Stream" marker is +repeated matches, and literal bytes), until the 'End Of Stream' marker is decoded. - Once the "End Of Stream" marker has been decoded, the decompressor reads + Once the 'End Of Stream' marker has been decoded, the decompressor reads and decodes the member trailer, and checks that the three integrity factors stored there (CRC, data size, and member size) match those computed from the data. -File: lzip.info, Node: Trailing data, Next: Examples, Prev: Stream format, Up: Top +File: lzip.info, Node: Quality assurance, Next: Algorithm, Prev: Stream format, Up: Top -8 Extra data appended to the file +7 Design, development, and testing of lzip +****************************************** + +There are two ways of constructing a software design: One way is to make it +so simple that there are obviously no deficiencies and the other way is to +make it so complicated that there are no obvious deficiencies. The first +method is far more difficult. +-- C.A.R. Hoare + + Lzip has been designed, written, and tested with great care to replace +gzip and bzip2 as general-purpose compressed format for Unix-like systems. +This chapter describes the lessons learned from these previous formats, and +their application to the design of lzip. The lzip format specification has +been reviewed carefully and is believed to be free from design errors. + +7.1 Format design +================= + +When gzip was designed in 1992, computers and operating systems were less +capable than they are today. The designers of gzip tried to work around some +of those limitations, like 8.3 file names, with additional fields in the +file format. + + Today those limitations have mostly disappeared, and the format of gzip +has proved to be unnecessarily complicated. It includes fields that were +never used, others that have lost their usefulness, and finally others that +have become too limited. + + Bzip2 was designed 5 years later, and its format is simpler than the one +of gzip. + + Probably the worst defect of the gzip format from the point of view of +data safety is the variable size of its header. If the byte at offset 3 +(flags) of a gzip member gets corrupted, it may become difficult to recover +the data, even if the compressed blocks are intact, because it can't be +known with certainty where the compressed blocks begin. + + By contrast, the header of a lzip member has a fixed length of 6. The +LZMA stream in a lzip member always starts at offset 6, making it trivial to +recover the data even if the whole header becomes corrupt. + + Bzip2 also provides a header of fixed length and marks the begin and end +of each compressed block with six magic bytes, making it possible to find +the compressed blocks even in case of file damage. But bzip2 does not store +the size of each compressed block, as lzip does. + + Lziprecover is able to provide unique data recovery capabilities because +the lzip format is extraordinarily safe. The simple and safe design of the +file format complements the embedded error detection provided by the LZMA +data stream. Any distance larger than the dictionary size acts as a +forbidden symbol, allowing the decompressor to detect the approximate +position of errors, and leaving very little work for the check sequence +(CRC and data sizes) in the detection of errors. Lzip is usually able to +detect all possible bit flips in the compressed data without resorting to +the check sequence. It would be difficult to write an automatic recovery +tool like lziprecover for the gzip format. And, as far as I know, it has +never been written. + + Lzip, like gzip and bzip2, uses a CRC32 to check the integrity of the +decompressed data because it provides optimal accuracy in the detection of +errors up to a compressed size of about 16 GiB, a size larger than that of +most files. In the case of lzip, the additional detection capability of the +decompressor reduces the probability of undetected errors several million +times more, resulting in a combined integrity checking optimally accurate +for any member size produced by lzip. Preliminary results suggest that the +lzip format is safe enough to be used in critical safety avionics systems. + + The lzip format is designed for long-term archiving. Therefore it +excludes any unneeded features that may interfere with the future +extraction of the decompressed data. + +7.1.1 Gzip format (mis)features not present in lzip +--------------------------------------------------- + +'Multiple algorithms' + Gzip provides a CM (Compression Method) field that has never been used + because it is a bad idea to begin with. New compression methods may + require additional fields, making it impossible to implement new + methods and, at the same time, keep the same format. This field does + not solve the problem of format proliferation; it just makes the + problem less obvious. + +'Optional fields in header' + Unless special precautions are taken, optional fields are generally a + bad idea because they produce a header of variable size. The gzip + header has 2 fields that, in addition to being optional, are + zero-terminated. This means that if any byte inside the field gets + zeroed, or if the terminating zero gets altered, gzip won't be able to + find neither the header CRC nor the compressed blocks. + +'Optional CRC for the header' + Using an optional CRC for the header is not only a bad idea, it is an + error; it circumvents the Hamming distance (HD) of the CRC and may + prevent the extraction of perfectly good data. For example, if the CRC + is used and the bit enabling it is reset by a bit flip, then the + header seems to be intact (in spite of being corrupt) while the + compressed blocks seem to be unrecoverable (in spite of being intact). + Very misleading indeed. + +'Metadata' + The gzip format stores some metadata, like the modification time of the + original file or the operating system on which compression took place. + This complicates reproducible compression (obtaining identical + compressed output from identical input). + + +7.1.2 Lzip format improvements over gzip and bzip2 +-------------------------------------------------- + +'64-bit size field' + Probably the most frequently reported shortcoming of the gzip format + is that it only stores the least significant 32 bits of the + uncompressed size. The size of any file larger or equal than 4 GiB + gets truncated. + + Bzip2 does not store the uncompressed size of the file. + + The lzip format provides a 64-bit field for the uncompressed size. + Additionally, lzip produces multimember output automatically when the + size is too large for a single member, allowing for an unlimited + uncompressed size. + +'Distributed index' + The lzip format provides a distributed index that, among other things, + helps plzip to decompress faster than pigz and helps lziprecover do + its job. Neither the gzip format nor the bzip2 format do provide an + index. + + A distributed index is safer and more scalable than a monolithic + index. The monolithic index introduces a single point of failure in + the compressed file and may limit the number of members or the total + uncompressed size. + + +7.2 Quality of implementation +============================= + +Our civilization depends critically on software; it had better be quality +software. +-- Bjarne Stroustrup + +'Accurate and robust error detection' + The lzip format provides 3-factor integrity checking, and the + decompressors report mismatches in each factor separately. This method + detects most false positives for corruption. If just one byte in one + factor fails but the other two factors match the data, it probably + means that the data are intact and the corruption just affects the + mismatching factor (CRC, data size, or member size) in the member + trailer. + +'Multiple implementations' + Just like the lzip format provides 3-factor protection against + undetected data corruption, the development methodology of the lzip + family of compressors provides 3-factor protection against undetected + programming errors. + + Three related but independent compressor implementations, lzip, clzip, + and minilzip/lzlib, are developed concurrently. Every stable release + of any of them is tested to check that it produces identical output to + the other two. This guarantees that all three implement the same + algorithm, and makes it unlikely that any of them may contain serious + undiscovered errors. In fact, no errors have been discovered in lzip + since 2009. + + Additionally, the three implementations have been extensively tested + with unzcrash, valgrind, and 'american fuzzy lop' without finding a + single vulnerability or false negative. *Note Unzcrash: + (lziprecover)Unzcrash. + +'Dictionary size' + Lzip automatically adapts the dictionary size to the size of each file. + In addition to reducing the amount of memory required for + decompression, this feature also minimizes the probability of being + affected by RAM errors during compression. + +'Exit status' + Returning a warning status of 2 is a design flaw of compress that + leaked into the design of gzip. Both bzip2 and lzip are free from this + flaw. + + + +File: lzip.info, Node: Algorithm, Next: Trailing data, Prev: Quality assurance, Up: Top + +8 Algorithm +*********** + +In spite of its name (Lempel-Ziv-Markov chain-Algorithm), LZMA is not a +concrete algorithm; it is more like "any algorithm using the LZMA coding +scheme". LZMA compression consists in describing the uncompressed data as a +succession of coding sequences from the set shown in Section 'What is +coded' (*note what-is-coded::), and then encoding them using a range +encoder. For example, the option '-0' of lzip uses the scheme in almost the +simplest way possible; issuing the longest match it can find, or a literal +byte if it can't find a match. Inversely, a more elaborate way of finding +coding sequences of minimum size than the one currently used by lzip could +be developed, and the resulting sequence could also be coded using the LZMA +coding scheme. + + Lzip currently implements two variants of the LZMA algorithm: fast (used +by option '-0') and normal (used by all other compression levels). + + The high compression of LZMA comes from combining two basic, well-proven +compression ideas: sliding dictionaries (LZ77) and Markov models (the thing +used by every compression algorithm that uses a range encoder or similar +order-0 entropy coder as its last stage) with segregation of contexts +according to what the bits are used for. + + Lzip is a two stage compressor. The first stage is a Lempel-Ziv coder, +which reduces redundancy by translating chunks of data to their +corresponding distance-length pairs. The second stage is a range encoder +that uses a different probability model for each type of data: distances, +lengths, literal bytes, etc. + + Here is how it works, step by step: + + 1) The member header is written to the output stream. + + 2) The first byte is coded literally, because there are no previous +bytes to which the match finder can refer to. + + 3) The main encoder advances to the next byte in the input data and +calls the match finder. + + 4) The match finder fills an array with the minimum distances before the +current byte where a match of a given length can be found. + + 5) Go back to step 3 until a sequence (formed of pairs, repeated +distances, and literal bytes) of minimum price has been formed. Where the +price represents the number of output bits produced. + + 6) The range encoder encodes the sequence produced by the main encoder +and sends the bytes produced to the output stream. + + 7) Go back to step 3 until the input data are finished or until the +member or volume size limits are reached. + + 8) The range encoder is flushed. + + 9) The member trailer is written to the output stream. + + 10) If there are more data to compress, go back to step 1. + + + During compression, lzip reads data in large blocks (one dictionary size +at a time). Therefore it may block for up to tens of seconds any process +feeding data to it through a pipe. This is normal. The blocking intervals +get longer with higher compression levels because dictionary size increases +(and compression speed decreases) with compression level. + +The ideas embodied in lzip are due to (at least) the following people: +Abraham Lempel and Jacob Ziv (for the LZ algorithm), Andrei Markov (for the +definition of Markov chains), G.N.N. Martin (for the definition of range +encoding), Igor Pavlov (for putting all the above together in LZMA), and +Julian Seward (for bzip2's CLI). + + +File: lzip.info, Node: Trailing data, Next: Examples, Prev: Algorithm, Up: Top + +9 Extra data appended to the file ********************************* Sometimes extra data are found appended to a lzip file after the last @@ -1037,7 +1065,7 @@ member. Such trailing data may be: example when writing to a tape. It is safe to append any amount of padding zero bytes to a lzip file. - * Useful data added by the user; an "End Of File" string (to check that + * Useful data added by the user; an 'End Of File' string (to check that the file has not been truncated), a cryptographically secure hash, a description of file contents, etc. It is safe to append any amount of text to a lzip file as long as none of the first four bytes of the @@ -1073,8 +1101,8 @@ where a file containing trailing data must be rejected, the option File: lzip.info, Node: Examples, Next: Problems, Prev: Trailing data, Up: Top -9 A small tutorial with examples -******************************** +10 A small tutorial with examples +********************************* WARNING! Even if lzip is bug-free, other causes may result in a corrupt compressed file (bugs in the system libraries, memory errors, etc). @@ -1086,38 +1114,32 @@ comparing the compressed file with the original because the corruption happens before lzip compresses the RAM contents, resulting in a valid compressed file containing wrong data. - Example 1: Extract all the files from archive 'foo.tar.lz'. tar -xf foo.tar.lz or lzip -cd foo.tar.lz | tar -xf - - Example 2: Replace a regular file with its compressed version 'file.lz' and show the compression ratio. lzip -v file - Example 3: Like example 2 but the created 'file.lz' is multimember with a member size of 1 MiB. The compression ratio is not shown. lzip -b 1MiB file - Example 4: Restore a regular file from its compressed version 'file.lz'. If the operation is successful, 'file.lz' is removed. lzip -d file.lz - Example 5: Check the integrity of the compressed file 'file.lz' and show status. lzip -tv file.lz - Example 6: The right way of concatenating the decompressed output of two or more compressed files. *Note Trailing data::. @@ -1126,19 +1148,16 @@ more compressed files. *Note Trailing data::. Do this instead lzip -cd file1.lz file2.lz file3.lz - Example 7: Decompress 'file.lz' partially until 10 KiB of decompressed data are produced. lzip -cd file.lz | dd bs=1024 count=10 - Example 8: Decompress 'file.lz' partially from decompressed byte at offset 10000 to decompressed byte at offset 14999 (5000 bytes are produced). lzip -cd file.lz | dd bs=1000 skip=10 count=5 - Example 9: Compress a whole device in /dev/sdc and send the output to 'file.lz'. @@ -1146,18 +1165,15 @@ Example 9: Compress a whole device in /dev/sdc and send the output to or lzip /dev/sdc -o file.lz - Example 10: Create a multivolume compressed tar archive with a volume size of 1440 KiB. tar -c some_directory | lzip -S 1440KiB -o volume_name - - Example 11: Extract a multivolume compressed tar archive. lzip -cd volume_name*.lz | tar -xf - - Example 12: Create a multivolume compressed backup of a large database file with a volume size of 650 MB, where each volume is a multimember file with a member size of 32 MiB. @@ -1167,7 +1183,7 @@ a member size of 32 MiB. File: lzip.info, Node: Problems, Next: Reference source code, Prev: Examples, Up: Top -10 Reporting bugs +11 Reporting bugs ***************** There are probably bugs in lzip. There are certainly errors and omissions @@ -1336,7 +1352,8 @@ public: Range_decoder() : member_pos( header_size ), code( 0 ), range( 0xFFFFFFFFU ) { - get_byte(); // discard first byte of the LZMA stream + if( get_byte() != 0 ) // check first LZMA byte + { std::fputs( "Nonzero first LZMA byte.\n", stderr ); std::exit( 2 ); } for( int i = 0; i < 4; ++i ) code = ( code << 8 ) | get_byte(); } @@ -1579,8 +1596,7 @@ bool LZ_decoder::decode_member() // Return false if error direct_bits ); else { - rep0 += - rdec.decode( direct_bits - dis_align_bits ) << dis_align_bits; + rep0 += rdec.decode( direct_bits-dis_align_bits ) << dis_align_bits; rep0 += rdec.decode_tree_reversed( bm_align, dis_align_bits ); if( rep0 == 0xFFFFFFFFU ) // marker found { @@ -1612,8 +1628,8 @@ int main( const int argc, const char * const argv[] ) "Lzd decompresses from standard input to standard output.\n" "\nCopyright (C) 2024 Antonio Diaz Diaz.\n" "License 2-clause BSD.\n" - "This is free software: you are free to change and redistribute it.\n" - "There is NO WARRANTY, to the extent permitted by law.\n" + "This is free software: you are free to change and redistribute " + "it.\nThere is NO WARRANTY, to the extent permitted by law.\n" "Report bugs to lzip-bug@nongnu.org\n" "Lzd home page: http://www.nongnu.org/lzip/lzd.html\n", PROGVERSION, argv[0] ); @@ -1625,6 +1641,7 @@ int main( const int argc, const char * const argv[] ) setmode( STDOUT_FILENO, O_BINARY ); #endif + bool empty = false, multi = false; for( bool first_member = true; ; first_member = false ) { Lzip_header header; // check header @@ -1639,12 +1656,12 @@ int main( const int argc, const char * const argv[] ) unsigned dict_size = 1 << ( header[5] & 0x1F ); dict_size -= ( dict_size / 16 ) * ( ( header[5] >> 5 ) & 7 ); if( dict_size < min_dictionary_size || dict_size > max_dictionary_size ) - { std::fputs( "Invalid dictionary size in member header.\n", stderr ); - return 2; } + { std::fputs( "Invalid dictionary size in member header.\n", + stderr ); return 2; } LZ_decoder decoder( dict_size ); // decode LZMA stream if( !decoder.decode_member() ) - { std::fputs( "Data error\n", stderr ); return 2; } + { std::fputs( "Data error.\n", stderr ); return 2; } Lzip_trailer trailer; // check trailer for( int i = 0; i < trailer_size; ++i ) trailer[i] = decoder.get_byte(); @@ -1652,25 +1669,28 @@ int main( const int argc, const char * const argv[] ) unsigned crc = 0; for( int i = 3; i >= 0; --i ) crc = ( crc << 8 ) + trailer[i]; if( crc != decoder.crc() ) - { std::fputs( "CRC mismatch\n", stderr ); retval = 2; } + { std::fputs( "CRC mismatch.\n", stderr ); retval = 2; } unsigned long long data_size = 0; for( int i = 11; i >= 4; --i ) data_size = ( data_size << 8 ) + trailer[i]; if( data_size != decoder.data_position() ) - { std::fputs( "Data size mismatch\n", stderr ); retval = 2; } + { std::fputs( "Data size mismatch.\n", stderr ); retval = 2; } + multi = !first_member; if( data_size == 0 ) empty = true; unsigned long long member_size = 0; for( int i = 19; i >= 12; --i ) member_size = ( member_size << 8 ) + trailer[i]; if( member_size != decoder.member_position() ) - { std::fputs( "Member size mismatch\n", stderr ); retval = 2; } + { std::fputs( "Member size mismatch.\n", stderr ); retval = 2; } if( retval ) return retval; } if( std::fclose( stdout ) != 0 ) { std::fprintf( stderr, "Error closing stdout: %s\n", std::strerror( errno ) ); return 1; } + if( empty && multi ) + { std::fputs( "Empty member not allowed.\n", stderr ); return 2; } return 0; } @@ -1684,6 +1704,7 @@ Concept index * Menu: * algorithm: Algorithm. (line 6) +* argument syntax: Argument syntax. (line 6) * bugs: Problems. (line 6) * examples: Examples. (line 6) * file format: File format. (line 6) @@ -1703,22 +1724,23 @@ Concept index Tag Table: Node: Top203 -Node: Introduction1197 -Node: Output7082 -Node: Invoking lzip8677 -Ref: --trailing-error9546 -Node: Quality assurance19644 -Node: Algorithm28457 -Node: File format31858 -Ref: coded-dict-size33287 -Node: Stream format34518 -Ref: what-is-coded36913 -Node: Trailing data45787 -Node: Examples48123 -Ref: concat-example49563 -Node: Problems50784 -Node: Reference source code51316 -Node: Concept index66361 +Node: Introduction1273 +Node: Output6965 +Node: Invoking lzip8560 +Ref: --trailing-error9397 +Node: Argument syntax19605 +Node: File format21367 +Ref: coded-dict-size22865 +Node: Stream format24097 +Ref: what-is-coded26621 +Node: Quality assurance35351 +Node: Algorithm44122 +Node: Trailing data47519 +Node: Examples49851 +Ref: concat-example51287 +Node: Problems52502 +Node: Reference source code53034 +Node: Concept index68345 End Tag Table diff --git a/doc/lzip.texi b/doc/lzip.texi index 865a413..9056169 100644 --- a/doc/lzip.texi +++ b/doc/lzip.texi @@ -6,8 +6,8 @@ @finalout @c %**end of header -@set UPDATED 1 March 2024 -@set VERSION 1.24.1 +@set UPDATED 26 November 2024 +@set VERSION 1.25-rc1 @dircategory Compression @direntry @@ -39,10 +39,11 @@ This manual is for Lzip (version @value{VERSION}, @value{UPDATED}). * Introduction:: Purpose and features of lzip * Output:: Meaning of lzip's output * Invoking lzip:: Command-line interface -* Quality assurance:: Design, development, and testing of lzip -* Algorithm:: How lzip compresses the data +* Argument syntax:: By convention, options start with a hyphen * File format:: Detailed format of the compressed file * Stream format:: Format of the LZMA stream in lzip files +* Quality assurance:: Design, development, and testing of lzip +* Algorithm:: How lzip compresses the data * Trailing data:: Extra data appended to the file * Examples:: A small tutorial with examples * Problems:: Reporting bugs @@ -64,16 +65,16 @@ distribute, and modify it. @uref{http://www.nongnu.org/lzip/lzip.html,,Lzip} is a lossless data compressor with a user interface similar to the one -of gzip or bzip2. Lzip uses a simplified form of the 'Lempel-Ziv-Markov -chain-Algorithm' (LZMA) stream format to maximize interoperability. The -maximum dictionary size is 512 MiB so that any lzip file can be decompressed -on 32-bit machines. Lzip provides accurate and robust 3-factor integrity -checking. Lzip can compress about as fast as gzip @w{(lzip -0)} or compress most -files more than bzip2 @w{(lzip -9)}. Decompression speed is intermediate between -gzip and bzip2. Lzip is better than gzip and bzip2 from a data recovery -perspective. Lzip has been designed, written, and tested with great care to -replace gzip and bzip2 as the standard general-purpose compressed format for -Unix-like systems. +of gzip or bzip2. Lzip uses a simplified form of LZMA (Lempel-Ziv-Markov +chain-Algorithm) designed to achieve complete interoperability between +implementations. The maximum dictionary size is 512 MiB so that any lzip +file can be decompressed on 32-bit machines. Lzip provides accurate and +robust 3-factor integrity checking. @w{@samp{lzip -0}} compresses about as fast as +gzip, while @w{@samp{lzip -9}} compresses most files more than bzip2. Decompression +speed is intermediate between gzip and bzip2. Lzip provides better data +recovery capabilities than gzip and bzip2. Lzip has been designed, written, +and tested with great care to replace gzip and bzip2 as general-purpose +compressed format for Unix-like systems. For compressing/decompressing large files on multiprocessor machines @uref{http://www.nongnu.org/lzip/manual/plzip_manual.html,,plzip} can be @@ -95,8 +96,7 @@ taking into account both data integrity and decoder availability: @itemize @bullet @item -The lzip format provides very safe integrity checking and some data -recovery means. The program +The program @uref{http://www.nongnu.org/lzip/manual/lziprecover_manual.html#Data-safety,,lziprecover} can repair bit flip errors (one of the most common forms of data corruption) in lzip files, and provides data recovery capabilities, including @@ -125,7 +125,7 @@ the beginning is a thing of the past. The member trailer stores the 32-bit CRC of the original data, the size of the original data, and the size of the member. These values, together with -the "End Of Stream" marker, provide a 3-factor integrity checking which +the 'End Of Stream' marker, provide a 3-factor integrity checking which guarantees that the decompressed version of the data is identical to the original. This guards against corruption of the compressed data, and against undetected bugs in lzip (hopefully very unlikely). The chances of data @@ -138,9 +138,8 @@ makes it safer than compressors returning ambiguous warning values (like gzip) when it is used as a back end for other programs like tar or zutils. Lzip automatically uses for each file the largest dictionary size that does -not exceed neither the file size nor the limit given. Keep in mind that the -decompression memory requirement is affected at compression time by the -choice of dictionary size limit. +not exceed neither the file size nor the limit given. The dictionary size +used for decompression is the same dictionary size used for compression. The amount of memory required for compression is about 1 or 2 times the dictionary size limit (1 if input file size is less than dictionary size @@ -181,7 +180,6 @@ also supported. Lzip can produce multimember files, and lziprecover can safely recover the undamaged members in case of file damage. Lzip can also split the compressed output in volumes of a given size, even when reading from standard input. -This allows the direct creation of multivolume compressed tar archives. Lzip is able to compress and decompress streams of unlimited size by automatically creating multimember output. The members so created are large, @@ -258,11 +256,8 @@ argument means standard input. It can be mixed with other @var{files} and is read just once, the first time it appears in the command line. Remember to prepend @file{./} to any file name beginning with a hyphen, or use @samp{--}. -lzip supports the following -@uref{http://www.nongnu.org/arg-parser/manual/arg_parser_manual.html#Argument-syntax,,options}: -@ifnothtml -@xref{Argument syntax,,,arg_parser}. -@end ifnothtml +@noindent +lzip supports the following options: @xref{Argument syntax}. @table @code @item -h @@ -283,9 +278,10 @@ garbage that can be safely ignored. @xref{concat-example}. @item -b @var{bytes} @itemx --member-size=@var{bytes} -When compressing, set the member size limit to @var{bytes}. It is advisable -to keep members smaller than RAM size so that they can be repaired with -lziprecover in case of corruption. A small member size may degrade +When compressing, set the member size limit to @var{bytes}. If @var{bytes} +is smaller than the compressed size, a multimember file is produced. It is +advisable to keep members smaller than RAM size so that they can be repaired +with lziprecover in case of corruption. A small member size may degrade compression ratio, so use it only when needed. Valid values range from @w{100 kB} to @w{2 PiB}. Defaults to @w{2 PiB}. @@ -307,7 +303,8 @@ already exists and @option{--force} has not been specified, lzip continues decompressing the rest of the files and exits with error status 1. If a file fails to decompress, or is a terminal, lzip exits immediately with error status 2 without decompressing the rest of the files. A terminal is -considered an uncompressed file, and therefore invalid. +considered an uncompressed file, and therefore invalid. A multimember file +with one or more empty members is accepted if redirected to standard input. @item -f @itemx --force @@ -316,7 +313,7 @@ Force overwrite of output files. @item -F @itemx --recompress When compressing, force re-compression of files whose name already has -the @samp{.lz} or @samp{.tlz} suffix. +the @file{.lz} or @file{.tlz} suffix. @item -k @itemx --keep @@ -330,7 +327,8 @@ even for multimember files. If more than one file is given, a final line containing the cumulative sizes is printed. With @option{-v}, the dictionary size, the number of members in the file, and the amount of trailing data (if any) are also printed. With @option{-vv}, the positions and sizes of each -member in multimember files are also printed. +member in multimember files are also printed. A multimember file with one or +more empty members is accepted if redirected to standard input. If any file is damaged, does not exist, can't be opened, or is not regular, the final exit status is @w{> 0}. @option{-lq} can be used to check quickly @@ -357,15 +355,15 @@ to @option{-c}. @option{-o} has no effect when testing or listing. In order to keep backward compatibility with lzip versions prior to 1.22, when compressing from standard input and no other file names are given, the -extension @samp{.lz} is appended to @var{file} unless it already ends in -@samp{.lz} or @samp{.tlz}. This feature will be removed in a future version +extension @file{.lz} is appended to @var{file} unless it already ends in +@file{.lz} or @file{.tlz}. This feature will be removed in a future version of lzip. Meanwhile, redirection may be used instead of @option{-o} to write -the compressed output to a file without the extension @samp{.lz} in its +the compressed output to a file without the extension @file{.lz} in its name: @w{@samp{lzip < file > foo}}. When compressing and splitting the output in volumes, @var{file} is used as -a prefix, and several files named @samp{@var{file}00001.lz}, -@samp{@var{file}00002.lz}, etc, are created. In this case, only one input +a prefix, and several files named @file{@var{file}00001.lz}, +@file{@var{file}00002.lz}, etc, are created. In this case, only one input file is allowed. @item -q @@ -391,7 +389,7 @@ is affected at compression time by the choice of dictionary size limit. @itemx --volume-size=@var{bytes} When compressing, and @option{-c} has not been also specified, split the compressed output into several volume files with names -@samp{original_name00001.lz}, @samp{original_name00002.lz}, etc, and set the +@file{original_name00001.lz}, @file{original_name00002.lz}, etc, and set the volume size limit to @var{bytes}. Input files are kept unchanged. Each volume is a complete, maybe multimember, lzip file. A small volume size may degrade compression ratio, so use it only when needed. Valid values range @@ -405,18 +403,17 @@ together with @option{-v} to see information about the files. If a file fails the test, does not exist, can't be opened, or is a terminal, lzip continues testing the rest of the files. A final diagnostic is shown at verbosity level 1 or higher if any file fails the test when testing multiple -files. +files. A multimember file with one or more empty members is accepted if +redirected to standard input. @item -v @itemx --verbose Verbose mode.@* -When compressing, show the compression ratio and size for each file -processed.@* -When decompressing or testing, further -v's (up to 4) increase the -verbosity level, showing status, compression ratio, dictionary size, -trailer contents (CRC, data size, member size), and up to 6 bytes of -trailing data (if any) both in hexadecimal and as a string of printable -ASCII characters.@* +When compressing, show the compression ratio and size for each file processed.@* +When decompressing or testing, further -v's (up to 4) increase the verbosity +level, showing status, compression ratio, dictionary size, trailer contents +(CRC, data size, member size), and up to 6 bytes of trailing data (if any) +both in hexadecimal and as a string of printable ASCII characters.@* Two or more @option{-v} options show the progress of (de)compression. @item -0 .. -9 @@ -436,7 +433,7 @@ given, the last setting is used. For example @w{@option{-9 -s64MiB}} is equivalent to @w{@option{-s64MiB -m273}} @multitable {Level} {Dictionary size (-s)} {Match length limit (-m)} -@item Level @tab Dictionary size (-s) @tab Match length limit (-m) +@headitem Level @tab Dictionary size (-s) @tab Match length limit (-m) @item -0 @tab 64 KiB @tab 16 bytes @item -1 @tab 1 MiB @tab 5 bytes @item -2 @tab 1.5 MiB @tab 6 bytes @@ -453,20 +450,11 @@ equivalent to @w{@option{-s64MiB -m273}} @itemx --best Aliases for GNU gzip compatibility. -@item --empty-error -Exit with error status 2 if any empty member is found in the input files. - -@item --marking-error -Exit with error status 2 if the first LZMA byte is non-zero in any member of -the input files. This may be caused by data corruption or by deliberate -insertion of tracking information in the file. Use -@w{@samp{lziprecover --clear-marking}} to clear any such non-zero bytes. - @item --loose-trailing When decompressing, testing, or listing, allow trailing data whose first bytes are so similar to the magic bytes of a lzip header that they can be confused with a corrupt header. Use this option if a file triggers a -"corrupt header" error and the cause is not indeed a corrupt header. +'corrupt header' error and the cause is not indeed a corrupt header. @end table @@ -477,7 +465,7 @@ and may be followed by a multiplier and an optional @samp{B} for "byte". Table of SI and binary prefixes (unit multipliers): @multitable {Prefix} {kilobyte (10^3 = 1000)} {|} {Prefix} {kibibyte (2^10 = 1024)} -@item Prefix @tab Value @tab | @tab Prefix @tab Value +@headitem Prefix @tab Value @tab | @tab Prefix @tab Value @item k @tab kilobyte (10^3 = 1000) @tab | @tab Ki @tab kibibyte (2^10 = 1024) @item M @tab megabyte (10^6) @tab | @tab Mi @tab mebibyte (2^20) @item G @tab gigabyte (10^9) @tab | @tab Gi @tab gibibyte (2^30) @@ -497,273 +485,51 @@ indicate a corrupt or invalid input file, 3 for an internal consistency error (e.g., bug) which caused lzip to panic. -@node Quality assurance -@chapter Design, development, and testing of lzip -@cindex quality assurance - -There are two ways of constructing a software design: One way is to make it -so simple that there are obviously no deficiencies and the other way is to -make it so complicated that there are no obvious deficiencies. The first -method is far more difficult.@* ---- C.A.R. Hoare - -Lzip has been designed, written, and tested with great care to replace gzip -and bzip2 as the standard general-purpose compressed format for Unix-like -systems. This chapter describes the lessons learned from these previous -formats, and their application to the design of lzip. The lzip format -specification has been reviewed carefully and is believed to be free from -design errors. - -@sp 1 -@section Format design - -When gzip was designed in 1992, computers and operating systems were much -less capable than they are today. The designers of gzip tried to work around -some of those limitations, like 8.3 file names, with additional fields in -the file format. - -Today those limitations have mostly disappeared, and the format of gzip has -proved to be unnecessarily complicated. It includes fields that were never -used, others that have lost their usefulness, and finally others that have -become too limited. - -Bzip2 was designed 5 years later, and its format is simpler than the one of -gzip. - -Probably the worst defect of the gzip format from the point of view of data -safety is the variable size of its header. If the byte at offset 3 (flags) -of a gzip member gets corrupted, it may become difficult to recover the -data, even if the compressed blocks are intact, because it can't be known -with certainty where the compressed blocks begin. - -By contrast, the header of a lzip member has a fixed length of 6. The LZMA -stream in a lzip member always starts at offset 6, making it trivial to -recover the data even if the whole header becomes corrupt. +@node Argument syntax +@chapter Syntax of command-line arguments +@cindex argument syntax -Bzip2 also provides a header of fixed length and marks the begin and end of -each compressed block with six magic bytes, making it possible to find the -compressed blocks even in case of file damage. But bzip2 does not store the -size of each compressed block, as lzip does. +POSIX recommends these conventions for command-line arguments. -Lziprecover is able to provide unique data recovery capabilities because the -lzip format is extraordinarily safe. The simple and safe design of the file -format complements the embedded error detection provided by the LZMA data -stream. Any distance larger than the dictionary size acts as a forbidden -symbol, allowing the decompressor to detect the approximate position of -errors, and leaving very little work for the check sequence (CRC and data -sizes) in the detection of errors. Lzip is usually able to detect all -possible bit flips in the compressed data without resorting to the check -sequence. It would be difficult to write an automatic recovery tool like -lziprecover for the gzip format. And, as far as I know, it has never been -written. - -Lzip, like gzip and bzip2, uses a CRC32 to check the integrity of the -decompressed data because it provides optimal accuracy in the detection of -errors up to a compressed size of about @w{16 GiB}, a size larger than that -of most files. In the case of lzip, the additional detection capability of -the decompressor reduces the probability of undetected errors several -million times more, resulting in a combined integrity checking optimally -accurate for any member size produced by lzip. Preliminary results suggest -that the lzip format is safe enough to be used in critical safety avionics -systems. - -The lzip format is designed for long-term archiving. Therefore it excludes -any unneeded features that may interfere with the future extraction of the -decompressed data. - -@sp 1 -@subsection Gzip format (mis)features not present in lzip - -@table @samp -@item Multiple algorithms - -Gzip provides a CM (Compression Method) field that has never been used -because it is a bad idea to begin with. New compression methods may require -additional fields, making it impossible to implement new methods and, at the -same time, keep the same format. This field does not solve the problem of -format proliferation; it just makes the problem less obvious. - -@item Optional fields in header - -Unless special precautions are taken, optional fields are generally a bad -idea because they produce a header of variable size. The gzip header has 2 -fields that, in addition to being optional, are zero-terminated. This means -that if any byte inside the field gets zeroed, or if the terminating zero -gets altered, gzip won't be able to find neither the header CRC nor the -compressed blocks. - -@item Optional CRC for the header - -Using an optional CRC for the header is not only a bad idea, it is an error; -it circumvents the Hamming distance (HD) of the CRC and may prevent the -extraction of perfectly good data. For example, if the CRC is used and the -bit enabling it is reset by a bit flip, then the header seems to be intact -(in spite of being corrupt) while the compressed blocks seem to be totally -unrecoverable (in spite of being intact). Very misleading indeed. - -@item Metadata - -The gzip format stores some metadata, like the modification time of the -original file or the operating system on which compression took place. This -complicates reproducible compression (obtaining identical compressed output -from identical input). - -@end table - -@subsection Lzip format improvements over gzip and bzip2 - -@table @samp -@item 64-bit size field - -Probably the most frequently reported shortcoming of the gzip format is that -it only stores the least significant 32 bits of the uncompressed size. The -size of any file larger or equal than @w{4 GiB} gets truncated. - -Bzip2 does not store the uncompressed size of the file. - -The lzip format provides a 64-bit field for the uncompressed size. -Additionally, lzip produces multimember output automatically when the size -is too large for a single member, allowing for an unlimited uncompressed -size. - -@item Distributed index - -The lzip format provides a distributed index that, among other things, helps -plzip to decompress several times faster than pigz and helps lziprecover do -its job. Neither the gzip format nor the bzip2 format do provide an index. - -A distributed index is safer and more scalable than a monolithic index. The -monolithic index introduces a single point of failure in the compressed file -and may limit the number of members or the total uncompressed size. - -@end table - -@section Quality of implementation - -Our civilization depends critically on software; it had better be quality -software.@* ---- Bjarne Stroustrup - -@table @samp -@item Accurate and robust error detection - -The lzip format provides 3-factor integrity checking, and the decompressors -report mismatches in each factor separately. This method detects most false -positives for corruption. If just one byte in one factor fails but the other -two factors match the data, it probably means that the data are intact and -the corruption just affects the mismatching factor (CRC, data size, or -member size) in the member trailer. - -@item Multiple implementations - -Just like the lzip format provides 3-factor protection against undetected -data corruption, the development methodology of the lzip family of -compressors provides 3-factor protection against undetected programming -errors. - -Three related but independent compressor implementations, lzip, clzip, and -minilzip/lzlib, are developed concurrently. Every stable release of any of -them is tested to check that it produces identical output to the other two. -This guarantees that all three implement the same algorithm, and makes it -unlikely that any of them may contain serious undiscovered errors. In fact, -no errors have been discovered in lzip since 2009. - -Additionally, the three implementations have been extensively tested with -@uref{http://www.nongnu.org/lzip/manual/lziprecover_manual.html#Unzcrash,,unzcrash}, -valgrind, and @samp{american fuzzy lop} without finding a single -vulnerability or false negative. -@ifnothtml -@xref{Unzcrash,,,lziprecover}. -@end ifnothtml - -@item Dictionary size - -Lzip automatically adapts the dictionary size to the size of each file. -In addition to reducing the amount of memory required for decompression, -this feature also minimizes the probability of being affected by RAM errors -during compression. @c key4_mask - -@item Exit status - -Returning a warning status of 2 is a design flaw of compress that leaked -into the design of gzip. Both bzip2 and lzip are free from this flaw. - -@end table - - -@node Algorithm -@chapter Algorithm -@cindex algorithm - -In spite of its name (Lempel-Ziv-Markov chain-Algorithm), LZMA is not a -concrete algorithm; it is more like "any algorithm using the LZMA coding -scheme". LZMA compression consists in describing the uncompressed data as a -succession of coding sequences from the set shown in Section @samp{What is -coded} (@pxref{what-is-coded}), and then encoding them using a range -encoder. For example, the option @option{-0} of lzip uses the scheme in almost -the simplest way possible; issuing the longest match it can find, or a -literal byte if it can't find a match. Inversely, a much more elaborated way -of finding coding sequences of minimum size than the one currently used by -lzip could be developed, and the resulting sequence could also be coded -using the LZMA coding scheme. - -Lzip currently implements two variants of the LZMA algorithm: fast -(used by option @option{-0}) and normal (used by all other compression levels). - -The high compression of LZMA comes from combining two basic, well-proven -compression ideas: sliding dictionaries (LZ77) and Markov models (the thing -used by every compression algorithm that uses a range encoder or similar -order-0 entropy coder as its last stage) with segregation of contexts -according to what the bits are used for. - -Lzip is a two stage compressor. The first stage is a Lempel-Ziv coder, -which reduces redundancy by translating chunks of data to their -corresponding distance-length pairs. The second stage is a range encoder -that uses a different probability model for each type of data: -distances, lengths, literal bytes, etc. - -Here is how it works, step by step: +@itemize @bullet +@item A command-line argument is an option if it begins with a hyphen +(@samp{-}). -1) The member header is written to the output stream. +@item Option names are single alphanumeric characters. -2) The first byte is coded literally, because there are no previous -bytes to which the match finder can refer to. +@item Certain options require an argument. -3) The main encoder advances to the next byte in the input data and -calls the match finder. +@item An option and its argument may or may not appear as separate tokens. +(In other words, the whitespace separating them is optional). +Thus, @w{@option{-o foo}} and @option{-ofoo} are equivalent. -4) The match finder fills an array with the minimum distances before the -current byte where a match of a given length can be found. +@item One or more options without arguments, followed by at most one option +that takes an argument, may follow a hyphen in a single token. +Thus, @option{-abc} is equivalent to @w{@option{-a -b -c}}. -5) Go back to step 3 until a sequence (formed of pairs, repeated -distances, and literal bytes) of minimum price has been formed. Where the -price represents the number of output bits produced. +@item Options typically precede other non-option arguments. -6) The range encoder encodes the sequence produced by the main encoder -and sends the bytes produced to the output stream. +@item The argument @samp{--} terminates all options; any following arguments +are treated as non-option arguments, even if they begin with a hyphen. -7) Go back to step 3 until the input data are finished or until the -member or volume size limits are reached. - -8) The range encoder is flushed. - -9) The member trailer is written to the output stream. - -10) If there are more data to compress, go back to step 1. - -@sp 1 -During compression, lzip reads data in large blocks (one dictionary size at -a time). Therefore it may block for up to tens of seconds any process -feeding data to it through a pipe. This is normal. The blocking intervals -get longer with higher compression levels because dictionary size increases -(and compression speed decreases) with compression level. +@item A token consisting of a single hyphen character is interpreted as an +ordinary non-option argument. By convention, it is used to specify standard +input, standard output, or a file named @samp{-}. +@end itemize @noindent -The ideas embodied in lzip are due to (at least) the following people: -Abraham Lempel and Jacob Ziv (for the LZ algorithm), Andrei Markov (for the -definition of Markov chains), G.N.N. Martin (for the definition of range -encoding), Igor Pavlov (for putting all the above together in LZMA), and -Julian Seward (for bzip2's CLI). +GNU adds @dfn{long options} to these conventions: + +@itemize @bullet +@item A long option consists of two hyphens (@samp{--}) followed by a name +made of alphanumeric characters and hyphens. Option names are typically one +to three words long, with hyphens to separate words. Abbreviations can be +used for the long option names as long as the abbreviations are unique. + +@item A long option and its argument may or may not appear as separate +tokens. In the latter case they must be separated by an equal sign @samp{=}. +Thus, @w{@option{--foo bar}} and @option{--foo=bar} are equivalent. +@end itemize @node File format @@ -774,7 +540,6 @@ Perfection is reached, not when there is no longer anything to add, but when there is no longer anything to take away.@* --- Antoine de Saint-Exupery -@sp 1 In the diagram below, a box like this: @verbatim @@ -793,12 +558,13 @@ represents one byte; a box like this: represents a variable number of bytes. -@sp 1 +@noindent A lzip file consists of one or more independent "members" (compressed data sets). The members simply appear one after another in the file, with no additional information before, between, or after them. Each member can encode in compressed form up to @w{16 EiB - 1 byte} of uncompressed data. -The size of a multimember file is unlimited. +The size of a multimember file is unlimited. Empty members (data size = 0) +are not allowed in multimember files. Each member has the following structure: @@ -829,7 +595,7 @@ Example: 0xD3 = 2^19 - 6 * 2^15 = 512 KiB - 6 * 32 KiB = 320 KiB@* Valid values for dictionary size range from 4 KiB to 512 MiB. @item LZMA stream -The LZMA stream, finished by an "End Of Stream" marker. Uses default values +The LZMA stream, terminated by an 'End Of Stream' marker. Uses default values for encoder properties. @xref{Stream format}, for a complete description. @item CRC32 (4 bytes) @@ -844,7 +610,6 @@ as a distributed index, improves the checking of stream integrity, and facilitates the safe recovery of undamaged members from multimember files. Lzip limits the member size to @w{2 PiB} to prevent the data size field from overflowing. - @end table @@ -852,8 +617,8 @@ overflowing. @chapter Format of the LZMA stream in lzip files @cindex format of the LZMA stream -The LZMA algorithm has three parameters, called "special LZMA -properties", to adjust it for some kinds of binary data. These +The LZMA algorithm has three parameters, called 'special LZMA +properties', to adjust it for some kinds of binary data. These parameters are: @samp{literal_context_bits} (with a default value of 3), @samp{literal_pos_state_bits} (with a default value of 0), and @samp{pos_state_bits} (with a default value of 2). As a general purpose @@ -861,14 +626,17 @@ compressor, lzip only uses the default values for these parameters. In particular @samp{literal_pos_state_bits} has been optimized away and does not even appear in the code. -Lzip finishes the LZMA stream with an "End Of Stream" (EOS) marker (the +The first byte of the LZMA stream is set to zero to help tools like grep +recognize lzip files as binary files. + +The LZMA stream is terminated by an 'End Of Stream' (EOS) marker (the distance-length pair @w{0xFFFFFFFFU, 2}), which in conjunction with the @samp{member size} field in the member trailer allows the checking of stream integrity. The EOS marker is the only LZMA marker allowed in lzip files. The LZMA stream in lzip files always has these two features (default properties and EOS marker) and is referred to in this document as LZMA-302eos. This simplified and marker-terminated form of the LZMA stream format has been -chosen to maximize interoperability and safety. +chosen to achieve complete interoperability and robust safety. The second stage of LZMA is a range encoder that uses a different probability model for each type of symbol: distances, lengths, literal @@ -885,12 +653,11 @@ about the range decoder that need to be described accurately, the source code of a real decompressor seems the only appropriate reference to use. What follows is a description of the decoding algorithm for LZMA-302eos -streams using as reference the source code of "lzd", an educational +streams using as reference the source code of lzd, an educational decompressor for lzip files, included in appendix A. @xref{Reference source code}. Lzd is written in C++11 and can be downloaded from the lzip download directory. -@sp 1 @section What is coded @anchor{what-is-coded} @@ -1066,7 +833,6 @@ decoded that is different from its corresponding bit in byte is decoded using the normal bit tree context. (See @samp{decode_matched} in the source). -@sp 1 @section The range decoder The LZMA stream is consumed one byte at a time by the range decoder. @@ -1080,25 +846,287 @@ not yet decoded) and @samp{code} (representing the current point within @samp{range}). @samp{range} is initialized to @w{2^32 - 1}, and @samp{code} is initialized to 0. -The range encoder produces a first 0 byte that must be ignored by the -range decoder. (See the @samp{Range_decoder} constructor in the source). - -@sp 1 @section Decoding and checking the LZMA stream After decoding the member header and obtaining the dictionary size, the range decoder is initialized and then the LZMA decoder enters a loop (see @samp{decode_member} in the source) where it invokes the range decoder with the appropriate contexts to decode the different coding -sequences (matches, repeated matches, and literal bytes), until the "End -Of Stream" marker is decoded. +sequences (matches, repeated matches, and literal bytes), until the 'End +Of Stream' marker is decoded. -Once the "End Of Stream" marker has been decoded, the decompressor reads and +Once the 'End Of Stream' marker has been decoded, the decompressor reads and decodes the member trailer, and checks that the three integrity factors stored there (CRC, data size, and member size) match those computed from the data. +@node Quality assurance +@chapter Design, development, and testing of lzip +@cindex quality assurance + +There are two ways of constructing a software design: One way is to make it +so simple that there are obviously no deficiencies and the other way is to +make it so complicated that there are no obvious deficiencies. The first +method is far more difficult.@* +--- C.A.R. Hoare + +Lzip has been designed, written, and tested with great care to replace gzip +and bzip2 as general-purpose compressed format for Unix-like systems. This +chapter describes the lessons learned from these previous formats, and their +application to the design of lzip. The lzip format specification has been +reviewed carefully and is believed to be free from design errors. + +@section Format design + +When gzip was designed in 1992, computers and operating systems were less +capable than they are today. The designers of gzip tried to work around some +of those limitations, like 8.3 file names, with additional fields in the +file format. + +Today those limitations have mostly disappeared, and the format of gzip has +proved to be unnecessarily complicated. It includes fields that were never +used, others that have lost their usefulness, and finally others that have +become too limited. + +Bzip2 was designed 5 years later, and its format is simpler than the one of +gzip. + +Probably the worst defect of the gzip format from the point of view of data +safety is the variable size of its header. If the byte at offset 3 (flags) +of a gzip member gets corrupted, it may become difficult to recover the +data, even if the compressed blocks are intact, because it can't be known +with certainty where the compressed blocks begin. + +By contrast, the header of a lzip member has a fixed length of 6. The LZMA +stream in a lzip member always starts at offset 6, making it trivial to +recover the data even if the whole header becomes corrupt. + +Bzip2 also provides a header of fixed length and marks the begin and end of +each compressed block with six magic bytes, making it possible to find the +compressed blocks even in case of file damage. But bzip2 does not store the +size of each compressed block, as lzip does. + +Lziprecover is able to provide unique data recovery capabilities because the +lzip format is extraordinarily safe. The simple and safe design of the file +format complements the embedded error detection provided by the LZMA data +stream. Any distance larger than the dictionary size acts as a forbidden +symbol, allowing the decompressor to detect the approximate position of +errors, and leaving very little work for the check sequence (CRC and data +sizes) in the detection of errors. Lzip is usually able to detect all +possible bit flips in the compressed data without resorting to the check +sequence. It would be difficult to write an automatic recovery tool like +lziprecover for the gzip format. And, as far as I know, it has never been +written. + +Lzip, like gzip and bzip2, uses a CRC32 to check the integrity of the +decompressed data because it provides optimal accuracy in the detection of +errors up to a compressed size of about @w{16 GiB}, a size larger than that +of most files. In the case of lzip, the additional detection capability of +the decompressor reduces the probability of undetected errors several +million times more, resulting in a combined integrity checking optimally +accurate for any member size produced by lzip. Preliminary results suggest +that the lzip format is safe enough to be used in critical safety avionics +systems. + +The lzip format is designed for long-term archiving. Therefore it excludes +any unneeded features that may interfere with the future extraction of the +decompressed data. + +@subsection Gzip format (mis)features not present in lzip + +@table @samp +@item Multiple algorithms + +Gzip provides a CM (Compression Method) field that has never been used +because it is a bad idea to begin with. New compression methods may require +additional fields, making it impossible to implement new methods and, at the +same time, keep the same format. This field does not solve the problem of +format proliferation; it just makes the problem less obvious. + +@item Optional fields in header + +Unless special precautions are taken, optional fields are generally a bad +idea because they produce a header of variable size. The gzip header has 2 +fields that, in addition to being optional, are zero-terminated. This means +that if any byte inside the field gets zeroed, or if the terminating zero +gets altered, gzip won't be able to find neither the header CRC nor the +compressed blocks. + +@item Optional CRC for the header + +Using an optional CRC for the header is not only a bad idea, it is an error; +it circumvents the Hamming distance (HD) of the CRC and may prevent the +extraction of perfectly good data. For example, if the CRC is used and the +bit enabling it is reset by a bit flip, then the header seems to be intact +(in spite of being corrupt) while the compressed blocks seem to be +unrecoverable (in spite of being intact). Very misleading indeed. + +@item Metadata + +The gzip format stores some metadata, like the modification time of the +original file or the operating system on which compression took place. This +complicates reproducible compression (obtaining identical compressed output +from identical input). + +@end table + +@subsection Lzip format improvements over gzip and bzip2 + +@table @samp +@item 64-bit size field + +Probably the most frequently reported shortcoming of the gzip format is that +it only stores the least significant 32 bits of the uncompressed size. The +size of any file larger or equal than @w{4 GiB} gets truncated. + +Bzip2 does not store the uncompressed size of the file. + +The lzip format provides a 64-bit field for the uncompressed size. +Additionally, lzip produces multimember output automatically when the size +is too large for a single member, allowing for an unlimited uncompressed +size. + +@item Distributed index + +The lzip format provides a distributed index that, among other things, helps +plzip to decompress faster than pigz and helps lziprecover do its job. +Neither the gzip format nor the bzip2 format do provide an index. + +A distributed index is safer and more scalable than a monolithic index. The +monolithic index introduces a single point of failure in the compressed file +and may limit the number of members or the total uncompressed size. + +@end table + +@section Quality of implementation + +Our civilization depends critically on software; it had better be quality +software.@* +--- Bjarne Stroustrup + +@table @samp +@item Accurate and robust error detection + +The lzip format provides 3-factor integrity checking, and the decompressors +report mismatches in each factor separately. This method detects most false +positives for corruption. If just one byte in one factor fails but the other +two factors match the data, it probably means that the data are intact and +the corruption just affects the mismatching factor (CRC, data size, or +member size) in the member trailer. + +@item Multiple implementations + +Just like the lzip format provides 3-factor protection against undetected +data corruption, the development methodology of the lzip family of +compressors provides 3-factor protection against undetected programming +errors. + +Three related but independent compressor implementations, lzip, clzip, and +minilzip/lzlib, are developed concurrently. Every stable release of any of +them is tested to check that it produces identical output to the other two. +This guarantees that all three implement the same algorithm, and makes it +unlikely that any of them may contain serious undiscovered errors. In fact, +no errors have been discovered in lzip since 2009. + +Additionally, the three implementations have been extensively tested with +@uref{http://www.nongnu.org/lzip/manual/lziprecover_manual.html#Unzcrash,,unzcrash}, +valgrind, and @samp{american fuzzy lop} without finding a single +vulnerability or false negative. +@ifnothtml +@xref{Unzcrash,,,lziprecover}. +@end ifnothtml + +@item Dictionary size + +Lzip automatically adapts the dictionary size to the size of each file. +In addition to reducing the amount of memory required for decompression, +this feature also minimizes the probability of being affected by RAM errors +during compression. @c key4_mask + +@item Exit status + +Returning a warning status of 2 is a design flaw of compress that leaked +into the design of gzip. Both bzip2 and lzip are free from this flaw. + +@end table + + +@node Algorithm +@chapter Algorithm +@cindex algorithm + +In spite of its name (Lempel-Ziv-Markov chain-Algorithm), LZMA is not a +concrete algorithm; it is more like "any algorithm using the LZMA coding +scheme". LZMA compression consists in describing the uncompressed data as a +succession of coding sequences from the set shown in Section @samp{What is +coded} (@pxref{what-is-coded}), and then encoding them using a range +encoder. For example, the option @option{-0} of lzip uses the scheme in +almost the simplest way possible; issuing the longest match it can find, or +a literal byte if it can't find a match. Inversely, a more elaborate way of +finding coding sequences of minimum size than the one currently used by lzip +could be developed, and the resulting sequence could also be coded using the +LZMA coding scheme. + +Lzip currently implements two variants of the LZMA algorithm: fast +(used by option @option{-0}) and normal (used by all other compression levels). + +The high compression of LZMA comes from combining two basic, well-proven +compression ideas: sliding dictionaries (LZ77) and Markov models (the thing +used by every compression algorithm that uses a range encoder or similar +order-0 entropy coder as its last stage) with segregation of contexts +according to what the bits are used for. + +Lzip is a two stage compressor. The first stage is a Lempel-Ziv coder, +which reduces redundancy by translating chunks of data to their +corresponding distance-length pairs. The second stage is a range encoder +that uses a different probability model for each type of data: +distances, lengths, literal bytes, etc. + +Here is how it works, step by step: + +1) The member header is written to the output stream. + +2) The first byte is coded literally, because there are no previous +bytes to which the match finder can refer to. + +3) The main encoder advances to the next byte in the input data and +calls the match finder. + +4) The match finder fills an array with the minimum distances before the +current byte where a match of a given length can be found. + +5) Go back to step 3 until a sequence (formed of pairs, repeated +distances, and literal bytes) of minimum price has been formed. Where the +price represents the number of output bits produced. + +6) The range encoder encodes the sequence produced by the main encoder +and sends the bytes produced to the output stream. + +7) Go back to step 3 until the input data are finished or until the +member or volume size limits are reached. + +8) The range encoder is flushed. + +9) The member trailer is written to the output stream. + +10) If there are more data to compress, go back to step 1. + +@sp 1 +During compression, lzip reads data in large blocks (one dictionary size at +a time). Therefore it may block for up to tens of seconds any process +feeding data to it through a pipe. This is normal. The blocking intervals +get longer with higher compression levels because dictionary size increases +(and compression speed decreases) with compression level. + +@noindent +The ideas embodied in lzip are due to (at least) the following people: +Abraham Lempel and Jacob Ziv (for the LZ algorithm), Andrei Markov (for the +definition of Markov chains), G.N.N. Martin (for the definition of range +encoding), Igor Pavlov (for putting all the above together in LZMA), and +Julian Seward (for bzip2's CLI). + + @node Trailing data @chapter Extra data appended to the file @cindex trailing data @@ -1113,7 +1141,7 @@ example when writing to a tape. It is safe to append any amount of padding zero bytes to a lzip file. @item -Useful data added by the user; an "End Of File" string (to check that the +Useful data added by the user; an 'End Of File' string (to check that the file has not been truncated), a cryptographically secure hash, a description of file contents, etc. It is safe to append any amount of text to a lzip file as long as none of the first four bytes of the text matches the @@ -1165,9 +1193,8 @@ compression can only be detected by comparing the compressed file with the original because the corruption happens before lzip compresses the RAM contents, resulting in a valid compressed file containing wrong data. -@sp 1 @noindent -Example 1: Extract all the files from archive @samp{foo.tar.lz}. +Example 1: Extract all the files from archive @file{foo.tar.lz}. @example tar -xf foo.tar.lz @@ -1175,43 +1202,38 @@ or lzip -cd foo.tar.lz | tar -xf - @end example -@sp 1 @noindent -Example 2: Replace a regular file with its compressed version @samp{file.lz} +Example 2: Replace a regular file with its compressed version @file{file.lz} and show the compression ratio. @example lzip -v file @end example -@sp 1 @noindent -Example 3: Like example 2 but the created @samp{file.lz} is multimember with +Example 3: Like example 2 but the created @file{file.lz} is multimember with a member size of @w{1 MiB}. The compression ratio is not shown. @example lzip -b 1MiB file @end example -@sp 1 @noindent Example 4: Restore a regular file from its compressed version -@samp{file.lz}. If the operation is successful, @samp{file.lz} is removed. +@file{file.lz}. If the operation is successful, @file{file.lz} is removed. @example lzip -d file.lz @end example -@sp 1 @noindent -Example 5: Check the integrity of the compressed file @samp{file.lz} and +Example 5: Check the integrity of the compressed file @file{file.lz} and show status. @example lzip -tv file.lz @end example -@sp 1 @anchor{concat-example} @noindent Example 6: The right way of concatenating the decompressed output of two or @@ -1224,28 +1246,25 @@ Do this instead lzip -cd file1.lz file2.lz file3.lz @end example -@sp 1 @noindent -Example 7: Decompress @samp{file.lz} partially until @w{10 KiB} of +Example 7: Decompress @file{file.lz} partially until @w{10 KiB} of decompressed data are produced. @example lzip -cd file.lz | dd bs=1024 count=10 @end example -@sp 1 @noindent -Example 8: Decompress @samp{file.lz} partially from decompressed byte at +Example 8: Decompress @file{file.lz} partially from decompressed byte at offset 10000 to decompressed byte at offset 14999 (5000 bytes are produced). @example lzip -cd file.lz | dd bs=1000 skip=10 count=5 @end example -@sp 1 @noindent Example 9: Compress a whole device in /dev/sdc and send the output to -@samp{file.lz}. +@file{file.lz}. @example lzip -c /dev/sdc > file.lz @@ -1253,7 +1272,6 @@ or lzip /dev/sdc -o file.lz @end example -@sp 1 @noindent Example 10: Create a multivolume compressed tar archive with a volume size of @w{1440 KiB}. @@ -1262,7 +1280,6 @@ of @w{1440 KiB}. tar -c some_directory | lzip -S 1440KiB -o volume_name - @end example -@sp 1 @noindent Example 11: Extract a multivolume compressed tar archive. @@ -1270,7 +1287,6 @@ Example 11: Extract a multivolume compressed tar archive. lzip -cd volume_name*.lz | tar -xf - @end example -@sp 1 @noindent Example 12: Create a multivolume compressed backup of a large database file with a volume size of @w{650 MB}, where each volume is a multimember file @@ -1452,7 +1468,8 @@ public: Range_decoder() : member_pos( header_size ), code( 0 ), range( 0xFFFFFFFFU ) { - get_byte(); // discard first byte of the LZMA stream + if( get_byte() != 0 ) // check first LZMA byte + { std::fputs( "Nonzero first LZMA byte.\n", stderr ); std::exit( 2 ); } for( int i = 0; i < 4; ++i ) code = ( code << 8 ) | get_byte(); } @@ -1695,8 +1712,7 @@ bool LZ_decoder::decode_member() // Return false if error direct_bits ); else { - rep0 += - rdec.decode( direct_bits - dis_align_bits ) << dis_align_bits; + rep0 += rdec.decode( direct_bits-dis_align_bits ) << dis_align_bits; rep0 += rdec.decode_tree_reversed( bm_align, dis_align_bits ); if( rep0 == 0xFFFFFFFFU ) // marker found { @@ -1728,8 +1744,8 @@ int main( const int argc, const char * const argv[] ) "Lzd decompresses from standard input to standard output.\n" "\nCopyright (C) 2024 Antonio Diaz Diaz.\n" "License 2-clause BSD.\n" - "This is free software: you are free to change and redistribute it.\n" - "There is NO WARRANTY, to the extent permitted by law.\n" + "This is free software: you are free to change and redistribute " + "it.\nThere is NO WARRANTY, to the extent permitted by law.\n" "Report bugs to lzip-bug@nongnu.org\n" "Lzd home page: http://www.nongnu.org/lzip/lzd.html\n", PROGVERSION, argv[0] ); @@ -1741,6 +1757,7 @@ int main( const int argc, const char * const argv[] ) setmode( STDOUT_FILENO, O_BINARY ); #endif + bool empty = false, multi = false; for( bool first_member = true; ; first_member = false ) { Lzip_header header; // check header @@ -1755,12 +1772,12 @@ int main( const int argc, const char * const argv[] ) unsigned dict_size = 1 << ( header[5] & 0x1F ); dict_size -= ( dict_size / 16 ) * ( ( header[5] >> 5 ) & 7 ); if( dict_size < min_dictionary_size || dict_size > max_dictionary_size ) - { std::fputs( "Invalid dictionary size in member header.\n", stderr ); - return 2; } + { std::fputs( "Invalid dictionary size in member header.\n", + stderr ); return 2; } LZ_decoder decoder( dict_size ); // decode LZMA stream if( !decoder.decode_member() ) - { std::fputs( "Data error\n", stderr ); return 2; } + { std::fputs( "Data error.\n", stderr ); return 2; } Lzip_trailer trailer; // check trailer for( int i = 0; i < trailer_size; ++i ) trailer[i] = decoder.get_byte(); @@ -1768,25 +1785,28 @@ int main( const int argc, const char * const argv[] ) unsigned crc = 0; for( int i = 3; i >= 0; --i ) crc = ( crc << 8 ) + trailer[i]; if( crc != decoder.crc() ) - { std::fputs( "CRC mismatch\n", stderr ); retval = 2; } + { std::fputs( "CRC mismatch.\n", stderr ); retval = 2; } unsigned long long data_size = 0; for( int i = 11; i >= 4; --i ) data_size = ( data_size << 8 ) + trailer[i]; if( data_size != decoder.data_position() ) - { std::fputs( "Data size mismatch\n", stderr ); retval = 2; } + { std::fputs( "Data size mismatch.\n", stderr ); retval = 2; } + multi = !first_member; if( data_size == 0 ) empty = true; unsigned long long member_size = 0; for( int i = 19; i >= 12; --i ) member_size = ( member_size << 8 ) + trailer[i]; if( member_size != decoder.member_position() ) - { std::fputs( "Member size mismatch\n", stderr ); retval = 2; } + { std::fputs( "Member size mismatch.\n", stderr ); retval = 2; } if( retval ) return retval; } if( std::fclose( stdout ) != 0 ) { std::fprintf( stderr, "Error closing stdout: %s\n", std::strerror( errno ) ); return 1; } + if( empty && multi ) + { std::fputs( "Empty member not allowed.\n", stderr ); return 2; } return 0; } @end verbatim @@ -478,7 +478,7 @@ bool LZ_encoder::encode_member( const unsigned long long member_size ) { const unsigned long long member_size_limit = member_size - Lzip_trailer::size - max_marker_size; - const bool best = ( match_len_limit > 12 ); + const bool best = match_len_limit > 12; const int dis_price_count = best ? 1 : 512; const int align_price_count = best ? 1 : dis_align_size; const int price_count = ( match_len_limit > 36 ) ? 1013 : 4093; @@ -529,7 +529,7 @@ bool LZ_encoder::encode_member( const unsigned long long member_size ) const int len = trials[i].price; int dis = trials[i].dis4; - bool bit = ( dis < 0 ); + bool bit = dis < 0; renc.encode_bit( bm_match[state()][pos_state], !bit ); if( bit ) // literal byte { @@ -548,11 +548,11 @@ bool LZ_encoder::encode_member( const unsigned long long member_size ) { crc32.update_buf( crc_, ptr_to_current_pos() - ahead, len ); mtf_reps( dis, reps ); - bit = ( dis < num_rep_distances ); + bit = dis < num_rep_distances; renc.encode_bit( bm_rep[state()], bit ); if( bit ) // repeated match { - bit = ( dis == 0 ); + bit = dis == 0; renc.encode_bit( bm_rep0[state()], !bit ); if( bit ) renc.encode_bit( bm_len[state()][pos_state], len > 1 ); diff --git a/encoder_base.cc b/encoder_base.cc index 69abd52..04f78a0 100644 --- a/encoder_base.cc +++ b/encoder_base.cc @@ -147,7 +147,7 @@ void Range_encoder::flush_data() if( pos > 0 ) { if( outfd >= 0 && writeblock( outfd, buffer, pos ) != pos ) - throw Error( "Write error" ); + throw Error( write_error_msg ); partial_member_pos += pos; pos = 0; show_cprogress(); diff --git a/encoder_base.h b/encoder_base.h index ef48e0c..dbe05fa 100644 --- a/encoder_base.h +++ b/encoder_base.h @@ -238,7 +238,7 @@ class Range_encoder { if( low >> 24 != 0xFF ) { - const bool carry = ( low > 0xFFFFFFFFU ); + const bool carry = low > 0xFFFFFFFFU; put_byte( cache + carry ); for( ; ff_count > 0; --ff_count ) put_byte( 0xFF + carry ); cache = low >> 24; @@ -378,13 +378,13 @@ public: void encode_len( Len_model & lm, int symbol, const int pos_state ) { - bool bit = ( ( symbol -= min_match_len ) >= len_low_symbols ); + bool bit = ( symbol -= min_match_len ) >= len_low_symbols; encode_bit( lm.choice1, bit ); if( !bit ) encode_tree3( lm.bm_low[pos_state], symbol ); else { - bit = ( ( symbol -= len_low_symbols ) >= len_mid_symbols ); + bit = ( symbol -= len_low_symbols ) >= len_mid_symbols; encode_bit( lm.choice2, bit ); if( !bit ) encode_tree3( lm.bm_mid[pos_state], symbol ); @@ -17,6 +17,7 @@ #define _FILE_OFFSET_BITS 64 +#include <cerrno> #include <cstdio> #include <cstring> #include <string> @@ -57,7 +58,7 @@ int list_files( const std::vector< std::string > & filenames, for( unsigned i = 0; i < filenames.size(); ++i ) { - const bool from_stdin = ( filenames[i] == "-" ); + const bool from_stdin = filenames[i] == "-"; if( from_stdin ) { if( stdin_used ) continue; else stdin_used = true; } const char * const input_filename = from_stdin ? "(stdin)" : filenames[i].c_str(); @@ -74,6 +75,8 @@ int list_files( const std::vector< std::string > & filenames, set_retval( retval, lzip_index.retval() ); continue; } + const bool multi_empty = !from_stdin && lzip_index.multi_empty(); + if( multi_empty ) set_retval( retval, 2 ); if( verbosity < 0 ) continue; const unsigned long long udata_size = lzip_index.udata_size(); const unsigned long long cdata_size = lzip_index.cdata_size(); @@ -85,6 +88,8 @@ int list_files( const std::vector< std::string > & filenames, if( verbosity >= 1 ) std::fputs( " dict memb trail ", stdout ); std::fputs( " uncompressed compressed saved name\n", stdout ); } + if( multi_empty ) + { std::fflush( stdout ); show_file_error( input_filename, empty_msg ); } if( verbosity >= 1 ) std::printf( "%s %5ld %6lld ", format_ds( lzip_index.dictionary_size() ), members, lzip_index.file_size() - cdata_size ); @@ -103,12 +108,16 @@ int list_files( const std::vector< std::string > & filenames, first_post = true; // reprint heading after list of members } std::fflush( stdout ); + if( std::ferror( stdout ) ) break; } - if( verbosity >= 0 && files > 1 ) + if( verbosity >= 0 && files > 1 && !std::ferror( stdout ) ) { if( verbosity >= 1 ) std::fputs( " ", stdout ); list_line( total_uncomp, total_comp, "(totals)" ); std::fflush( stdout ); } + if( verbosity >= 0 && ( std::ferror( stdout ) || std::fclose( stdout ) != 0 ) ) + { show_file_error( "(stdout)", write_error_msg, errno ); + set_retval( retval, 1 ); } return retval; } @@ -313,14 +313,10 @@ struct Lzip_trailer struct Cl_options // command-line options { - bool ignore_empty; - bool ignore_marking; bool ignore_trailing; bool loose_trailing; - Cl_options() - : ignore_empty( true ), ignore_marking( true ), - ignore_trailing( true ), loose_trailing( false ) {} + Cl_options() : ignore_trailing( true ), loose_trailing( false ) {} }; @@ -337,8 +333,9 @@ const char * const bad_magic_msg = "Bad magic number (file not in lzip format)." const char * const bad_dict_msg = "Invalid dictionary size in member header."; const char * const corrupt_mm_msg = "Corrupt header in multimember file."; const char * const empty_msg = "Empty member not allowed."; -const char * const marking_msg = "Marking data not allowed."; +const char * const nonzero_msg = "Nonzero first LZMA byte."; const char * const trailing_msg = "Trailing data not allowed."; +const char * const write_error_msg = "Write error"; // defined in decoder.cc int readblock( const int fd, uint8_t * const buf, const int size ); diff --git a/lzip_index.cc b/lzip_index.cc index 6c1caf2..d9cfd0c 100644 --- a/lzip_index.cc +++ b/lzip_index.cc @@ -70,13 +70,10 @@ void Lzip_index::set_num_error( const char * const msg, unsigned long long num ) bool Lzip_index::read_header( const int fd, Lzip_header & header, - const long long pos, const bool ignore_marking ) + const long long pos ) { if( seek_read( fd, header.data, header.size, pos ) != header.size ) { set_errno_error( "Error reading member header: " ); return false; } - uint8_t byte; - if( !ignore_marking && readblock( fd, &byte, 1 ) == 1 && byte != 0 ) - { error_ = marking_msg; retval_ = 2; return false; } return true; } @@ -110,8 +107,7 @@ bool Lzip_index::skip_trailing_data( const int fd, unsigned long long & pos, { while( i > trailer.size && buffer[i-9] == 0 ) --i; continue; } if( member_size > ipos + i || !trailer.check_consistency() ) continue; Lzip_header header; - if( !read_header( fd, header, ipos + i - member_size, - cl_opts.ignore_marking ) ) return false; + if( !read_header( fd, header, ipos + i - member_size ) ) return false; if( !header.check() ) continue; const Lzip_header & header2 = *(const Lzip_header *)( buffer + i ); const bool full_h2 = bsize - i >= header.size; @@ -126,15 +122,12 @@ bool Lzip_index::skip_trailing_data( const int fd, unsigned long long & pos, { error_ = corrupt_mm_msg; retval_ = 2; return false; } if( !cl_opts.ignore_trailing ) { error_ = trailing_msg; retval_ = 2; return false; } - const unsigned long long data_size = trailer.data_size(); - if( !cl_opts.ignore_empty && data_size == 0 ) - { error_ = empty_msg; retval_ = 2; return false; } pos = ipos + i - member_size; // good member const unsigned dictionary_size = header.dictionary_size(); if( dictionary_size_ < dictionary_size ) dictionary_size_ = dictionary_size; - member_vector.push_back( Member( 0, data_size, pos, member_size, - dictionary_size ) ); + member_vector.push_back( Member( 0, trailer.data_size(), pos, + member_size, dictionary_size ) ); return true; } if( ipos == 0 ) @@ -154,16 +147,16 @@ Lzip_index::Lzip_index( const int infd, const Cl_options & cl_opts ) { if( insize < 0 ) { set_errno_error( "Input file is not seekable: " ); return; } + Lzip_header header; + if( insize >= header.size && + ( !read_header( infd, header, 0 ) || + !check_header( header ) ) ) return; if( insize < min_member_size ) - { error_ = "Input file is too short."; retval_ = 2; return; } + { error_ = "Input file is truncated."; retval_ = 2; return; } if( insize > INT64_MAX ) { error_ = "Input file is too long (2^63 bytes or more)."; retval_ = 2; return; } - Lzip_header header; - if( !read_header( infd, header, 0, cl_opts.ignore_marking ) || - !check_header( header ) ) return; - unsigned long long pos = insize; // always points to a header or to EOF while( pos >= min_member_size ) { @@ -178,23 +171,19 @@ Lzip_index::Lzip_index( const int infd, const Cl_options & cl_opts ) { if( skip_trailing_data( infd, pos, cl_opts ) ) continue; return; } set_num_error( "Bad trailer at pos ", pos - trailer.size ); break; } - if( !read_header( infd, header, pos - member_size, cl_opts.ignore_marking ) ) - break; + if( !read_header( infd, header, pos - member_size ) ) break; if( !header.check() ) // bad header { if( member_vector.empty() ) { if( skip_trailing_data( infd, pos, cl_opts ) ) continue; return; } set_num_error( "Bad header at pos ", pos - member_size ); break; } - const unsigned long long data_size = trailer.data_size(); - if( !cl_opts.ignore_empty && data_size == 0 ) - { error_ = empty_msg; retval_ = 2; break; } pos -= member_size; // good member const unsigned dictionary_size = header.dictionary_size(); if( dictionary_size_ < dictionary_size ) dictionary_size_ = dictionary_size; - member_vector.push_back( Member( 0, data_size, pos, member_size, - dictionary_size ) ); + member_vector.push_back( Member( 0, trailer.data_size(), pos, + member_size, dictionary_size ) ); } if( pos != 0 || member_vector.empty() || retval_ != 0 ) { diff --git a/lzip_index.h b/lzip_index.h index 928a7c7..8d2ed1a 100644 --- a/lzip_index.h +++ b/lzip_index.h @@ -59,8 +59,7 @@ class Lzip_index bool check_header( const Lzip_header & header ); void set_errno_error( const char * const msg ); void set_num_error( const char * const msg, unsigned long long num ); - bool read_header( const int fd, Lzip_header & header, const long long pos, - const bool ignore_marking ); + bool read_header( const int fd, Lzip_header & header, const long long pos ); bool skip_trailing_data( const int fd, unsigned long long & pos, const Cl_options & cl_opts ); @@ -72,6 +71,14 @@ public: int retval() const { return retval_; } unsigned dictionary_size() const { return dictionary_size_; } + bool multi_empty() const // multimember file with empty member(s) + { + if( member_vector.size() > 1 ) + for( unsigned long i = 0; i < member_vector.size(); ++i ) + if( member_vector[i].dblock.size() == 0 ) return true; + return false; + } + long long udata_size() const { if( member_vector.empty() ) return 0; return member_vector.back().dblock.end(); } @@ -113,22 +113,22 @@ bool delete_output_on_interrupt = false; void show_help() { std::printf( "Lzip is a lossless data compressor with a user interface similar to the one\n" - "of gzip or bzip2. Lzip uses a simplified form of the 'Lempel-Ziv-Markov\n" - "chain-Algorithm' (LZMA) stream format to maximize interoperability. The\n" - "maximum dictionary size is 512 MiB so that any lzip file can be decompressed\n" - "on 32-bit machines. Lzip provides accurate and robust 3-factor integrity\n" - "checking. Lzip can compress about as fast as gzip (lzip -0) or compress most\n" - "files more than bzip2 (lzip -9). Decompression speed is intermediate between\n" - "gzip and bzip2. Lzip is better than gzip and bzip2 from a data recovery\n" - "perspective. Lzip has been designed, written, and tested with great care to\n" - "replace gzip and bzip2 as the standard general-purpose compressed format for\n" - "Unix-like systems.\n" + "of gzip or bzip2. Lzip uses a simplified form of LZMA (Lempel-Ziv-Markov\n" + "chain-Algorithm) designed to achieve complete interoperability between\n" + "implementations. The maximum dictionary size is 512 MiB so that any lzip\n" + "file can be decompressed on 32-bit machines. Lzip provides accurate and\n" + "robust 3-factor integrity checking. 'lzip -0' compresses about as fast as\n" + "gzip, while 'lzip -9' compresses most files more than bzip2. Decompression\n" + "speed is intermediate between gzip and bzip2. Lzip provides better data\n" + "recovery capabilities than gzip and bzip2. Lzip has been designed, written,\n" + "and tested with great care to replace gzip and bzip2 as general-purpose\n" + "compressed format for Unix-like systems.\n" "\nUsage: %s [options] [files]\n", invocation_name ); std::printf( "\nOptions:\n" " -h, --help display this help and exit\n" " -V, --version output version information and exit\n" " -a, --trailing-error exit with error status if trailing data\n" - " -b, --member-size=<bytes> set member size limit in bytes\n" + " -b, --member-size=<bytes> set member size limit of multimember files\n" " -c, --stdout write to standard output, keep input files\n" " -d, --decompress decompress, test compressed file integrity\n" " -f, --force overwrite existing output files\n" @@ -145,8 +145,6 @@ void show_help() " -0 .. -9 set compression level [default 6]\n" " --fast alias for -0\n" " --best alias for -9\n" - " --empty-error exit with error status if empty member in file\n" - " --marking-error exit with error status if 1st LZMA byte not 0\n" " --loose-trailing allow trailing data seeming corrupt header\n" "\nIf no file names are given, or if a file is '-', lzip compresses or\n" "decompresses from standard input to standard output.\n" @@ -215,7 +213,7 @@ const char * format_ds( const unsigned dictionary_size ) const char * p = ""; const char * np = " "; unsigned num = dictionary_size; - bool exact = ( num % factor == 0 ); + bool exact = num % factor == 0; for( int i = 0; i < n && ( num > 9999 || ( exact && num >= factor ) ); ++i ) { num /= factor; if( num % factor != 0 ) exact = false; @@ -244,7 +242,7 @@ const char * format_num3( unsigned long long num ) char * const buf = buffer[current++]; current %= buffers; char * p = buf + bufsize - 1; // fill the buffer backwards *p = 0; // terminator - if( num > 1024 ) + if( num > 9999 ) { char prefix = 0; // try binary first, then si for( int i = 0; i < n && num != 0 && num % 1024 == 0; ++i ) @@ -290,7 +288,7 @@ unsigned long long getnum( const char * const arg, if( !errno && tail[0] ) { - const unsigned factor = ( tail[1] == 'i' ) ? 1024 : 1000; + const unsigned factor = (tail[1] == 'i') ? 1024 : 1000; int exponent = 0; // 0 = bad multiplier switch( tail[0] ) { @@ -406,9 +404,9 @@ int open_instream( const char * const name, struct stat * const in_statsp, { const int i = fstat( infd, in_statsp ); const mode_t mode = in_statsp->st_mode; - const bool can_read = ( i == 0 && !reg_only && - ( S_ISBLK( mode ) || S_ISCHR( mode ) || - S_ISFIFO( mode ) || S_ISSOCK( mode ) ) ); + const bool can_read = i == 0 && !reg_only && + ( S_ISBLK( mode ) || S_ISCHR( mode ) || + S_ISFIFO( mode ) || S_ISSOCK( mode ) ); if( i != 0 || ( !S_ISREG( mode ) && ( !can_read || one_to_one ) ) ) { if( verbosity >= 0 ) @@ -431,7 +429,7 @@ int open_instream2( const char * const name, struct stat * const in_statsp, if( program_mode == m_compress && !recompress && eindex >= 0 ) { if( verbosity >= 0 ) - std::fprintf( stderr, "%s: %s: Input file already has '%s' suffix.\n", + std::fprintf( stderr, "%s: %s: Input file already has '%s' suffix, ignored.\n", program_name, name, known_extensions[eindex].from ); return -1; } @@ -475,8 +473,8 @@ bool open_outstream( const bool force, const bool protect ) if( force ) flags |= O_TRUNC; else flags |= O_EXCL; outfd = -1; - if( output_filename.size() && - output_filename[output_filename.size()-1] == '/' ) errno = EISDIR; + if( output_filename.size() && output_filename.end()[-1] == '/' ) + errno = EISDIR; else { if( !protect && !make_dirs( output_filename ) ) { show_file_error( output_filename.c_str(), @@ -600,7 +598,6 @@ int compress( const unsigned long long cfile_size, const Lzma_options & encoder_options, const Pretty_print & pp, const struct stat * const in_statsp, const bool zero ) { - int retval = 0; LZ_encoder_base * encoder = 0; // polymorphic encoder if( verbosity >= 1 ) pp(); @@ -618,9 +615,10 @@ int compress( const unsigned long long cfile_size, } unsigned long long in_size = 0, out_size = 0, partial_volume_size = 0; + int retval = 0; while( true ) // encode one member per iteration { - const unsigned long long size = ( volume_size > 0 ) ? + const unsigned long long size = (volume_size > 0) ? std::min( member_size, volume_size - partial_volume_size ) : member_size; show_cprogress( cfile_size, in_size, encoder, &pp ); // init if( !encoder->encode_member( size ) ) @@ -699,11 +697,12 @@ bool show_trailing_data( const uint8_t * const data, const int size, int decompress( const unsigned long long cfile_size, const int infd, const Cl_options & cl_opts, const Pretty_print & pp, - const bool testing ) + const bool from_stdin, const bool testing ) { unsigned long long partial_file_pos = 0; Range_decoder rdec( infd ); int retval = 0; + bool empty = false, multi = false; for( bool first_member = true; ; first_member = false ) { @@ -743,7 +742,7 @@ int decompress( const unsigned long long cfile_size, const int infd, LZ_decoder decoder( rdec, dictionary_size, outfd ); show_dprogress( cfile_size, partial_file_pos, &rdec, &pp ); // init - const int result = decoder.decode_member( cl_opts, pp ); + const int result = decoder.decode_member( pp ); partial_file_pos += rdec.member_position(); if( result != 0 ) { @@ -754,15 +753,18 @@ int decompress( const unsigned long long cfile_size, const int infd, "File ends unexpectedly" : "Decoder error", partial_file_pos ); } - else if( result == 5 ) pp( empty_msg ); - else if( result == 6 ) pp( marking_msg ); + else if( result == 5 ) pp( nonzero_msg ); retval = 2; break; } + if( !from_stdin ) { multi = !first_member; + if( decoder.data_position() == 0 ) empty = true; } if( verbosity >= 2 ) { std::fputs( testing ? "ok\n" : "done\n", stderr ); pp.reset(); } } if( verbosity == 1 && retval == 0 ) std::fputs( testing ? "ok\n" : "done\n", stderr ); + if( empty && multi && retval == 0 ) + { show_file_error( pp.name(), empty_msg ); retval = 2; } return retval; } @@ -891,41 +893,39 @@ int main( const int argc, const char * const argv[] ) bool zero = false; if( argc > 0 ) invocation_name = argv[0]; - enum { opt_eer = 256, opt_lt, opt_mer }; + enum { opt_lt = 256 }; const Arg_parser::Option options[] = { - { '0', "fast", Arg_parser::no }, - { '1', 0, Arg_parser::no }, - { '2', 0, Arg_parser::no }, - { '3', 0, Arg_parser::no }, - { '4', 0, Arg_parser::no }, - { '5', 0, Arg_parser::no }, - { '6', 0, Arg_parser::no }, - { '7', 0, Arg_parser::no }, - { '8', 0, Arg_parser::no }, - { '9', "best", Arg_parser::no }, - { 'a', "trailing-error", Arg_parser::no }, - { 'b', "member-size", Arg_parser::yes }, - { 'c', "stdout", Arg_parser::no }, - { 'd', "decompress", Arg_parser::no }, - { 'f', "force", Arg_parser::no }, - { 'F', "recompress", Arg_parser::no }, - { 'h', "help", Arg_parser::no }, - { 'k', "keep", Arg_parser::no }, - { 'l', "list", Arg_parser::no }, - { 'm', "match-length", Arg_parser::yes }, - { 'n', "threads", Arg_parser::yes }, - { 'o', "output", Arg_parser::yes }, - { 'q', "quiet", Arg_parser::no }, - { 's', "dictionary-size", Arg_parser::yes }, - { 'S', "volume-size", Arg_parser::yes }, - { 't', "test", Arg_parser::no }, - { 'v', "verbose", Arg_parser::no }, - { 'V', "version", Arg_parser::no }, - { opt_eer, "empty-error", Arg_parser::no }, - { opt_lt, "loose-trailing", Arg_parser::no }, - { opt_mer, "marking-error", Arg_parser::no }, - { 0, 0, Arg_parser::no } }; + { '0', "fast", Arg_parser::no }, + { '1', 0, Arg_parser::no }, + { '2', 0, Arg_parser::no }, + { '3', 0, Arg_parser::no }, + { '4', 0, Arg_parser::no }, + { '5', 0, Arg_parser::no }, + { '6', 0, Arg_parser::no }, + { '7', 0, Arg_parser::no }, + { '8', 0, Arg_parser::no }, + { '9', "best", Arg_parser::no }, + { 'a', "trailing-error", Arg_parser::no }, + { 'b', "member-size", Arg_parser::yes }, + { 'c', "stdout", Arg_parser::no }, + { 'd', "decompress", Arg_parser::no }, + { 'f', "force", Arg_parser::no }, + { 'F', "recompress", Arg_parser::no }, + { 'h', "help", Arg_parser::no }, + { 'k', "keep", Arg_parser::no }, + { 'l', "list", Arg_parser::no }, + { 'm', "match-length", Arg_parser::yes }, + { 'n', "threads", Arg_parser::yes }, + { 'o', "output", Arg_parser::yes }, + { 'q', "quiet", Arg_parser::no }, + { 's', "dictionary-size", Arg_parser::yes }, + { 'S', "volume-size", Arg_parser::yes }, + { 't', "test", Arg_parser::no }, + { 'v', "verbose", Arg_parser::no }, + { 'V', "version", Arg_parser::no }, + { opt_lt, "loose-trailing", Arg_parser::no }, + { 0, 0, Arg_parser::no } }; const Arg_parser parser( argc, argv, options ); if( parser.error().size() ) // bad option @@ -941,9 +941,8 @@ int main( const int argc, const char * const argv[] ) const char * const arg = sarg.c_str(); switch( code ) { - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - zero = ( code == '0' ); + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': zero = code == '0'; encoder_options = option_mapping[code-'0']; break; case 'a': cl_opts.ignore_trailing = false; break; case 'b': member_size = getnum( arg, pn, 100000, max_member_size ); break; @@ -957,7 +956,7 @@ int main( const int argc, const char * const argv[] ) case 'm': encoder_options.match_len_limit = getnum( arg, pn, min_match_len_limit, max_match_len ); zero = false; break; - case 'n': break; + case 'n': break; // ignored case 'o': if( sarg == "-" ) to_stdout = true; else { default_output_filename = sarg; } break; case 'q': verbosity = -1; break; @@ -967,9 +966,7 @@ int main( const int argc, const char * const argv[] ) case 't': set_mode( program_mode, m_test ); break; case 'v': if( verbosity < 4 ) ++verbosity; break; case 'V': show_version(); return 0; - case opt_eer: cl_opts.ignore_empty = false; break; - case opt_lt: cl_opts.loose_trailing = true; break; - case opt_mer: cl_opts.ignore_marking = false; break; + case opt_lt: cl_opts.loose_trailing = true; break; default: internal_error( "uncaught option." ); } } // end process options @@ -1023,9 +1020,10 @@ int main( const int argc, const char * const argv[] ) { std::string input_filename; int infd; + const bool from_stdin = filenames[i] == "-"; pp.set_name( filenames[i] ); - if( filenames[i] == "-" ) + if( from_stdin ) { if( stdin_used ) continue; else stdin_used = true; infd = STDIN_FILENO; @@ -1072,7 +1070,8 @@ int main( const int argc, const char * const argv[] ) tmp = compress( cfile_size, member_size, volume_size, infd, encoder_options, pp, in_statsp, zero ); else - tmp = decompress( cfile_size, infd, cl_opts, pp, program_mode == m_test ); + tmp = decompress( cfile_size, infd, cl_opts, pp, from_stdin, + program_mode == m_test ); } catch( std::bad_alloc & ) { pp( ( program_mode == m_compress ) ? diff --git a/testsuite/check.sh b/testsuite/check.sh index dd9a60a..e42a6d8 100755 --- a/testsuite/check.sh +++ b/testsuite/check.sh @@ -28,12 +28,10 @@ if [ -d tmp ] ; then rm -rf tmp ; fi mkdir tmp cd "${objdir}"/tmp || framework_failure -cat "${testdir}"/test.txt > in || framework_failure +cp "${testdir}"/test.txt in || framework_failure in_lz="${testdir}"/test.txt.lz -in_em="${testdir}"/test_em.txt.lz fox_lz="${testdir}"/fox.lz -fox6_lz="${testdir}"/fox6.lz -f6mk_lz="${testdir}"/fox6_mark.lz +fnz_lz="${testdir}"/fox_nz.lz fail=0 test_failed() { fail=1 ; printf " $1" ; [ -z "$2" ] || printf "($2)" ; } @@ -70,7 +68,7 @@ done "${LZIP}" -q -o out.lz nx_file [ $? = 1 ] || test_failed $LINENO [ ! -e out.lz ] || test_failed $LINENO -"${LZIP}" -qf -S100k -o out in in +"${LZIP}" -qf -S100k -o out in in # only one file with -o and -S [ $? = 1 ] || test_failed $LINENO { [ ! -e out ] && [ ! -e out.lz ] ; } || test_failed $LINENO # these are for code coverage @@ -106,37 +104,25 @@ printf "LZIP\001+.............................." | "${LZIP}" -t 2> /dev/null printf "\ntesting decompression..." -for i in "${in_lz}" "${in_em}" ; do - "${LZIP}" -lq "$i" || test_failed $LINENO "$i" - "${LZIP}" -t "$i" || test_failed $LINENO "$i" - "${LZIP}" -d "$i" -o out || test_failed $LINENO "$i" - cmp in out || test_failed $LINENO "$i" - "${LZIP}" -cd "$i" > out || test_failed $LINENO "$i" - cmp in out || test_failed $LINENO "$i" - "${LZIP}" -d "$i" -o - > out || test_failed $LINENO "$i" - cmp in out || test_failed $LINENO "$i" - "${LZIP}" -d < "$i" > out || test_failed $LINENO "$i" - cmp in out || test_failed $LINENO "$i" - rm -f out || framework_failure -done - -lines=`"${LZIP}" -tvv "${in_em}" 2>&1 | wc -l` || test_failed $LINENO -[ "${lines}" -eq 8 ] || test_failed $LINENO "${lines}" -"${LZIP}" -tq "${in_em}" --empty-error -[ $? = 2 ] || test_failed $LINENO - -lines=`"${LZIP}" -lvv "${in_em}" | wc -l` || test_failed $LINENO -[ "${lines}" -eq 11 ] || test_failed $LINENO "${lines}" -"${LZIP}" -lq "${in_em}" --empty-error -[ $? = 2 ] || test_failed $LINENO +"${LZIP}" -l "${in_lz}" > /dev/null || test_failed $LINENO +"${LZIP}" -t "${in_lz}" || test_failed $LINENO +"${LZIP}" -d "${in_lz}" -o out || test_failed $LINENO +cmp in out || test_failed $LINENO +"${LZIP}" -cd "${in_lz}" > out || test_failed $LINENO +cmp in out || test_failed $LINENO +"${LZIP}" -d "${in_lz}" -o - > out || test_failed $LINENO +cmp in out || test_failed $LINENO +"${LZIP}" -d < "${in_lz}" > out || test_failed $LINENO +cmp in out || test_failed $LINENO +rm -f out || framework_failure -cat "${in_lz}" > out.lz || framework_failure +cp "${in_lz}" out.lz || framework_failure "${LZIP}" -dk out.lz || test_failed $LINENO cmp in out || test_failed $LINENO rm -f out || framework_failure "${LZIP}" -cd "${fox_lz}" > fox || test_failed $LINENO -cat fox > copy || framework_failure -cat "${in_lz}" > copy.lz || framework_failure +cp fox copy || framework_failure +cp "${in_lz}" copy.lz || framework_failure "${LZIP}" -d copy.lz out.lz 2> /dev/null # skip copy, decompress out [ $? = 1 ] || test_failed $LINENO [ ! -e out.lz ] || test_failed $LINENO @@ -147,7 +133,7 @@ cmp in out || test_failed $LINENO cmp in copy || test_failed $LINENO rm -f copy out || framework_failure -cat "${in_lz}" > out.lz || framework_failure +cp "${in_lz}" out.lz || framework_failure "${LZIP}" -d -S100k out.lz || test_failed $LINENO # ignore -S [ ! -e out.lz ] || test_failed $LINENO cmp in out || test_failed $LINENO @@ -155,7 +141,6 @@ cmp in out || test_failed $LINENO printf "to be overwritten" > out || framework_failure "${LZIP}" -df -o out < "${in_lz}" || test_failed $LINENO cmp in out || test_failed $LINENO -rm -f out || framework_failure "${LZIP}" -d -o ./- "${in_lz}" || test_failed $LINENO cmp in ./- || test_failed $LINENO rm -f ./- || framework_failure @@ -163,12 +148,12 @@ rm -f ./- || framework_failure cmp in ./- || test_failed $LINENO rm -f ./- || framework_failure -cat "${in_lz}" > anyothername || framework_failure +cp "${in_lz}" anyothername || framework_failure "${LZIP}" -dv - anyothername - < "${in_lz}" > out 2> /dev/null || test_failed $LINENO cmp in out || test_failed $LINENO cmp in anyothername.out || test_failed $LINENO -rm -f out anyothername.out || framework_failure +rm -f anyothername.out || framework_failure "${LZIP}" -lq in "${in_lz}" [ $? = 2 ] || test_failed $LINENO @@ -185,7 +170,7 @@ cat out in | cmp in - || test_failed $LINENO # out must be empty [ $? = 1 ] || test_failed $LINENO cmp in out || test_failed $LINENO rm -f out || framework_failure -cat "${in_lz}" > out.lz || framework_failure +cp "${in_lz}" out.lz || framework_failure for i in 1 2 3 4 5 6 7 ; do printf "g" >> out.lz || framework_failure "${LZIP}" -alvv out.lz "${in_lz}" > /dev/null 2>&1 @@ -206,7 +191,7 @@ cmp in out || test_failed $LINENO rm -f out || framework_failure cat in in > in2 || framework_failure -"${LZIP}" -lq "${in_lz}" "${in_lz}" || test_failed $LINENO +"${LZIP}" -l "${in_lz}" "${in_lz}" > /dev/null || test_failed $LINENO "${LZIP}" -t "${in_lz}" "${in_lz}" || test_failed $LINENO "${LZIP}" -cd "${in_lz}" "${in_lz}" -o out > out2 || test_failed $LINENO [ ! -e out ] || test_failed $LINENO # override -o @@ -217,6 +202,11 @@ cmp in2 out2 || test_failed $LINENO rm -f out2 || framework_failure cat "${in_lz}" "${in_lz}" > out2.lz || framework_failure +lines=`"${LZIP}" -tvv out2.lz 2>&1 | wc -l` || test_failed $LINENO +[ "${lines}" -eq 2 ] || test_failed $LINENO "${lines}" +lines=`"${LZIP}" -lvv out2.lz | wc -l` || test_failed $LINENO +[ "${lines}" -eq 5 ] || test_failed $LINENO "${lines}" + printf "\ngarbage" >> out2.lz || framework_failure "${LZIP}" -tvvvv out2.lz 2> /dev/null || test_failed $LINENO "${LZIP}" -alq out2.lz @@ -236,15 +226,6 @@ printf "to be overwritten" > out2 || framework_failure cmp in2 out2 || test_failed $LINENO rm -f out2 || framework_failure -"${LZIP}" -cd "${fox6_lz}" > out || test_failed $LINENO -"${LZIP}" -cd "${f6mk_lz}" > copy || test_failed $LINENO -cmp copy out || test_failed $LINENO -rm -f copy out || framework_failure -"${LZIP}" -lq "${f6mk_lz}" --marking-error -[ $? = 2 ] || test_failed $LINENO -"${LZIP}" -tq "${f6mk_lz}" --marking-error -[ $? = 2 ] || test_failed $LINENO - "${LZIP}" -d "${fox_lz}" -o a/b/c/fox || test_failed $LINENO cmp fox a/b/c/fox || test_failed $LINENO rm -rf a || framework_failure @@ -255,6 +236,21 @@ rm -rf a || framework_failure [ $? = 1 ] || test_failed $LINENO [ ! -e a ] || test_failed $LINENO +touch empty em || framework_failure +"${LZIP}" -0 em || test_failed $LINENO +"${LZIP}" -l em.lz > /dev/null || test_failed $LINENO +"${LZIP}" -dk em.lz || test_failed $LINENO +cmp empty em || test_failed $LINENO +cat em.lz em.lz | "${LZIP}" -t || test_failed $LINENO +cat em.lz em.lz | "${LZIP}" -d > em || test_failed $LINENO +cmp empty em || test_failed $LINENO +cat em.lz "${in_lz}" | "${LZIP}" -t || test_failed $LINENO +cat em.lz "${in_lz}" | "${LZIP}" -d > out || test_failed $LINENO +cmp in out || test_failed $LINENO +cat "${in_lz}" em.lz | "${LZIP}" -t || test_failed $LINENO +cat "${in_lz}" em.lz | "${LZIP}" -d > out || test_failed $LINENO +cmp in out || test_failed $LINENO + printf "\ntesting compression..." "${LZIP}" -c -0 in in in -S100k -o out3.lz > copy2.lz || test_failed $LINENO @@ -263,7 +259,7 @@ printf "\ntesting compression..." "${LZIP}" -d copy2.lz -o out2 || test_failed $LINENO [ -e copy2.lz ] || test_failed $LINENO cmp in2 out2 || test_failed $LINENO -rm -f in2 out2 copy2.lz || framework_failure +rm -f copy2.lz || framework_failure "${LZIP}" -cf "${in_lz}" > lzlz 2> /dev/null # /dev/null is a tty on OS/2 [ $? = 1 ] || test_failed $LINENO @@ -331,7 +327,7 @@ rm -f in8 || framework_failure "${LZIP}" -t out00001.lz out00002.lz || test_failed $LINENO "${LZIP}" -cd out00001.lz out00002.lz | cmp in8.lz - || test_failed $LINENO [ ! -e out00003.lz ] || test_failed $LINENO -rm -f out00001.lz || framework_failure +rm -f out00001.lz out00002.lz || framework_failure "${LZIP}" -1 -S100k -o a/b/c/out < in8.lz || test_failed $LINENO "${LZIP}" -t a/b/c/out00001.lz a/b/c/out00002.lz || test_failed $LINENO "${LZIP}" -cd a/b/c/out00001.lz a/b/c/out00002.lz | cmp in8.lz - || @@ -357,11 +353,44 @@ rm -rf a || framework_failure printf "\ntesting bad input..." +cat em.lz em.lz > ee.lz || framework_failure +"${LZIP}" -l < ee.lz > /dev/null || test_failed $LINENO +"${LZIP}" -t < ee.lz || test_failed $LINENO +"${LZIP}" -d < ee.lz > em || test_failed $LINENO +cmp empty em || test_failed $LINENO +"${LZIP}" -lq ee.lz +[ $? = 2 ] || test_failed $LINENO +"${LZIP}" -tq ee.lz +[ $? = 2 ] || test_failed $LINENO +"${LZIP}" -dq ee.lz +[ $? = 2 ] || test_failed $LINENO +[ ! -e ee ] || test_failed $LINENO +"${LZIP}" -cdq ee.lz > em +[ $? = 2 ] || test_failed $LINENO +cmp empty em || test_failed $LINENO +rm -f empty em || framework_failure +cat "${in_lz}" em.lz "${in_lz}" > inein.lz || framework_failure +"${LZIP}" -l < inein.lz > /dev/null || test_failed $LINENO +"${LZIP}" -t < inein.lz || test_failed $LINENO +"${LZIP}" -d < inein.lz > out2 || test_failed $LINENO +cmp in2 out2 || test_failed $LINENO +"${LZIP}" -lq inein.lz +[ $? = 2 ] || test_failed $LINENO +"${LZIP}" -tq inein.lz +[ $? = 2 ] || test_failed $LINENO +"${LZIP}" -dq inein.lz +[ $? = 2 ] || test_failed $LINENO +[ ! -e inein ] || test_failed $LINENO +"${LZIP}" -cdq inein.lz > out2 +[ $? = 2 ] || test_failed $LINENO +cmp in2 out2 || test_failed $LINENO +rm -f in2 out2 inein.lz em.lz || framework_failure + headers='LZIp LZiP LZip LzIP LzIp LziP lZIP lZIp lZiP lzIP' -body='\001\014\000\203\377\373\377\377\300\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000$\000\000\000\000\000\000\000' -cat "${in_lz}" > int.lz || framework_failure +body='\001\014\000\000\101\376\367\377\377\340\000\200\000\215\357\002\322\001\000\000\000\000\000\000\000\045\000\000\000\000\000\000\000' +cp "${in_lz}" int.lz || framework_failure printf "LZIP${body}" >> int.lz || framework_failure -if "${LZIP}" -tq int.lz ; then +if "${LZIP}" -t int.lz ; then for header in ${headers} ; do printf "${header}${body}" > int.lz || framework_failure "${LZIP}" -lq int.lz # first member @@ -380,7 +409,7 @@ if "${LZIP}" -tq int.lz ; then [ $? = 2 ] || test_failed $LINENO ${header} "${LZIP}" -cdq --loose-trailing int.lz > /dev/null [ $? = 2 ] || test_failed $LINENO ${header} - cat "${in_lz}" > int.lz || framework_failure + cp "${in_lz}" int.lz || framework_failure printf "${header}${body}" >> int.lz || framework_failure "${LZIP}" -lq int.lz # trailing data [ $? = 2 ] || test_failed $LINENO ${header} @@ -390,7 +419,7 @@ if "${LZIP}" -tq int.lz ; then [ $? = 2 ] || test_failed $LINENO ${header} "${LZIP}" -cdq int.lz > /dev/null [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -lq --loose-trailing int.lz || + "${LZIP}" -l --loose-trailing int.lz > /dev/null || test_failed $LINENO ${header} "${LZIP}" -t --loose-trailing int.lz || test_failed $LINENO ${header} @@ -408,10 +437,14 @@ if "${LZIP}" -tq int.lz ; then [ $? = 2 ] || test_failed $LINENO ${header} done else - printf "\nwarning: skipping header test: 'printf' does not work on your system." + printf "warning: skipping header test: 'printf' does not work on your system." fi rm -f int.lz || framework_failure +"${LZIP}" -l "${fnz_lz}" > /dev/null || test_failed $LINENO +"${LZIP}" -tq "${fnz_lz}" +[ $? = 2 ] || test_failed $LINENO + for i in fox_v2.lz fox_s11.lz fox_de20.lz \ fox_bcrc.lz fox_crc0.lz fox_das46.lz fox_mes81.lz ; do "${LZIP}" -tq "${testdir}"/$i @@ -423,13 +456,13 @@ for i in fox_bcrc.lz fox_crc0.lz fox_das46.lz fox_mes81.lz ; do [ $? = 2 ] || test_failed $LINENO $i cmp fox out || test_failed $LINENO $i done -rm -f fox out || framework_failure +rm -f fox || framework_failure cat "${in_lz}" "${in_lz}" > in2.lz || framework_failure cat "${in_lz}" "${in_lz}" "${in_lz}" > in3.lz || framework_failure -if dd if=in3.lz of=trunc.lz bs=14752 count=1 2> /dev/null && - [ -e trunc.lz ] && cmp in2.lz trunc.lz > /dev/null 2>&1 ; then - for i in 6 20 14734 14753 14754 14755 14756 14757 14758 ; do +if dd if=in3.lz of=trunc.lz bs=14682 count=1 2> /dev/null && + [ -e trunc.lz ] && cmp in2.lz trunc.lz ; then + for i in 6 20 14664 14683 14684 14685 14686 14687 14688 ; do dd if=in3.lz of=trunc.lz bs=$i count=1 2> /dev/null "${LZIP}" -lq trunc.lz [ $? = 2 ] || test_failed $LINENO $i @@ -443,11 +476,11 @@ if dd if=in3.lz of=trunc.lz bs=14752 count=1 2> /dev/null && [ $? = 2 ] || test_failed $LINENO $i done else - printf "\nwarning: skipping truncation test: 'dd' does not work on your system." + printf "warning: skipping truncation test: 'dd' does not work on your system." fi rm -f in2.lz in3.lz trunc.lz || framework_failure -cat "${in_lz}" > ingin.lz || framework_failure +cp "${in_lz}" ingin.lz || framework_failure printf "g" >> ingin.lz || framework_failure cat "${in_lz}" >> ingin.lz || framework_failure "${LZIP}" -lq ingin.lz @@ -456,17 +489,21 @@ cat "${in_lz}" >> ingin.lz || framework_failure [ $? = 2 ] || test_failed $LINENO "${LZIP}" -atq < ingin.lz [ $? = 2 ] || test_failed $LINENO -"${LZIP}" -acdq ingin.lz > /dev/null +"${LZIP}" -acdq ingin.lz > out [ $? = 2 ] || test_failed $LINENO -"${LZIP}" -adq < ingin.lz > /dev/null +cmp in out || test_failed $LINENO +"${LZIP}" -adq < ingin.lz > out [ $? = 2 ] || test_failed $LINENO +cmp in out || test_failed $LINENO "${LZIP}" -t ingin.lz || test_failed $LINENO "${LZIP}" -t < ingin.lz || test_failed $LINENO +"${LZIP}" -dk ingin.lz || test_failed $LINENO +cmp in ingin || test_failed $LINENO "${LZIP}" -cd ingin.lz > out || test_failed $LINENO cmp in out || test_failed $LINENO "${LZIP}" -d < ingin.lz > out || test_failed $LINENO cmp in out || test_failed $LINENO -rm -f out ingin.lz || framework_failure +rm -f out ingin ingin.lz || framework_failure echo if [ ${fail} = 0 ] ; then diff --git a/testsuite/fox6.lz b/testsuite/fox6.lz Binary files differdeleted file mode 100644 index 8401b99..0000000 --- a/testsuite/fox6.lz +++ /dev/null diff --git a/testsuite/fox6_mark.lz b/testsuite/fox6_mark.lz Binary files differdeleted file mode 100644 index 32b2ac0..0000000 --- a/testsuite/fox6_mark.lz +++ /dev/null diff --git a/testsuite/fox_nz.lz b/testsuite/fox_nz.lz Binary files differnew file mode 100644 index 0000000..44a4b58 --- /dev/null +++ b/testsuite/fox_nz.lz diff --git a/testsuite/test.txt b/testsuite/test.txt index 9196a3a..423f0c0 100644 --- a/testsuite/test.txt +++ b/testsuite/test.txt @@ -1,8 +1,7 @@ GNU GENERAL PUBLIC LICENSE Version 2, June 1991 - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Copyright (C) 1989, 1991 Free Software Foundation, Inc. <http://fsf.org/> Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. @@ -339,8 +338,7 @@ Public License instead of this License. GNU GENERAL PUBLIC LICENSE
Version 2, June 1991
- Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
- 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc. <http://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
diff --git a/testsuite/test.txt.lz b/testsuite/test.txt.lz Binary files differindex 22cea6e..5dc169f 100644 --- a/testsuite/test.txt.lz +++ b/testsuite/test.txt.lz diff --git a/testsuite/test_em.txt.lz b/testsuite/test_em.txt.lz Binary files differdeleted file mode 100644 index 7e96250..0000000 --- a/testsuite/test_em.txt.lz +++ /dev/null |