diff options
| -rw-r--r-- | ChangeLog | 45 | ||||
| -rw-r--r-- | INSTALL | 2 | ||||
| -rw-r--r-- | NEWS | 46 | ||||
| -rw-r--r-- | README | 4 | ||||
| -rw-r--r-- | arg_parser.cc | 2 | ||||
| -rw-r--r-- | arg_parser.h | 2 | ||||
| -rwxr-xr-x | configure | 6 | ||||
| -rw-r--r-- | decoder.cc | 67 | ||||
| -rw-r--r-- | decoder.h | 11 | ||||
| -rw-r--r-- | doc/lzip.1 | 9 | ||||
| -rw-r--r-- | doc/lzip.info | 287 | ||||
| -rw-r--r-- | doc/lzip.texi | 235 | ||||
| -rw-r--r-- | encoder.cc | 2 | ||||
| -rw-r--r-- | encoder.h | 6 | ||||
| -rw-r--r-- | encoder_base.cc | 30 | ||||
| -rw-r--r-- | encoder_base.h | 22 | ||||
| -rw-r--r-- | fast_encoder.cc | 2 | ||||
| -rw-r--r-- | fast_encoder.h | 6 | ||||
| -rw-r--r-- | file_index.cc | 37 | ||||
| -rw-r--r-- | file_index.h | 8 | ||||
| -rw-r--r-- | list.cc | 8 | ||||
| -rw-r--r-- | lzip.h | 46 | ||||
| -rw-r--r-- | main.cc | 288 | ||||
| -rwxr-xr-x | testsuite/check.sh | 70 |
24 files changed, 795 insertions, 446 deletions
@@ -1,3 +1,20 @@ +2018-02-11 Antonio Diaz Diaz <antonio@gnu.org> + + * Version 1.20 released. + * main.cc: Added new option '--loose-trailing'. + * Improved corrupt header detection to HD=3. + * main.cc: Show corrupt or truncated header in multimember file. + * main.cc (main): Option '-S, --volume-size' now keeps input files. + * encoder_base.*: Adjust dictionary size for each member. + * Replaced 'bits/byte' with inverse compression ratio in output. + * Show progress of decompression at verbosity level 2 (-vv). + * Show progress of (de)compression only if stderr is a terminal. + * main.cc: Show final diagnostic when testing multiple files. + * main.cc: Do not add a second .lz extension to the arg of -o. + * decoder.cc (verify_trailer): Show stored sizes also in hex. + Show dictionary size at verbosity level 4 (-vvvv). + * lzip.texi: Added chapter 'Output'. + 2017-04-13 Antonio Diaz Diaz <antonio@gnu.org> * Version 1.19 released. @@ -23,8 +40,8 @@ * lzip.texi: Added chapter 'Trailing data'. * configure: Avoid warning on some shells when testing for g++. * Makefile.in: Detect the existence of install-info. - * testsuite/check.sh: A POSIX shell is required to run the tests. - * testsuite/check.sh: Don't check error messages. + * check.sh: A POSIX shell is required to run the tests. + * check.sh: Don't check error messages. 2015-07-12 Antonio Diaz Diaz <antonio@gnu.org> @@ -76,7 +93,7 @@ * Small change in '--help' output and man page. * Changed quote characters in messages as advised by GNU Standards. * configure: 'datadir' renamed to 'datarootdir'. - * 'testsuite/unzcrash.cc' has been moved to package 'lziprecover'. + * 'unzcrash.cc' has been moved to package 'lziprecover'. 2011-04-30 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -97,10 +114,10 @@ * main.cc (open_instream): Don't show the message " and '--stdout' was not specified" for directories, etc. * lziprecover.cc: If '-v' is not specified show errors only. - * testsuite/unzcrash.cc: Use Arg_parser. - * testsuite/unzcrash.cc: Added new option '-b, --bits'. - * testsuite/unzcrash.cc: Added new option '-p, --position'. - * testsuite/unzcrash.cc: Added new option '-s, --size'. + * unzcrash.cc: Use Arg_parser. + * unzcrash.cc: Added new option '-b, --bits'. + * unzcrash.cc: Added new option '-p, --position'. + * unzcrash.cc: Added new option '-s, --size'. 2010-09-16 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -154,7 +171,7 @@ * Version 1.9 released. * main.cc (main): Return at least 1 if closing stdout fails. * Makefile.in: Added '--name' option to help2man invocation. - * testsuite/check.sh: Use 'test1' instead of 'COPYING' for testing. + * check.sh: Use 'test1' instead of 'COPYING' for testing. 2009-09-02 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -179,12 +196,12 @@ is smaller than dictionary size limit. * decoder.cc: Added extra flush calls to improve partial decompression of corrupt files. - * '--test' no more needs '/dev/null'. + * '--test' no longer needs '/dev/null'. * Removed some 'bashisms' from lzdiff and lzgrep. * Dictionary size for options '-1' to '-4' has been changed. * main.cc (signal_handler): Declared as 'extern "C"'. * Makefile.in: Extra files are now installed by default. - * testsuite/check.sh: Test lziprecover. + * check.sh: Test lziprecover. * Added 'export LC_ALL=C' to all scripts. 2009-04-12 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -199,7 +216,7 @@ * Lzdiff and lzgrep now accept '-h' for '--help' and '-V' for '--version'. * Makefile.in: Man page is now installed by default. - * testsuite/check.sh: Verify that files are open in binary mode. + * check.sh: Verify that files are opened in binary mode. 2009-01-24 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -210,7 +227,7 @@ * Added new option '-o, --output'. * main.cc: Read from non regular files if '--stdout' is specified. * Added 'lziprecover', a member recoverer program. - * testsuite/unzcrash.cc: Test all 1-byte errors. + * unzcrash.cc: Test all 1-byte errors. 2008-12-21 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -219,7 +236,7 @@ dictionary size for each file during compression, saving memory during decompression. * Implemented decompression of version 1 files. - * testsuite/check.sh: Replaced 'diff -q' with 'cmp'. + * check.sh: Replaced 'diff -q' with 'cmp'. 2008-12-10 Antonio Diaz Diaz <ant_diaz@teleline.es> @@ -276,7 +293,7 @@ * Version 0.1 released. -Copyright (C) 2008-2017 Antonio Diaz Diaz. +Copyright (C) 2008-2018 Antonio Diaz Diaz. This file is a collection of facts, and thus it is not copyrightable, but just in case, you have unlimited permission to copy, distribute and @@ -58,7 +58,7 @@ After running 'configure', you can run 'make' and 'make install' as explained above. -Copyright (C) 2008-2017 Antonio Diaz Diaz. +Copyright (C) 2008-2018 Antonio Diaz Diaz. This file is free documentation: you have unlimited permission to copy, distribute and modify it. @@ -1,16 +1,42 @@ -Changes in version 1.19: +Changes in version 1.20: -The option '-l, --list' has been ported from lziprecover. +The option '--loose-trailing', has been added. -It is now an error to specify two or more different operations in the -command line (--decompress, --list or --test). +The test used by lzip to discriminate trailing data from a corrupt +header in multimember or concatenated files has been improved to a +Hamming distance (HD) of 3, and the 3 bit flips must happen in different +magic bytes for the test to fail. As a consequence some kinds of files +no longer can be appended to a lzip file as trailing data unless the +'--loose-trailing' option is used when decompressing. +Lziprecover can be used to remove conflicting trailing data from a file. -Compression time of option '-0' has been slightly reduced. +The contents of a corrupt or truncated header found in a multimember +file are now shown, after the error message, in the same format as +trailing data. -Decompression time has been reduced by 2%. +Option '-S, --volume-size' now keeps input files unchanged. -In test mode, lzip now continues checking the rest of the files if any -input file is a terminal. +When creating multimember files or splitting the output in volumes, the +dictionary size is now adjusted for each member individually. -Trailing data are now shown both in hexadecimal and as a string of -printable ASCII characters. +The 'bits/byte' ratio has been replaced with the inverse compression +ratio in the output. + +The progress of decompression is now shown at verbosity level 2 (-vv) or +higher. + +Progress of (de)compression is only shown if stderr is a terminal. + +A final diagnostic is now shown at verbosity level 1 (-v) or higher if +any file fails the test when testing multiple files. + +A second '.lz' extension is no longer added to the argument of '-o' if +it already ends in '.lz' or '.tlz'. + +In case of (de)compressed size mismatch, the stored size is now also +shown in hexadecimal to ease visual comparison. + +The dictionary size is now shown at verbosity level 4 (-vvvv) when +decompressing or testing. + +The new chapter "Meaning of lzip's output" has been added to the manual. @@ -73,7 +73,7 @@ incomprehensible and therefore pointless. Lzip will correctly decompress a file which is the concatenation of two or more compressed files. The result is the concatenation of the -corresponding uncompressed files. Integrity testing of concatenated +corresponding decompressed files. Integrity testing of concatenated compressed files is also supported. Lzip can produce multimember files, and lziprecover can safely recover @@ -111,7 +111,7 @@ range encoding), Igor Pavlov (for putting all the above together in LZMA), and Julian Seward (for bzip2's CLI). -Copyright (C) 2008-2017 Antonio Diaz Diaz. +Copyright (C) 2008-2018 Antonio Diaz Diaz. This file is free documentation: you have unlimited permission to copy, distribute and modify it. diff --git a/arg_parser.cc b/arg_parser.cc index cc7d1e2..008ebc8 100644 --- a/arg_parser.cc +++ b/arg_parser.cc @@ -1,5 +1,5 @@ /* Arg_parser - POSIX/GNU command line argument parser. (C++ version) - Copyright (C) 2006-2017 Antonio Diaz Diaz. + Copyright (C) 2006-2018 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided diff --git a/arg_parser.h b/arg_parser.h index 95b0320..f015881 100644 --- a/arg_parser.h +++ b/arg_parser.h @@ -1,5 +1,5 @@ /* Arg_parser - POSIX/GNU command line argument parser. (C++ version) - Copyright (C) 2006-2017 Antonio Diaz Diaz. + Copyright (C) 2006-2018 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided @@ -1,12 +1,12 @@ #! /bin/sh # configure script for Lzip - LZMA lossless data compressor -# Copyright (C) 2008-2017 Antonio Diaz Diaz. +# Copyright (C) 2008-2018 Antonio Diaz Diaz. # # This configure script is free software: you have unlimited permission # to copy, distribute and modify it. pkgname=lzip -pkgversion=1.19 +pkgversion=1.20 progname=lzip srctrigger=doc/${pkgname}.texi @@ -168,7 +168,7 @@ echo "LDFLAGS = ${LDFLAGS}" rm -f Makefile cat > Makefile << EOF # Makefile for Lzip - LZMA lossless data compressor -# Copyright (C) 2008-2017 Antonio Diaz Diaz. +# Copyright (C) 2008-2018 Antonio Diaz Diaz. # This file was generated automatically by configure. Don't edit. # # This Makefile is free software: you have unlimited permission @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -31,23 +31,6 @@ #include "decoder.h" -void Pretty_print::operator()( const char * const msg ) const - { - if( verbosity >= 0 ) - { - if( first_post ) - { - first_post = false; - std::fprintf( stderr, " %s: ", name_.c_str() ); - for( unsigned i = name_.size(); i < longest_name; ++i ) - std::fputc( ' ', stderr ); - if( !msg ) std::fflush( stderr ); - } - if( msg ) std::fprintf( stderr, "%s\n", msg ); - } - } - - /* Returns the number of bytes really read. If (returned value < size) and (errno == 0), means EOF was reached. */ @@ -94,6 +77,7 @@ bool Range_decoder::read_block() at_stream_end = ( stream_pos < buffer_size ); partial_member_pos += pos; pos = 0; + show_dprogress(); } return pos < stream_pos; } @@ -134,45 +118,56 @@ bool LZ_decoder::verify_trailer( const Pretty_print & pp ) const while( size < File_trailer::size ) trailer.data[size++] = 0; } - if( trailer.data_crc() != crc() ) + const unsigned td_crc = trailer.data_crc(); + if( td_crc != crc() ) { error = true; if( verbosity >= 0 ) { pp(); - std::fprintf( stderr, "CRC mismatch; trailer says %08X, data CRC is %08X\n", - trailer.data_crc(), crc() ); + std::fprintf( stderr, "CRC mismatch; stored %08X, computed %08X\n", + td_crc, crc() ); } } - if( trailer.data_size() != data_size ) + const unsigned long long td_size = trailer.data_size(); + if( td_size != data_size ) { error = true; if( verbosity >= 0 ) { pp(); - std::fprintf( stderr, "Data size mismatch; trailer says %llu, data size is %llu (0x%llX)\n", - trailer.data_size(), data_size, data_size ); + std::fprintf( stderr, "Data size mismatch; stored %llu (0x%llX), computed %llu (0x%llX)\n", + td_size, td_size, data_size, data_size ); } } - if( trailer.member_size() != member_size ) + const unsigned long long tm_size = trailer.member_size(); + if( tm_size != member_size ) { error = true; if( verbosity >= 0 ) { pp(); - std::fprintf( stderr, "Member size mismatch; trailer says %llu, member size is %llu (0x%llX)\n", - trailer.member_size(), member_size, member_size ); + std::fprintf( stderr, "Member size mismatch; stored %llu (0x%llX), computed %llu (0x%llX)\n", + tm_size, tm_size, member_size, member_size ); } } - if( !error && verbosity >= 2 && data_size > 0 && member_size > 0 ) - std::fprintf( stderr, "%6.3f:1, %6.3f bits/byte, %5.2f%% saved. ", - (double)data_size / member_size, - ( 8.0 * member_size ) / data_size, - 100.0 * ( 1.0 - ( (double)member_size / data_size ) ) ); - if( !error && verbosity >= 4 ) - std::fprintf( stderr, "CRC %08X, decompressed %9llu, compressed %8llu. ", - crc(), data_size, member_size ); - return !error; + if( error ) return false; + if( verbosity >= 2 ) + { + if( verbosity >= 4 ) show_header( dictionary_size ); + if( data_size == 0 || member_size == 0 ) + std::fputs( "no data compressed. ", stderr ); + else + std::fprintf( stderr, "%6.3f:1, %5.2f%% ratio, %5.2f%% saved. ", + (double)data_size / member_size, + ( 100.0 * member_size ) / data_size, + 100.0 - ( ( 100.0 * member_size ) / data_size ) ); + if( verbosity >= 4 ) std::fprintf( stderr, "CRC %08X, ", td_crc ); + if( verbosity >= 3 ) + std::fprintf( stderr, "decompressed %9llu, compressed %8llu. ", + data_size, member_size ); + } + return true; } @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -222,11 +222,7 @@ class LZ_decoder bool verify_trailer( const Pretty_print & pp ) const; uint8_t peek_prev() const - { - if( pos > 0 ) return buffer[pos-1]; - if( pos_wrapped ) return buffer[dictionary_size-1]; - return 0; // prev_byte of first byte - } + { return buffer[((pos > 0) ? pos : dictionary_size)-1]; } uint8_t peek( const unsigned distance ) const { @@ -287,7 +283,8 @@ public: crc_( 0xFFFFFFFFU ), outfd( ofd ), pos_wrapped( false ) - {} + // prev_byte of first byte; also for peek( 0 ) on corrupt file + { buffer[dictionary_size-1] = 0; } ~LZ_decoder() { delete[] buffer; } @@ -1,5 +1,5 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.1. -.TH LZIP "1" "April 2017" "lzip 1.19" "User Commands" +.TH LZIP "1" "February 2018" "lzip 1.20" "User Commands" .SH NAME lzip \- reduces the size of files .SH SYNOPSIS @@ -52,7 +52,7 @@ suppress all messages set dictionary size limit in bytes [8 MiB] .TP \fB\-S\fR, \fB\-\-volume\-size=\fR<bytes> -set volume size limit in bytes +set volume size limit in bytes, implies \fB\-k\fR .TP \fB\-t\fR, \fB\-\-test\fR test compressed file integrity @@ -68,6 +68,9 @@ alias for \fB\-0\fR .TP \fB\-\-best\fR alias for \fB\-9\fR +.TP +\fB\-\-loose\-trailing\fR +allow trailing data seeming corrupt header .PP If no file names are given, or if a file is '\-', lzip compresses or decompresses from standard input to standard output. @@ -90,7 +93,7 @@ Report bugs to lzip\-bug@nongnu.org .br Lzip home page: http://www.nongnu.org/lzip/lzip.html .SH COPYRIGHT -Copyright \(co 2017 Antonio Diaz Diaz. +Copyright \(co 2018 Antonio Diaz Diaz. License GPLv2+: GNU GPL version 2 or later <http://gnu.org/licenses/gpl.html> .br This is free software: you are free to change and redistribute it. diff --git a/doc/lzip.info b/doc/lzip.info index cac370c..8591b5b 100644 --- a/doc/lzip.info +++ b/doc/lzip.info @@ -11,11 +11,12 @@ File: lzip.info, Node: Top, Next: Introduction, Up: (dir) Lzip Manual *********** -This manual is for Lzip (version 1.19, 13 April 2017). +This manual is for Lzip (version 1.20, 11 February 2018). * Menu: * Introduction:: Purpose and features of lzip +* Output:: Meaning of lzip's output * Invoking lzip:: Command line interface * Quality assurance:: Design, development and testing of lzip * File format:: Detailed format of the compressed file @@ -28,13 +29,13 @@ This manual is for Lzip (version 1.19, 13 April 2017). * Concept index:: Index of concepts - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This manual is free documentation: you have unlimited permission to copy, distribute and modify it. -File: lzip.info, Node: Introduction, Next: Invoking lzip, Prev: Top, Up: Top +File: lzip.info, Node: Introduction, Next: Output, Prev: Top, Up: Top 1 Introduction ************** @@ -50,7 +51,7 @@ archiving, taking into account both data integrity and decoder availability: * The lzip format provides very safe integrity checking and some data - recovery means. The lziprecover program can repair bit-flip errors + recovery means. The lziprecover program can repair bit flip errors (one of the most common forms of data corruption) in lzip files, and provides data recovery capabilities, including error-checked merging of damaged copies of a file. *Note Data safety: @@ -124,7 +125,7 @@ incomprehensible and therefore pointless. Lzip will correctly decompress a file which is the concatenation of two or more compressed files. The result is the concatenation of the -corresponding uncompressed files. Integrity testing of concatenated +corresponding decompressed files. Integrity testing of concatenated compressed files is also supported. Lzip can produce multimember files, and lziprecover can safely @@ -137,14 +138,58 @@ multivolume compressed tar archives. automatically creating multimember output. The members so created are large, about 2 PiB each. + +File: lzip.info, Node: Output, Next: Invoking lzip, Prev: Introduction, Up: Top + +2 Meaning of lzip's output +************************** + +The output of lzip looks like this: + + lzip -v foo + foo: 6.676:1, 14.98% ratio, 85.02% saved, 450560 in, 67493 out. + + lzip -tvv foo.lz + foo.lz: 6.676:1, 14.98% ratio, 85.02% saved. ok + + The meaning of each field is as follows: + +'N:1' + The compression ratio (uncompressed_size / compressed_size), shown + as N to 1. + +'ratio' + The inverse compression ratio + (compressed_size / uncompressed_size), shown as a percentage. A + decimal ratio is easily obtained by moving the decimal point two + places to the left; 14.98% = 0.1498. + +'saved' + The space saved by compression (1 - ratio), shown as a percentage. + +'in' + The size of the uncompressed data. When decompressing or testing, + it is shown as 'decompressed'. Note that lzip always prints the + uncompressed size before the compressed size when compressing, + decompressing, testing or listing. + +'out' + The size of the compressed data. When decompressing or testing, it + is shown as 'compressed'. + + + When decompressing or testing at verbosity level 4 (-vvvv), the +dictionary size used to compress the file and the CRC32 of the +uncompressed data are also shown. + LANGUAGE NOTE: Uncompressed = not compressed = plain data; it may never have been compressed. Decompressed is used to refer to data which have undergone the process of decompression. -File: lzip.info, Node: Invoking lzip, Next: Quality assurance, Prev: Introduction, Up: Top +File: lzip.info, Node: Invoking lzip, Next: Quality assurance, Prev: Output, Up: Top -2 Invoking lzip +3 Invoking lzip *************** The format for running lzip is: @@ -155,7 +200,7 @@ The format for running lzip is: other FILES and is read just once, the first time it appears in the command line. - Lzip supports the following options: + lzip supports the following options: '-h' '--help' @@ -174,9 +219,10 @@ command line. '-b BYTES' '--member-size=BYTES' - Set the member size limit to BYTES. A small member size may - degrade compression ratio, so use it only when needed. Valid values - range from 100 kB to 2 PiB. Defaults to 2 PiB. + When compressing, set the member size limit to BYTES. A small + member size may degrade compression ratio, so use it only when + needed. Valid values range from 100 kB to 2 PiB. Defaults to + 2 PiB. '-c' '--stdout' @@ -184,15 +230,15 @@ command line. unchanged. If compressing several files, each file is compressed independently. This option is needed when reading from a named pipe (fifo) or from a device. Use it also to recover as much of - the uncompressed data as possible when decompressing a corrupt + the decompressed data as possible when decompressing a corrupt file. '-d' '--decompress' - Decompress the specified file(s). If a file does not exist or - can't be opened, lzip continues decompressing the rest of the - files. If a file fails to decompress, lzip exits immediately - without decompressing the rest of the files. + Decompress the specified files. If a file does not exist or can't + be opened, lzip continues decompressing the rest of the files. If + a file fails to decompress, or is a terminal, lzip exits + immediately without decompressing the rest of the files. '-f' '--force' @@ -200,8 +246,8 @@ command line. '-F' '--recompress' - Force re-compression of files whose name already has the '.lz' or - '.tlz' suffix. + When compressing, force re-compression of files whose name already + has the '.lz' or '.tlz' suffix. '-k' '--keep' @@ -211,7 +257,7 @@ command line. '-l' '--list' Print the uncompressed size, compressed size and percentage saved - of the specified file(s). Trailing data are ignored. The values + of the specified files. Trailing data are ignored. The values produced are correct even for multimember files. If more than one file is given, a final line containing the cumulative sizes is printed. With '-v', the dictionary size, the number of members in @@ -225,19 +271,20 @@ command line. '-m BYTES' '--match-length=BYTES' - Set the match length limit in bytes. After a match this long is - found, the search is finished. Valid values range from 5 to 273. - Larger values usually give better compression ratios but longer - compression times. + When compressing, set the match length limit in bytes. After a + match this long is found, the search is finished. Valid values + range from 5 to 273. Larger values usually give better compression + ratios but longer compression times. '-o FILE' '--output=FILE' When reading from standard input and '--stdout' has not been specified, use 'FILE' as the virtual name of the uncompressed - file. This produces a file named 'FILE' when decompressing, a file - named 'FILE.lz' when compressing, and several files named - 'FILE00001.lz', 'FILE00002.lz', etc, when compressing and - splitting the output in volumes. + file. This produces a file named 'FILE' when decompressing, or a + file named 'FILE.lz' when compressing. A second '.lz' extension is + not added if 'FILE' already ends in '.lz' or '.tlz'. When + compressing and splitting the output in volumes, several files + named 'FILE00001.lz', 'FILE00002.lz', etc, are created. '-q' '--quiet' @@ -245,13 +292,13 @@ command line. '-s BYTES' '--dictionary-size=BYTES' - Set the dictionary size limit in bytes. Lzip will use the smallest - possible dictionary size for each file without exceeding this - limit. Valid values range from 4 KiB to 512 MiB. Values 12 to 29 - are interpreted as powers of two, meaning 2^12 to 2^29 bytes. Note - that dictionary sizes are quantized. If the specified size does - not match one of the valid sizes, it will be rounded upwards by - adding up to (BYTES / 8) to it. + When compressing, set the dictionary size limit in bytes. Lzip + will use the smallest possible dictionary size for each file + without exceeding this limit. Valid values range from 4 KiB to + 512 MiB. Values 12 to 29 are interpreted as powers of two, meaning + 2^12 to 2^29 bytes. Note that dictionary sizes are quantized. If + the specified size does not match one of the valid sizes, it will + be rounded upwards by adding up to (BYTES / 8) to it. For maximum compression you should use a dictionary size limit as large as possible, but keep in mind that the decompression memory @@ -260,38 +307,40 @@ command line. '-S BYTES' '--volume-size=BYTES' - Split the compressed output into several volume files with names - 'original_name00001.lz', 'original_name00002.lz', etc, and set the - volume size limit to BYTES. Each volume is a complete, maybe - multimember, lzip file. A small volume size may degrade compression - ratio, so use it only when needed. Valid values range from 100 kB - to 4 EiB. + When compressing, split the compressed output into several volume + files with names 'original_name00001.lz', 'original_name00002.lz', + etc, and set the volume size limit to BYTES. Input files are kept + unchanged. Each volume is a complete, maybe multimember, lzip + file. A small volume size may degrade compression ratio, so use it + only when needed. Valid values range from 100 kB to 4 EiB. '-t' '--test' - Check integrity of the specified file(s), but don't decompress - them. This really performs a trial decompression and throws away - the result. Use it together with '-v' to see information about - the file(s). If a file fails the test, does not exist, can't be - opened, or is a terminal, lzip continues checking the rest of the - files. + Check integrity of the specified files, but don't decompress them. + This really performs a trial decompression and throws away the + result. Use it together with '-v' to see information about the + files. If a file fails the test, does not exist, can't be opened, + or is a terminal, lzip continues checking the rest of the files. A + final diagnostic is shown at verbosity level 1 or higher if any + file fails the test when testing multiple files. '-v' '--verbose' Verbose mode. - When compressing, show the compression ratio for each file - processed. A second '-v' shows the progress of compression. + When compressing, show the compression ratio and size for each file + processed. When decompressing or testing, further -v's (up to 4) increase the verbosity level, showing status, compression ratio, dictionary size, trailer contents (CRC, data size, member size), and up to 6 bytes of trailing data (if any) both in hexadecimal and as a string of printable ASCII characters. + Two or more '-v' options show the progress of (de)compression. '-0 .. -9' Set the compression parameters (dictionary size and match length limit) as shown in the table below. The default compression level is '-6'. Note that '-9' can be much slower than '-0'. These - options have no effect when decompressing. + options have no effect when decompressing, testing or listing. The bidimensional parameter space of LZMA can't be mapped to a linear scale optimal for all files. If your files are large, very @@ -314,6 +363,13 @@ command line. '--best' Aliases for GNU gzip compatibility. +'--loose-trailing' + When decompressing, testing or listing, allow trailing data whose + first bytes are so similar to the magic bytes of a lzip header + that they can be confused with a corrupt header. Use this option + if a file triggers a "corrupt header" error and the cause is not + indeed a corrupt header. + Numbers given as arguments to options may be followed by a multiplier and an optional 'B' for "byte". @@ -339,7 +395,7 @@ caused lzip to panic. File: lzip.info, Node: Quality assurance, Next: File format, Prev: Invoking lzip, Up: Top -3 Design, development and testing of lzip +4 Design, development and testing of lzip ***************************************** There are two ways of constructing a software design: One way is to make @@ -354,7 +410,7 @@ describes the lessons learned from previous compressors (gzip and bzip2), and their application to the design of lzip. -3.1 Format design +4.1 Format design ================= When gzip was designed in 1992, computers and operating systems were @@ -372,7 +428,7 @@ one of gzip. Probably the worst defect of the gzip format from the point of view of data safety is the variable size of its header. If the byte at -offset 3 (flags) of a gzip member gets corrupted, it may become very +offset 3 (flags) of a gzip member gets corrupted, it may become difficult to recover the data, even if the compressed blocks are intact, because it can't be known with certainty where the compressed blocks begin. @@ -394,8 +450,8 @@ error detection. Any distance larger than the dictionary size acts as a forbidden symbol, allowing the decompressor to detect the approximate position of errors, and leaving very little work for the check sequence (CRC and data sizes) in the detection of errors. Lzip is usually able -to detect all posible bit-flips in the compressed data without -resorting to the check sequence. It would be very difficult to write an +to detect all posible bit flips in the compressed data without +resorting to the check sequence. It would be difficult to write an automatic recovery tool like lziprecover for the gzip format. And, as far as I know, it has never been written. @@ -404,15 +460,14 @@ decompressed data because it provides more accurate error detection than CRC64 up to a compressed size of about 16 GiB, a size larger than that of most files. In the case of lzip, the additional detection capability of the decompressor reduces the probability of undetected errors more -than a million times, making CRC32 more accurate than CRC64 up to about -20 PiB of compressed size. +than a million times beyond what the CRC32 alone provides. The lzip format is designed for long-term archiving. Therefore it excludes any unneeded features that may interfere with the future -extraction of the uncompressed data. +extraction of the decompressed data. -3.1.1 Gzip format (mis)features not present in lzip +4.1.1 Gzip format (mis)features not present in lzip --------------------------------------------------- 'Multiple algorithms' @@ -433,16 +488,22 @@ extraction of the uncompressed data. compressed blocks. 'Optional CRC for the header' - Using an optional checksum for the header is not only a bad idea, - it is an error; it may prevent the extraction of perfectly good - data. For example, if the checksum is used and the bit enabling it - is reset by a bit-flip, the header will appear to be intact (in - spite of being corrupt) while the compressed blocks will appear to - be totally unrecoverable (in spite of being intact). Very - misleading indeed. - - -3.1.2 Lzip format improvements over gzip and bzip2 + Using an optional CRC for the header is not only a bad idea, it is + an error; it circumvents the HD of the CRC and may prevent the + extraction of perfectly good data. For example, if the CRC is used + and the bit enabling it is reset by a bit flip, the header will + appear to be intact (in spite of being corrupt) while the + compressed blocks will appear to be totally unrecoverable (in + spite of being intact). Very misleading indeed. + +'Metadata' + The gzip format stores some metadata, like the modification time + of the original file or the operating system on which compression + took place. This complicates reproducible compression (obtaining + identical compressed output from identical input). + + +4.1.2 Lzip format improvements over gzip and bzip2 -------------------------------------------------- '64-bit size field' @@ -470,7 +531,7 @@ extraction of the uncompressed data. total uncompressed size. -3.2 Quality of implementation +4.2 Quality of implementation ============================= 'Accurate and robust error detection' @@ -516,7 +577,7 @@ extraction of the uncompressed data. File: lzip.info, Node: File format, Next: Algorithm, Prev: Quality assurance, Up: Top -4 File format +5 File format ************* Perfection is reached, not when there is no longer anything to add, but @@ -587,7 +648,7 @@ additional information before, between, or after them. File: lzip.info, Node: Algorithm, Next: Stream format, Prev: File format, Up: Top -5 Algorithm +6 Algorithm *********** In spite of its name (Lempel-Ziv-Markov chain-Algorithm), LZMA is not a @@ -653,7 +714,7 @@ LZMA), and Julian Seward (for bzip2's CLI). File: lzip.info, Node: Stream format, Next: Trailing data, Prev: Algorithm, Up: Top -6 Format of the LZMA stream in lzip files +7 Format of the LZMA stream in lzip files ***************************************** The LZMA algorithm has three parameters, called "special LZMA @@ -693,7 +754,7 @@ the lzip download directory. The source code of lzd is included in appendix A. *Note Reference source code::. -6.1 What is coded +7.1 What is coded ================= The LZMA stream includes literals, matches and repeated matches (matches @@ -768,7 +829,7 @@ slot + direct_bits distances from 4 to 127 slot + (direct_bits - 4) + 4 bits distances from 128 to 2^32 - 1 -6.2 The coding contexts +7.2 The coding contexts ======================= These contexts ('Bit_model' in the source), are integers or arrays of @@ -858,7 +919,7 @@ difference is found, the rest of the byte is decoded using the normal bit tree context. (See 'decode_matched' in the source). -6.3 The range decoder +7.3 The range decoder ===================== The LZMA stream is consumed one byte at a time by the range decoder. @@ -878,7 +939,7 @@ range decoder. This is done by shifting 5 bytes in the initialization of source). -6.4 Decoding the LZMA stream +7.4 Decoding the LZMA stream ============================ After decoding the member header and obtaining the dictionary size, the @@ -891,7 +952,7 @@ Stream" marker is decoded. File: lzip.info, Node: Trailing data, Next: Examples, Prev: Stream format, Up: Top -7 Extra data appended to the file +8 Extra data appended to the file ********************************* Sometimes extra data are found appended to a lzip file after the last @@ -903,10 +964,11 @@ member. Such trailing data may be: * Useful data added by the user; a cryptographically secure hash, a description of file contents, etc. It is safe to append any amount - of text to a lzip file as long as the text does not begin with the - string "LZIP", and does not contain any zero bytes (null - characters). Nonzero bytes and zero bytes can't be safely mixed in - trailing data. + of text to a lzip file as long as none of the first four bytes of + the text match the corresponding byte in the string "LZIP", and + the text does not contain any zero bytes (null characters). + Nonzero bytes and zero bytes can't be safely mixed in trailing + data. * Garbage added by some not totally successful copy operation. @@ -914,12 +976,17 @@ member. Such trailing data may be: and hash value (for a chosen hash) coincide with those of another file. - * In very rare cases, trailing data could be the corrupt header of - another member. In multimember or concatenated files the - probability of corruption happening in the magic bytes is 5 times - smaller than the probability of getting a false positive caused by - the corruption of the integrity information itself. Therefore it - can be considered to be below the noise level. + * In rare cases, trailing data could be the corrupt header of another + member. In multimember or concatenated files the probability of + corruption happening in the magic bytes is 5 times smaller than the + probability of getting a false positive caused by the corruption + of the integrity information itself. Therefore it can be + considered to be below the noise level. Additionally, the test + used by lzip to discriminate trailing data from a corrupt header + has a Hamming distance (HD) of 3, and the 3 bit flips must happen + in different magic bytes for the test to fail. In any case, the + option '--trailing-error' guarantees that any corrupt header will + be detected. Trailing data are in no way part of the lzip file format, but tools reading lzip files are expected to behave as correctly and usefully as @@ -933,7 +1000,7 @@ cases where a file containing trailing data must be rejected, the option File: lzip.info, Node: Examples, Next: Problems, Prev: Trailing data, Up: Top -8 A small tutorial with examples +9 A small tutorial with examples ******************************** WARNING! Even if lzip is bug-free, other causes may result in a corrupt @@ -1015,8 +1082,8 @@ file with a member size of 32 MiB. File: lzip.info, Node: Problems, Next: Reference source code, Prev: Examples, Up: Top -9 Reporting bugs -**************** +10 Reporting bugs +***************** There are probably bugs in lzip. There are certainly errors and omissions in this manual. If you report them, they will get fixed. If @@ -1034,7 +1101,7 @@ Appendix A Reference source code ******************************** /* Lzd - Educational decompressor for the lzip format - Copyright (C) 2013-2017 Antonio Diaz Diaz. + Copyright (C) 2013-2018 Antonio Diaz Diaz. This program is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided @@ -1350,9 +1417,9 @@ bool LZ_decoder::decode_member() // Returns false if error Bit_model bm_align[dis_align_size]; Len_model match_len_model; Len_model rep_len_model; - unsigned rep0 = 0; // rep[0-3] latest four distances - unsigned rep1 = 0; // used for efficient coding of - unsigned rep2 = 0; // repeated distances + unsigned rep0 = 0; // rep[0-3] latest four distances + unsigned rep1 = 0; // used for efficient coding of + unsigned rep2 = 0; // repeated distances unsigned rep3 = 0; State state; @@ -1447,7 +1514,7 @@ int main( const int argc, const char * const argv[] ) "It is not safe to use lzd for any real work.\n" "\nUsage: %s < file.lz > file\n", argv[0] ); std::printf( "Lzd decompresses from standard input to standard output.\n" - "\nCopyright (C) 2017 Antonio Diaz Diaz.\n" + "\nCopyright (C) 2018 Antonio Diaz Diaz.\n" "This is free software: you are free to change and redistribute it.\n" "There is NO WARRANTY, to the extent permitted by law.\n" "Report bugs to lzip-bug@nongnu.org\n" @@ -1492,7 +1559,7 @@ int main( const int argc, const char * const argv[] ) } if( std::fclose( stdout ) != 0 ) - { std::fprintf( stderr, "Can't close stdout: %s\n", std::strerror( errno ) ); + { std::fprintf( stderr, "Error closing stdout: %s\n", std::strerror( errno ) ); return 1; } return 0; } @@ -1515,6 +1582,7 @@ Concept index * introduction: Introduction. (line 6) * invoking: Invoking lzip. (line 6) * options: Invoking lzip. (line 6) +* output: Output. (line 6) * quality assurance: Quality assurance. (line 6) * reference source code: Reference source code. (line 6) * trailing data: Trailing data. (line 6) @@ -1525,19 +1593,20 @@ Concept index Tag Table: Node: Top208 -Node: Introduction1147 -Node: Invoking lzip6367 -Ref: --trailing-error6931 -Node: Quality assurance13849 -Node: File format22003 -Node: Algorithm24407 -Node: Stream format27233 -Node: Trailing data37973 -Node: Examples39874 -Ref: concat-example41047 -Node: Problems42085 -Node: Reference source code42615 -Node: Concept index56932 +Node: Introduction1202 +Node: Output6228 +Node: Invoking lzip7740 +Ref: --trailing-error8298 +Node: Quality assurance15946 +Node: File format24354 +Node: Algorithm26758 +Node: Stream format29584 +Node: Trailing data40324 +Node: Examples42600 +Ref: concat-example43773 +Node: Problems44811 +Node: Reference source code45343 +Node: Concept index59659 End Tag Table diff --git a/doc/lzip.texi b/doc/lzip.texi index 17a2b1e..d2efdc9 100644 --- a/doc/lzip.texi +++ b/doc/lzip.texi @@ -6,8 +6,8 @@ @finalout @c %**end of header -@set UPDATED 13 April 2017 -@set VERSION 1.19 +@set UPDATED 11 February 2018 +@set VERSION 1.20 @dircategory Data Compression @direntry @@ -36,6 +36,7 @@ This manual is for Lzip (version @value{VERSION}, @value{UPDATED}). @menu * Introduction:: Purpose and features of lzip +* Output:: Meaning of lzip's output * Invoking lzip:: Command line interface * Quality assurance:: Design, development and testing of lzip * File format:: Detailed format of the compressed file @@ -49,7 +50,7 @@ This manual is for Lzip (version @value{VERSION}, @value{UPDATED}). @end menu @sp 1 -Copyright @copyright{} 2008-2017 Antonio Diaz Diaz. +Copyright @copyright{} 2008-2018 Antonio Diaz Diaz. This manual is free documentation: you have unlimited permission to copy, distribute and modify it. @@ -74,7 +75,7 @@ availability: The lzip format provides very safe integrity checking and some data recovery means. The @uref{http://www.nongnu.org/lzip/manual/lziprecover_manual.html#Data-safety,,lziprecover} -program can repair bit-flip errors (one of the most common forms of data +program can repair bit flip errors (one of the most common forms of data corruption) in lzip files, and provides data recovery capabilities, including error-checked merging of damaged copies of a file. @ifnothtml @@ -123,9 +124,9 @@ choice of dictionary size limit. The amount of memory required for compression is about 1 or 2 times the dictionary size limit (1 if input file size is less than dictionary size limit, else 2) plus 9 times the dictionary size really used. The option -@samp{-0} is special and only requires about 1.5 MiB at most. The amount -of memory required for decompression is about 46 kB larger than the -dictionary size really used. +@samp{-0} is special and only requires about @w{1.5 MiB} at most. The +amount of memory required for decompression is about @w{46 kB} larger +than the dictionary size really used. When compressing, lzip replaces every file given in the command line with a compressed version of itself, with the name "original_name.lz". @@ -154,7 +155,7 @@ incomprehensible and therefore pointless. Lzip will correctly decompress a file which is the concatenation of two or more compressed files. The result is the concatenation of the -corresponding uncompressed files. Integrity testing of concatenated +corresponding decompressed files. Integrity testing of concatenated compressed files is also supported. Lzip can produce multimember files, and lziprecover can safely recover @@ -165,7 +166,53 @@ compressed tar archives. Lzip is able to compress and decompress streams of unlimited size by automatically creating multimember output. The members so created are -large, about 2 PiB each. +large, about @w{2 PiB} each. + + +@node Output +@chapter Meaning of lzip's output +@cindex output + +The output of lzip looks like this: + +@example +lzip -v foo + foo: 6.676:1, 14.98% ratio, 85.02% saved, 450560 in, 67493 out. + +lzip -tvv foo.lz + foo.lz: 6.676:1, 14.98% ratio, 85.02% saved. ok +@end example + +The meaning of each field is as follows: + +@table @code +@item N:1 +The compression ratio @w{(uncompressed_size / compressed_size)}, shown +as N to 1. + +@item ratio +The inverse compression ratio @w{(compressed_size / uncompressed_size)}, +shown as a percentage. A decimal ratio is easily obtained by moving the +decimal point two places to the left; @w{14.98% = 0.1498}. + +@item saved +The space saved by compression @w{(1 - ratio)}, shown as a percentage. + +@item in +The size of the uncompressed data. When decompressing or testing, it is +shown as @code{decompressed}. Note that lzip always prints the +uncompressed size before the compressed size when compressing, +decompressing, testing or listing. + +@item out +The size of the compressed data. When decompressing or testing, it is +shown as @code{compressed}. + +@end table + +When decompressing or testing at verbosity level 4 (-vvvv), the +dictionary size used to compress the file and the CRC32 of the +uncompressed data are also shown. LANGUAGE NOTE: Uncompressed = not compressed = plain data; it may never have been compressed. Decompressed is used to refer to data which have @@ -190,7 +237,7 @@ lzip [@var{options}] [@var{files}] mixed with other @var{files} and is read just once, the first time it appears in the command line. -Lzip supports the following options: +lzip supports the following options: @table @code @item -h @@ -210,24 +257,24 @@ garbage that can be safely ignored. @xref{concat-example}. @item -b @var{bytes} @itemx --member-size=@var{bytes} -Set the member size limit to @var{bytes}. A small member size may -degrade compression ratio, so use it only when needed. Valid values -range from 100 kB to 2 PiB. Defaults to 2 PiB. +When compressing, set the member size limit to @var{bytes}. A small +member size may degrade compression ratio, so use it only when needed. +Valid values range from @w{100 kB} to @w{2 PiB}. Defaults to @w{2 PiB}. @item -c @itemx --stdout Compress or decompress to standard output; keep input files unchanged. If compressing several files, each file is compressed independently. This option is needed when reading from a named pipe (fifo) or from a -device. Use it also to recover as much of the uncompressed data as +device. Use it also to recover as much of the decompressed data as possible when decompressing a corrupt file. @item -d @itemx --decompress -Decompress the specified file(s). If a file does not exist or can't be +Decompress the specified files. If a file does not exist or can't be opened, lzip continues decompressing the rest of the files. If a file -fails to decompress, lzip exits immediately without decompressing the -rest of the files. +fails to decompress, or is a terminal, lzip exits immediately without +decompressing the rest of the files. @item -f @itemx --force @@ -235,8 +282,8 @@ Force overwrite of output files. @item -F @itemx --recompress -Force re-compression of files whose name already has the @samp{.lz} or -@samp{.tlz} suffix. +When compressing, force re-compression of files whose name already has +the @samp{.lz} or @samp{.tlz} suffix. @item -k @itemx --keep @@ -245,7 +292,7 @@ Keep (don't delete) input files during compression or decompression. @item -l @itemx --list Print the uncompressed size, compressed size and percentage saved of the -specified file(s). Trailing data are ignored. The values produced are +specified files. Trailing data are ignored. The values produced are correct even for multimember files. If more than one file is given, a final line containing the cumulative sizes is printed. With @samp{-v}, the dictionary size, the number of members in the file, and the amount @@ -258,18 +305,21 @@ verifies that none of the specified files contain trailing data. @item -m @var{bytes} @itemx --match-length=@var{bytes} -Set the match length limit in bytes. After a match this long is found, -the search is finished. Valid values range from 5 to 273. Larger values -usually give better compression ratios but longer compression times. +When compressing, set the match length limit in bytes. After a match +this long is found, the search is finished. Valid values range from 5 to +273. Larger values usually give better compression ratios but longer +compression times. @item -o @var{file} @itemx --output=@var{file} When reading from standard input and @samp{--stdout} has not been specified, use @samp{@var{file}} as the virtual name of the uncompressed -file. This produces a file named @samp{@var{file}} when decompressing, a -file named @samp{@var{file}.lz} when compressing, and several files -named @samp{@var{file}00001.lz}, @samp{@var{file}00002.lz}, etc, when -compressing and splitting the output in volumes. +file. This produces a file named @samp{@var{file}} when decompressing, +or a file named @samp{@var{file}.lz} when compressing. A second +@samp{.lz} extension is not added if @samp{@var{file}} already ends in +@samp{.lz} or @samp{.tlz}. When compressing and splitting the output in +volumes, several files named @samp{@var{file}00001.lz}, +@samp{@var{file}00002.lz}, etc, are created. @item -q @itemx --quiet @@ -277,12 +327,12 @@ Quiet operation. Suppress all messages. @item -s @var{bytes} @itemx --dictionary-size=@var{bytes} -Set the dictionary size limit in bytes. Lzip will use the smallest -possible dictionary size for each file without exceeding this limit. -Valid values range from 4 KiB to 512 MiB. Values 12 to 29 are -interpreted as powers of two, meaning 2^12 to 2^29 bytes. Note that -dictionary sizes are quantized. If the specified size does not match one -of the valid sizes, it will be rounded upwards by adding up to +When compressing, set the dictionary size limit in bytes. Lzip will use +the smallest possible dictionary size for each file without exceeding +this limit. Valid values range from @w{4 KiB} to @w{512 MiB}. Values 12 +to 29 are interpreted as powers of two, meaning 2^12 to 2^29 bytes. Note +that dictionary sizes are quantized. If the specified size does not +match one of the valid sizes, it will be rounded upwards by adding up to @w{(@var{bytes} / 8)} to it. For maximum compression you should use a dictionary size limit as large @@ -291,37 +341,40 @@ is affected at compression time by the choice of dictionary size limit. @item -S @var{bytes} @itemx --volume-size=@var{bytes} -Split the compressed output into several volume files with names -@samp{original_name00001.lz}, @samp{original_name00002.lz}, etc, and set -the volume size limit to @var{bytes}. Each volume is a complete, maybe -multimember, lzip file. A small volume size may degrade compression -ratio, so use it only when needed. Valid values range from 100 kB to 4 -EiB. +When compressing, split the compressed output into several volume files +with names @samp{original_name00001.lz}, @samp{original_name00002.lz}, +etc, and set the volume size limit to @var{bytes}. Input files are kept +unchanged. Each volume is a complete, maybe multimember, lzip file. A +small volume size may degrade compression ratio, so use it only when +needed. Valid values range from @w{100 kB} to @w{4 EiB}. @item -t @itemx --test -Check integrity of the specified file(s), but don't decompress them. -This really performs a trial decompression and throws away the result. -Use it together with @samp{-v} to see information about the file(s). If -a file fails the test, does not exist, can't be opened, or is a -terminal, lzip continues checking the rest of the files. +Check integrity of the specified files, but don't decompress them. This +really performs a trial decompression and throws away the result. Use it +together with @samp{-v} to see information about the files. If a file +fails the test, does not exist, can't be opened, or is a terminal, lzip +continues checking the rest of the files. A final diagnostic is shown at +verbosity level 1 or higher if any file fails the test when testing +multiple files. @item -v @itemx --verbose Verbose mode.@* -When compressing, show the compression ratio for each file processed. A -second @samp{-v} shows the progress of compression.@* +When compressing, show the compression ratio and size for each file +processed.@* When decompressing or testing, further -v's (up to 4) increase the verbosity level, showing status, compression ratio, dictionary size, trailer contents (CRC, data size, member size), and up to 6 bytes of trailing data (if any) both in hexadecimal and as a string of printable -ASCII characters. +ASCII characters.@* +Two or more @samp{-v} options show the progress of (de)compression. @item -0 .. -9 Set the compression parameters (dictionary size and match length limit) as shown in the table below. The default compression level is @samp{-6}. Note that @samp{-9} can be much slower than @samp{-0}. These options -have no effect when decompressing. +have no effect when decompressing, testing or listing. The bidimensional parameter space of LZMA can't be mapped to a linear scale optimal for all files. If your files are large, very repetitive, @@ -346,6 +399,12 @@ etc, you may need to use the @samp{--dictionary-size} and @itemx --best Aliases for GNU gzip compatibility. +@item --loose-trailing +When decompressing, testing or listing, allow trailing data whose first +bytes are so similar to the magic bytes of a lzip header that they can +be confused with a corrupt header. Use this option if a file triggers a +"corrupt header" error and the cause is not indeed a corrupt header. + @end table Numbers given as arguments to options may be followed by a multiplier @@ -405,7 +464,7 @@ of gzip. Probably the worst defect of the gzip format from the point of view of data safety is the variable size of its header. If the byte at offset 3 -(flags) of a gzip member gets corrupted, it may become very difficult to +(flags) of a gzip member gets corrupted, it may become difficult to recover the data, even if the compressed blocks are intact, because it can't be known with certainty where the compressed blocks begin. @@ -426,22 +485,21 @@ distance larger than the dictionary size acts as a forbidden symbol, allowing the decompressor to detect the approximate position of errors, and leaving very little work for the check sequence (CRC and data sizes) in the detection of errors. Lzip is usually able to detect all posible -bit-flips in the compressed data without resorting to the check -sequence. It would be very difficult to write an automatic recovery tool -like lziprecover for the gzip format. And, as far as I know, it has -never been written. +bit flips in the compressed data without resorting to the check +sequence. It would be difficult to write an automatic recovery tool like +lziprecover for the gzip format. And, as far as I know, it has never +been written. Lzip, like gzip and bzip2, uses a CRC32 to check the integrity of the decompressed data because it provides more accurate error detection than -CRC64 up to a compressed size of about 16 GiB, a size larger than that -of most files. In the case of lzip, the additional detection capability -of the decompressor reduces the probability of undetected errors more -than a million times, making CRC32 more accurate than CRC64 up to about -20 PiB of compressed size. +CRC64 up to a compressed size of about @w{16 GiB}, a size larger than +that of most files. In the case of lzip, the additional detection +capability of the decompressor reduces the probability of undetected +errors more than a million times beyond what the CRC32 alone provides. The lzip format is designed for long-term archiving. Therefore it excludes any unneeded features that may interfere with the future -extraction of the uncompressed data. +extraction of the decompressed data. @sp 1 @subsection Gzip format (mis)features not present in lzip @@ -467,12 +525,20 @@ header CRC nor the compressed blocks. @item Optional CRC for the header -Using an optional checksum for the header is not only a bad idea, it is -an error; it may prevent the extraction of perfectly good data. For -example, if the checksum is used and the bit enabling it is reset by a -bit-flip, the header will appear to be intact (in spite of being -corrupt) while the compressed blocks will appear to be totally -unrecoverable (in spite of being intact). Very misleading indeed. +Using an optional CRC for the header is not only a bad idea, it is an +error; it circumvents the HD of the CRC and may prevent the extraction +of perfectly good data. For example, if the CRC is used and the bit +enabling it is reset by a bit flip, the header will appear to be intact +(in spite of being corrupt) while the compressed blocks will appear to +be totally unrecoverable (in spite of being intact). Very misleading +indeed. + +@item Metadata + +The gzip format stores some metadata, like the modification time of the +original file or the operating system on which compression took place. +This complicates reproducible compression (obtaining identical +compressed output from identical input). @end table @@ -483,7 +549,7 @@ unrecoverable (in spite of being intact). Very misleading indeed. Probably the most frequently reported shortcoming of the gzip format is that it only stores the least significant 32 bits of the uncompressed -size. The size of any file larger than 4 GiB gets truncated. +size. The size of any file larger than @w{4 GiB} gets truncated. Bzip2 does not store the uncompressed size of the file. @@ -960,9 +1026,10 @@ padding zero bytes to a lzip file. @item Useful data added by the user; a cryptographically secure hash, a description of file contents, etc. It is safe to append any amount of -text to a lzip file as long as the text does not begin with the string -"LZIP", and does not contain any zero bytes (null characters). Nonzero -bytes and zero bytes can't be safely mixed in trailing data. +text to a lzip file as long as none of the first four bytes of the text +match the corresponding byte in the string "LZIP", and the text does not +contain any zero bytes (null characters). Nonzero bytes and zero bytes +can't be safely mixed in trailing data. @item Garbage added by some not totally successful copy operation. @@ -972,12 +1039,16 @@ Malicious data added to the file in order to make its total size and hash value (for a chosen hash) coincide with those of another file. @item -In very rare cases, trailing data could be the corrupt header of another +In rare cases, trailing data could be the corrupt header of another member. In multimember or concatenated files the probability of corruption happening in the magic bytes is 5 times smaller than the probability of getting a false positive caused by the corruption of the integrity information itself. Therefore it can be considered to be below -the noise level. +the noise level. Additionally, the test used by lzip to discriminate +trailing data from a corrupt header has a Hamming distance (HD) of 3, +and the 3 bit flips must happen in different magic bytes for the test to +fail. In any case, the option @samp{--trailing-error} guarantees that +any corrupt header will be detected. @end itemize Trailing data are in no way part of the lzip file format, but tools @@ -1013,7 +1084,7 @@ lzip -v file @sp 1 @noindent Example 2: Like example 1 but the created @samp{file.lz} is multimember -with a member size of 1 MiB. The compression ratio is not shown. +with a member size of @w{1 MiB}. The compression ratio is not shown. @example lzip -b 1MiB file @@ -1062,7 +1133,7 @@ Do this instead @sp 1 @noindent -Example 7: Decompress @samp{file.lz} partially until 10 KiB of +Example 7: Decompress @samp{file.lz} partially until @w{10 KiB} of decompressed data are produced. @example @@ -1081,7 +1152,7 @@ lzip -cd file.lz | dd bs=1000 skip=10 count=5 @sp 1 @noindent Example 9: Create a multivolume compressed tar archive with a volume -size of 1440 KiB. +size of @w{1440 KiB}. @example tar -c some_directory | lzip -S 1440KiB -o volume_name @@ -1098,8 +1169,8 @@ lzip -cd volume_name*.lz | tar -xf - @sp 1 @noindent Example 11: Create a multivolume compressed backup of a large database -file with a volume size of 650 MB, where each volume is a multimember -file with a member size of 32 MiB. +file with a volume size of @w{650 MB}, where each volume is a +multimember file with a member size of @w{32 MiB}. @example lzip -b 32MiB -S 650MB big_db @@ -1127,7 +1198,7 @@ find by running @w{@code{lzip --version}}. @verbatim /* Lzd - Educational decompressor for the lzip format - Copyright (C) 2013-2017 Antonio Diaz Diaz. + Copyright (C) 2013-2018 Antonio Diaz Diaz. This program is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided @@ -1443,9 +1514,9 @@ bool LZ_decoder::decode_member() // Returns false if error Bit_model bm_align[dis_align_size]; Len_model match_len_model; Len_model rep_len_model; - unsigned rep0 = 0; // rep[0-3] latest four distances - unsigned rep1 = 0; // used for efficient coding of - unsigned rep2 = 0; // repeated distances + unsigned rep0 = 0; // rep[0-3] latest four distances + unsigned rep1 = 0; // used for efficient coding of + unsigned rep2 = 0; // repeated distances unsigned rep3 = 0; State state; @@ -1540,7 +1611,7 @@ int main( const int argc, const char * const argv[] ) "It is not safe to use lzd for any real work.\n" "\nUsage: %s < file.lz > file\n", argv[0] ); std::printf( "Lzd decompresses from standard input to standard output.\n" - "\nCopyright (C) 2017 Antonio Diaz Diaz.\n" + "\nCopyright (C) 2018 Antonio Diaz Diaz.\n" "This is free software: you are free to change and redistribute it.\n" "There is NO WARRANTY, to the extent permitted by law.\n" "Report bugs to lzip-bug@nongnu.org\n" @@ -1585,7 +1656,7 @@ int main( const int argc, const char * const argv[] ) } if( std::fclose( stdout ) != 0 ) - { std::fprintf( stderr, "Can't close stdout: %s\n", std::strerror( errno ) ); + { std::fprintf( stderr, "Error closing stdout: %s\n", std::strerror( errno ) ); return 1; } return 0; } @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -252,7 +252,7 @@ class LZ_encoder : public LZ_encoder_base int sequence_optimizer( const int reps[num_rep_distances], const State state ); - enum { before = max_num_trials, + enum { before_size = max_num_trials, // bytes to keep in buffer after pos after_size = ( 2 * max_match_len ) + 1, dict_factor = 2, @@ -265,7 +265,7 @@ public: LZ_encoder( const int dict_size, const int len_limit, const int ifd, const int outfd ) : - LZ_encoder_base( before, dict_size, after_size, dict_factor, + LZ_encoder_base( before_size, dict_size, after_size, dict_factor, num_prev_positions23, pos_array_factor, ifd, outfd ), cycles( ( len_limit < max_match_len ) ? 16 + ( len_limit / 2 ) : 256 ), match_len_limit( len_limit ), diff --git a/encoder_base.cc b/encoder_base.cc index 55ce376..4e2b765 100644 --- a/encoder_base.cc +++ b/encoder_base.cc @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -68,16 +68,17 @@ void Matchfinder_base::normalize_pos() } -Matchfinder_base::Matchfinder_base( const int before, const int dict_size, - const int after_size, const int dict_factor, - const int num_prev_positions23, +Matchfinder_base::Matchfinder_base( const int before_size_, + const int dict_size, const int after_size, + const int dict_factor, const int num_prev_positions23_, const int pos_array_factor, const int ifd ) : partial_data_pos( 0 ), - before_size( before ), + before_size( before_size_ ), pos( 0 ), cyclic_pos( 0 ), stream_pos( 0 ), + num_prev_positions23( num_prev_positions23_ ), infd( ifd ), at_stream_end( false ) { @@ -105,8 +106,8 @@ Matchfinder_base::Matchfinder_base( const int before, const int dict_size, size >>= 1; key4_mask = size - 1; size += num_prev_positions23; - num_prev_positions = size; + pos_array_size = pos_array_factor * ( dictionary_size + 1 ); size += pos_array_size; if( size * sizeof prev_positions[0] <= size ) prev_positions = 0; @@ -125,8 +126,19 @@ void Matchfinder_base::reset() stream_pos -= pos; pos = 0; cyclic_pos = 0; - for( int i = 0; i < num_prev_positions; ++i ) prev_positions[i] = 0; read_block(); + if( at_stream_end && stream_pos < dictionary_size ) + { + dictionary_size = std::max( (int)min_dictionary_size, stream_pos ); + int size = 1 << std::max( 16, real_bits( dictionary_size - 1 ) - 2 ); + if( dictionary_size > 1 << 26 ) // 64 MiB + size >>= 1; + key4_mask = size - 1; + size += num_prev_positions23; + num_prev_positions = size; + pos_array = prev_positions + num_prev_positions; + } + for( int i = 0; i < num_prev_positions; ++i ) prev_positions[i] = 0; } @@ -138,7 +150,7 @@ void Range_encoder::flush_data() throw Error( "Write error" ); partial_member_pos += pos; pos = 0; - show_progress(); + show_cprogress(); } } @@ -177,5 +189,5 @@ void LZ_encoder_base::reset() bm_align[0].reset( dis_align_size ); match_len_model.reset(); rep_len_model.reset(); - renc.reset(); + renc.reset( dictionary_size ); } diff --git a/encoder_base.h b/encoder_base.h index e186c2d..6a651f0 100644 --- a/encoder_base.h +++ b/encoder_base.h @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -181,14 +181,15 @@ protected: int stream_pos; // first byte not yet read from file int pos_limit; // when reached, a new block must be read int key4_mask; + const int num_prev_positions23; int num_prev_positions; // size of prev_positions int pos_array_size; const int infd; // input file descriptor bool at_stream_end; // stream_pos shows real end of file - Matchfinder_base( const int before, const int dict_size, - const int after_size, const int dict_factor, - const int num_prev_positions23, + Matchfinder_base( const int before_size_, + const int dict_size, const int after_size, + const int dict_factor, const int num_prev_positions23_, const int pos_array_factor, const int ifd ); ~Matchfinder_base() @@ -250,7 +251,7 @@ class Range_encoder void operator=( const Range_encoder & ); // declared as private public: - void reset() + void reset( const unsigned dictionary_size ) { low = 0; partial_member_pos = 0; @@ -258,18 +259,17 @@ public: range = 0xFFFFFFFFU; ff_count = 0; cache = 0; + header.dictionary_size( dictionary_size ); for( int i = 0; i < File_header::size; ++i ) put_byte( header.data[i] ); } Range_encoder( const unsigned dictionary_size, const int ofd ) : - buffer( new uint8_t[buffer_size] ), - outfd( ofd ) + buffer( new uint8_t[buffer_size] ), outfd( ofd ) { header.set_magic(); - header.dictionary_size( dictionary_size ); - reset(); + reset( dictionary_size ); } ~Range_encoder() { delete[] buffer; } @@ -417,13 +417,13 @@ protected: Len_model rep_len_model; Range_encoder renc; - LZ_encoder_base( const int before, const int dict_size, + LZ_encoder_base( const int before_size, const int dict_size, const int after_size, const int dict_factor, const int num_prev_positions23, const int pos_array_factor, const int ifd, const int outfd ) : - Matchfinder_base( before, dict_size, after_size, dict_factor, + Matchfinder_base( before_size, dict_size, after_size, dict_factor, num_prev_positions23, pos_array_factor, ifd ), crc_( 0xFFFFFFFFU ), renc( dictionary_size, outfd ) diff --git a/fast_encoder.cc b/fast_encoder.cc index 57b21ef..3c5d32b 100644 --- a/fast_encoder.cc +++ b/fast_encoder.cc @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/fast_encoder.h b/fast_encoder.h index 37ab8ef..89b6a0c 100644 --- a/fast_encoder.h +++ b/fast_encoder.h @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -42,7 +42,7 @@ class FLZ_encoder : public LZ_encoder_base } } - enum { before = 0, + enum { before_size = 0, dict_size = 65536, // bytes to keep in buffer after pos after_size = max_match_len, @@ -53,7 +53,7 @@ class FLZ_encoder : public LZ_encoder_base public: FLZ_encoder( const int ifd, const int outfd ) : - LZ_encoder_base( before, dict_size, after_size, dict_factor, + LZ_encoder_base( before_size, dict_size, after_size, dict_factor, num_prev_positions23, pos_array_factor, ifd, outfd ) {} diff --git a/file_index.cc b/file_index.cc index 4a1759c..137f82a 100644 --- a/file_index.cc +++ b/file_index.cc @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -59,7 +59,8 @@ void File_index::set_num_error( const char * const msg, unsigned long long num ) // If successful, push last member and set pos to member header. -bool File_index::skip_trailing_data( const int fd, long long & pos ) +bool File_index::skip_trailing_data( const int fd, long long & pos, + const bool ignore_trailing, const bool loose_trailing ) { enum { block_size = 16384, buffer_size = block_size + File_trailer::size - 1 + File_header::size }; @@ -94,10 +95,13 @@ bool File_index::skip_trailing_data( const int fd, long long & pos ) if( !header.verify_magic() || !header.verify_version() || !isvalid_ds( dictionary_size ) ) continue; if( (*(File_header *)( buffer + i )).verify_prefix( bsize - i ) ) - { - error_ = "Last member in input file is truncated or corrupt."; - retval_ = 2; return false; - } + { error_ = "Last member in input file is truncated or corrupt."; + retval_ = 2; return false; } + if( !loose_trailing && bsize - i >= File_header::size && + (*(File_header *)( buffer + i )).verify_corrupt() ) + { error_ = corrupt_mm_msg; retval_ = 2; return false; } + if( !ignore_trailing ) + { error_ = trailing_msg; retval_ = 2; return false; } pos = ipos + i - member_size; member_vector.push_back( Member( 0, trailer.data_size(), pos, member_size, dictionary_size ) ); @@ -115,7 +119,8 @@ bool File_index::skip_trailing_data( const int fd, long long & pos ) } -File_index::File_index( const int infd, const bool ignore_trailing ) +File_index::File_index( const int infd, const bool ignore_trailing, + const bool loose_trailing ) : isize( lseek( infd, 0, SEEK_END ) ), retval_( 0 ) { if( isize < 0 ) @@ -146,11 +151,10 @@ File_index::File_index( const int infd, const bool ignore_trailing ) const unsigned long long member_size = trailer.member_size(); if( member_size < min_member_size || member_size > (unsigned long long)pos ) { - if( !member_vector.empty() ) - set_num_error( "Member size in trailer is corrupt at pos ", pos - 8 ); - else if( skip_trailing_data( infd, pos ) ) - { if( ignore_trailing ) continue; - error_ = trailing_msg; retval_ = 2; return; } + if( member_vector.empty() ) + { if( skip_trailing_data( infd, pos, ignore_trailing, loose_trailing ) ) + continue; else return; } + set_num_error( "Member size in trailer is corrupt at pos ", pos - 8 ); break; } if( seek_read( infd, header.data, File_header::size, @@ -160,11 +164,10 @@ File_index::File_index( const int infd, const bool ignore_trailing ) if( !header.verify_magic() || !header.verify_version() || !isvalid_ds( dictionary_size ) ) { - if( !member_vector.empty() ) - set_num_error( "Bad header at pos ", pos - member_size ); - else if( skip_trailing_data( infd, pos ) ) - { if( ignore_trailing ) continue; - error_ = trailing_msg; retval_ = 2; return; } + if( member_vector.empty() ) + { if( skip_trailing_data( infd, pos, ignore_trailing, loose_trailing ) ) + continue; else return; } + set_num_error( "Bad header at pos ", pos - member_size ); break; } pos -= member_size; diff --git a/file_index.h b/file_index.h index fbfbef1..7cba508 100644 --- a/file_index.h +++ b/file_index.h @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -55,10 +55,12 @@ class File_index void set_errno_error( const char * const msg ); void set_num_error( const char * const msg, unsigned long long num ); - bool skip_trailing_data( const int fd, long long & pos ); + bool skip_trailing_data( const int fd, long long & pos, + const bool ignore_trailing, const bool loose_trailing ); public: - File_index( const int infd, const bool ignore_trailing ); + File_index( const int infd, const bool ignore_trailing, + const bool loose_trailing ); long members() const { return member_vector.size(); } const std::string & error() const { return error_; } @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -37,7 +37,7 @@ void list_line( const unsigned long long uncomp_size, { if( uncomp_size > 0 ) std::printf( "%15llu %15llu %6.2f%% %s\n", uncomp_size, comp_size, - 100.0 * ( 1.0 - ( (double)comp_size / uncomp_size ) ), + 100.0 - ( ( 100.0 * comp_size ) / uncomp_size ), input_filename ); else std::printf( "%15llu %15llu -INF%% %s\n", uncomp_size, comp_size, @@ -48,7 +48,7 @@ void list_line( const unsigned long long uncomp_size, int list_files( const std::vector< std::string > & filenames, - const bool ignore_trailing ) + const bool ignore_trailing, const bool loose_trailing ) { unsigned long long total_comp = 0, total_uncomp = 0; int files = 0, retval = 0; @@ -65,7 +65,7 @@ int list_files( const std::vector< std::string > & filenames, open_instream( input_filename, &in_stats, true, true ); if( infd < 0 ) { if( retval < 1 ) retval = 1; continue; } - const File_index file_index( infd, ignore_trailing ); + const File_index file_index( infd, ignore_trailing, loose_trailing ); close( infd ); if( file_index.retval() != 0 ) { @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -113,16 +113,19 @@ struct Len_model }; -class Pretty_print +// defined in main.cc +extern int verbosity; + +class Pretty_print // requires global var 'int verbosity' { std::string name_; + std::string padded_name; const char * const stdin_name; unsigned longest_name; mutable bool first_post; public: - Pretty_print( const std::vector< std::string > & filenames, - const int verbosity ) + Pretty_print( const std::vector< std::string > & filenames ) : stdin_name( "(stdin)" ), longest_name( 0 ), first_post( false ) { if( verbosity <= 0 ) return; @@ -140,6 +143,9 @@ public: { if( filename.size() && filename != "-" ) name_ = filename; else name_ = stdin_name; + padded_name = " "; padded_name += name_; padded_name += ": "; + if( name_.size() < longest_name ) + padded_name.append( longest_name - name_.size(), ' ' ); first_post = true; } @@ -208,11 +214,19 @@ struct File_header void set_magic() { std::memcpy( data, magic_string, 4 ); data[4] = 1; } bool verify_magic() const { return ( std::memcmp( data, magic_string, 4 ) == 0 ); } - bool verify_prefix( const int size ) const // detect truncated header + + bool verify_prefix( const int sz ) const // detect (truncated) header { - for( int i = 0; i < size && i < 4; ++i ) + for( int i = 0; i < sz && i < 4; ++i ) if( data[i] != magic_string[i] ) return false; - return ( size > 0 ); + return ( sz > 0 ); + } + bool verify_corrupt() const // detect corrupt header + { + int matches = 0; + for( int i = 0; i < 4; ++i ) + if( data[i] == magic_string[i] ) ++matches; + return ( matches > 1 && matches < 4 ); } uint8_t version() const { return data[4]; } @@ -292,6 +306,7 @@ struct Error const char * const bad_magic_msg = "Bad magic number (file not in lzip format)."; const char * const bad_dict_msg = "Invalid dictionary size in member header."; +const char * const corrupt_mm_msg = "Corrupt header in multimember file."; const char * const trailing_msg = "Trailing data not allowed."; // defined in decoder.cc @@ -300,13 +315,13 @@ int writeblock( const int fd, const uint8_t * const buf, const int size ); // defined in list.cc int list_files( const std::vector< std::string > & filenames, - const bool ignore_trailing ); + const bool ignore_trailing, const bool loose_trailing ); // defined in main.cc -extern int verbosity; struct stat; const char * bad_version( const unsigned version ); const char * format_ds( const unsigned dictionary_size ); +void show_header( const unsigned dictionary_size ); int open_instream( const char * const name, struct stat * const in_statsp, const bool no_ofile, const bool reg_only = false ); void show_error( const char * const msg, const int errcode = 0, @@ -315,7 +330,12 @@ void show_file_error( const char * const filename, const char * const msg, const int errcode = 0 ); void internal_error( const char * const msg ); class Matchfinder_base; -void show_progress( const unsigned long long partial_size = 0, - const Matchfinder_base * const m = 0, - const Pretty_print * const p = 0, - const unsigned long long cfile_size = 0 ); +void show_cprogress( const unsigned long long cfile_size = 0, + const unsigned long long partial_size = 0, + const Matchfinder_base * const m = 0, + const Pretty_print * const p = 0 ); +class Range_decoder; +void show_dprogress( const unsigned long long cfile_size = 0, + const unsigned long long partial_size = 0, + const Range_decoder * const d = 0, + const Pretty_print * const p = 0 ); @@ -1,5 +1,5 @@ /* Lzip - LZMA lossless data compressor - Copyright (C) 2008-2017 Antonio Diaz Diaz. + Copyright (C) 2008-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -75,7 +75,7 @@ namespace { const char * const Program_name = "Lzip"; const char * const program_name = "lzip"; -const char * const program_year = "2017"; +const char * const program_year = "2018"; const char * invocation_name = 0; const struct { const char * from; const char * to; } known_extensions[] = { @@ -115,12 +115,13 @@ void show_help() " -o, --output=<file> if reading standard input, write to <file>\n" " -q, --quiet suppress all messages\n" " -s, --dictionary-size=<bytes> set dictionary size limit in bytes [8 MiB]\n" - " -S, --volume-size=<bytes> set volume size limit in bytes\n" + " -S, --volume-size=<bytes> set volume size limit in bytes, implies -k\n" " -t, --test test compressed file integrity\n" " -v, --verbose be verbose (a 2nd -v gives more)\n" " -0 .. -9 set compression level [default 6]\n" " --fast alias for -0\n" " --best alias for -9\n" + " --loose-trailing allow trailing data seeming corrupt header\n" "If no file names are given, or if a file is '-', lzip compresses or\n" "decompresses from standard input to standard output.\n" "Numbers may be followed by a multiplier: k = kB = 10^3 = 1000,\n" @@ -151,6 +152,21 @@ void show_version() } // end namespace +void Pretty_print::operator()( const char * const msg ) const + { + if( verbosity >= 0 ) + { + if( first_post ) + { + first_post = false; + std::fputs( padded_name.c_str(), stderr ); + if( !msg ) std::fflush( stderr ); + } + if( msg ) std::fprintf( stderr, "%s\n", msg ); + } + } + + const char * bad_version( const unsigned version ) { static char buf[80]; @@ -178,14 +194,13 @@ const char * format_ds( const unsigned dictionary_size ) return buf; } -namespace { void show_header( const unsigned dictionary_size ) { - if( verbosity >= 3 ) - std::fprintf( stderr, "dictionary %s. ", format_ds( dictionary_size ) ); + std::fprintf( stderr, "dictionary %s, ", format_ds( dictionary_size ) ); } +namespace { unsigned long long getnum( const char * const ptr, const unsigned long long llimit, @@ -271,6 +286,35 @@ int extension_index( const std::string & name ) return -1; } + +void set_c_outname( const std::string & name, const bool force_ext, + const bool multifile ) + { + output_filename = name; + if( multifile ) output_filename += "00001"; + if( force_ext || multifile || extension_index( output_filename ) < 0 ) + output_filename += known_extensions[0].from; + } + + +void set_d_outname( const std::string & name, const int eindex ) + { + if( eindex >= 0 ) + { + const std::string from( known_extensions[eindex].from ); + if( name.size() > from.size() ) + { + output_filename.assign( name, 0, name.size() - from.size() ); + output_filename += known_extensions[eindex].to; + return; + } + } + output_filename = name; output_filename += ".out"; + if( verbosity >= 1 ) + std::fprintf( stderr, "%s: Can't guess original name for '%s' -- using '%s'\n", + program_name, name.c_str(), output_filename.c_str() ); + } + } // end namespace int open_instream( const char * const name, struct stat * const in_statsp, @@ -318,33 +362,6 @@ int open_instream2( const char * const name, struct stat * const in_statsp, } -void set_c_outname( const std::string & name, const bool multifile ) - { - output_filename = name; - if( multifile ) output_filename += "00001"; - output_filename += known_extensions[0].from; - } - - -void set_d_outname( const std::string & name, const int eindex ) - { - if( eindex >= 0 ) - { - const std::string from( known_extensions[eindex].from ); - if( name.size() > from.size() ) - { - output_filename.assign( name, 0, name.size() - from.size() ); - output_filename += known_extensions[eindex].to; - return; - } - } - output_filename = name; output_filename += ".out"; - if( verbosity >= 1 ) - std::fprintf( stderr, "%s: Can't guess original name for '%s' -- using '%s'\n", - program_name, name.c_str(), output_filename.c_str() ); - } - - bool open_outstream( const bool force, const bool from_stdin ) { const mode_t usr_rw = S_IRUSR | S_IWUSR; @@ -451,13 +468,12 @@ bool next_filename() } -int compress( const unsigned long long member_size, +int compress( const unsigned long long cfile_size, + const unsigned long long member_size, const unsigned long long volume_size, const int infd, const Lzma_options & encoder_options, const Pretty_print & pp, const struct stat * const in_statsp, const bool zero ) { - const unsigned long long cfile_size = - (in_statsp && S_ISREG( in_statsp->st_mode )) ? in_statsp->st_size / 100 : 0; int retval = 0; LZ_encoder_base * encoder = 0; // polymorphic encoder if( verbosity >= 1 ) pp(); @@ -481,7 +497,7 @@ int compress( const unsigned long long member_size, { const unsigned long long size = ( volume_size > 0 ) ? std::min( member_size, volume_size - partial_volume_size ) : member_size; - show_progress( in_size, encoder, &pp, cfile_size ); // init + show_cprogress( cfile_size, in_size, encoder, &pp ); // init if( !encoder->encode_member( size ) ) { pp( "Encoder error." ); retval = 1; break; } in_size += encoder->data_position(); @@ -510,11 +526,11 @@ int compress( const unsigned long long member_size, if( in_size == 0 || out_size == 0 ) std::fputs( " no data compressed.\n", stderr ); else - std::fprintf( stderr, "%6.3f:1, %6.3f bits/byte, " - "%5.2f%% saved, %llu in, %llu out.\n", + std::fprintf( stderr, "%6.3f:1, %5.2f%% ratio, %5.2f%% saved, " + "%llu in, %llu out.\n", (double)in_size / out_size, - ( 8.0 * out_size ) / in_size, - 100.0 * ( 1.0 - ( (double)out_size / in_size ) ), + ( 100.0 * out_size ) / in_size, + 100.0 - ( ( 100.0 * out_size ) / in_size ), in_size, out_size ); } } @@ -539,9 +555,9 @@ unsigned char xdigit( const unsigned value ) bool show_trailing_data( const uint8_t * const data, const int size, const Pretty_print & pp, const bool all, - const bool ignore_trailing ) + const int ignore_trailing ) // -1 = show { - if( verbosity >= 4 || !ignore_trailing ) + if( verbosity >= 4 || ignore_trailing <= 0 ) { std::string msg; if( !all ) msg = "first bytes of "; @@ -557,14 +573,15 @@ bool show_trailing_data( const uint8_t * const data, const int size, { if( std::isprint( data[i] ) ) msg += data[i]; else msg += '.'; } msg += '\''; pp( msg.c_str() ); - if( !ignore_trailing ) show_file_error( pp.name(), trailing_msg ); + if( ignore_trailing == 0 ) show_file_error( pp.name(), trailing_msg ); } - return ignore_trailing; + return ( ignore_trailing > 0 ); } -int decompress( const int infd, const Pretty_print & pp, - const bool ignore_trailing, const bool testing ) +int decompress( const unsigned long long cfile_size, const int infd, + const Pretty_print & pp, const bool ignore_trailing, + const bool loose_trailing, const bool testing ) { int retval = 0; @@ -578,8 +595,13 @@ int decompress( const int infd, const Pretty_print & pp, const int size = rdec.read_data( header.data, File_header::size ); if( rdec.finished() ) // End Of File { - if( first_member || header.verify_prefix( size ) ) - { pp( "File ends unexpectedly at member header." ); retval = 2; } + if( first_member ) + { show_file_error( pp.name(), "File ends unexpectedly at member header." ); + retval = 2; } + else if( header.verify_prefix( size ) ) + { pp( "Truncated header in multimember file." ); + show_trailing_data( header.data, size, pp, true, -1 ); + retval = 2; } else if( size > 0 && !show_trailing_data( header.data, size, pp, true, ignore_trailing ) ) retval = 2; @@ -589,6 +611,10 @@ int decompress( const int infd, const Pretty_print & pp, { if( first_member ) { show_file_error( pp.name(), bad_magic_msg ); retval = 2; } + else if( !loose_trailing && header.verify_corrupt() ) + { pp( corrupt_mm_msg ); + show_trailing_data( header.data, size, pp, false, -1 ); + retval = 2; } else if( !show_trailing_data( header.data, size, pp, false, ignore_trailing ) ) retval = 2; break; @@ -599,10 +625,10 @@ int decompress( const int infd, const Pretty_print & pp, if( !isvalid_ds( dictionary_size ) ) { pp( bad_dict_msg ); retval = 2; break; } - if( verbosity >= 2 || ( verbosity == 1 && first_member ) ) - { pp(); show_header( dictionary_size ); } + if( verbosity >= 2 || ( verbosity == 1 && first_member ) ) pp(); LZ_decoder decoder( rdec, dictionary_size, outfd ); + show_dprogress( cfile_size, partial_file_pos, &rdec, &pp ); // init const int result = decoder.decode_member( pp ); partial_file_pos += rdec.member_position(); if( result != 0 ) @@ -678,25 +704,61 @@ void internal_error( const char * const msg ) } -void show_progress( const unsigned long long partial_size, - const Matchfinder_base * const m, - const Pretty_print * const p, - const unsigned long long cfile_size ) +void show_cprogress( const unsigned long long cfile_size, + const unsigned long long partial_size, + const Matchfinder_base * const m, + const Pretty_print * const p ) { static unsigned long long csize = 0; // file_size / 100 static unsigned long long psize = 0; static const Matchfinder_base * mb = 0; static const Pretty_print * pp = 0; + static bool enabled = true; - if( verbosity < 2 ) return; - if( m ) // initialize static vars - { csize = cfile_size; psize = partial_size; mb = m; pp = p; } + if( !enabled ) return; + if( p ) // initialize static vars + { + if( verbosity < 2 || !isatty( STDERR_FILENO ) ) { enabled = false; return; } + csize = cfile_size; psize = partial_size; mb = m; pp = p; + } if( mb && pp ) { const unsigned long long pos = psize + mb->data_position(); if( csize > 0 ) - std::fprintf( stderr, "%4llu%%", pos / csize ); - std::fprintf( stderr, " %.1f MB\r", pos / 1000000.0 ); + std::fprintf( stderr, "%4llu%% %.1f MB\r", pos / csize, pos / 1000000.0 ); + else + std::fprintf( stderr, " %.1f MB\r", pos / 1000000.0 ); + pp->reset(); (*pp)(); // restore cursor position + } + } + + +void show_dprogress( const unsigned long long cfile_size, + const unsigned long long partial_size, + const Range_decoder * const d, + const Pretty_print * const p ) + { + static unsigned long long csize = 0; // file_size / 100 + static unsigned long long psize = 0; + static const Range_decoder * rdec = 0; + static const Pretty_print * pp = 0; + static int counter = 0; + static bool enabled = true; + + if( !enabled ) return; + if( p ) // initialize static vars + { + if( verbosity < 2 || !isatty( STDERR_FILENO ) ) { enabled = false; return; } + csize = cfile_size; psize = partial_size; rdec = d; pp = p; counter = 0; + } + if( rdec && pp && --counter <= 0 ) + { + const unsigned long long pos = psize + rdec->member_position(); + counter = 7; // update display every 114688 bytes + if( csize > 0 ) + std::fprintf( stderr, "%4llu%% %.1f MB\r", pos / csize, pos / 1000000.0 ); + else + std::fprintf( stderr, " %.1f MB\r", pos / 1000000.0 ); pp->reset(); (*pp)(); // restore cursor position } } @@ -708,7 +770,7 @@ int main( const int argc, const char * const argv[] ) to the corresponding LZMA compression modes. */ const Lzma_options option_mapping[] = { - { 1 << 16, 16 }, // -0 entry values not used + { 1 << 16, 16 }, // -0 { 1 << 20, 5 }, // -1 { 3 << 19, 6 }, // -2 { 1 << 21, 8 }, // -3 @@ -725,47 +787,49 @@ int main( const int argc, const char * const argv[] ) unsigned long long volume_size = 0; std::string default_output_filename; std::vector< std::string > filenames; - int infd = -1; Mode program_mode = m_compress; bool force = false; bool ignore_trailing = true; bool keep_input_files = false; + bool loose_trailing = false; bool recompress = false; bool to_stdout = false; bool zero = false; invocation_name = argv[0]; + enum { opt_lt = 256 }; const Arg_parser::Option options[] = { - { '0', "fast", Arg_parser::no }, - { '1', 0, Arg_parser::no }, - { '2', 0, Arg_parser::no }, - { '3', 0, Arg_parser::no }, - { '4', 0, Arg_parser::no }, - { '5', 0, Arg_parser::no }, - { '6', 0, Arg_parser::no }, - { '7', 0, Arg_parser::no }, - { '8', 0, Arg_parser::no }, - { '9', "best", Arg_parser::no }, - { 'a', "trailing-error", Arg_parser::no }, - { 'b', "member-size", Arg_parser::yes }, - { 'c', "stdout", Arg_parser::no }, - { 'd', "decompress", Arg_parser::no }, - { 'f', "force", Arg_parser::no }, - { 'F', "recompress", Arg_parser::no }, - { 'h', "help", Arg_parser::no }, - { 'k', "keep", Arg_parser::no }, - { 'l', "list", Arg_parser::no }, - { 'm', "match-length", Arg_parser::yes }, - { 'n', "threads", Arg_parser::yes }, - { 'o', "output", Arg_parser::yes }, - { 'q', "quiet", Arg_parser::no }, - { 's', "dictionary-size", Arg_parser::yes }, - { 'S', "volume-size", Arg_parser::yes }, - { 't', "test", Arg_parser::no }, - { 'v', "verbose", Arg_parser::no }, - { 'V', "version", Arg_parser::no }, - { 0 , 0, Arg_parser::no } }; + { '0', "fast", Arg_parser::no }, + { '1', 0, Arg_parser::no }, + { '2', 0, Arg_parser::no }, + { '3', 0, Arg_parser::no }, + { '4', 0, Arg_parser::no }, + { '5', 0, Arg_parser::no }, + { '6', 0, Arg_parser::no }, + { '7', 0, Arg_parser::no }, + { '8', 0, Arg_parser::no }, + { '9', "best", Arg_parser::no }, + { 'a', "trailing-error", Arg_parser::no }, + { 'b', "member-size", Arg_parser::yes }, + { 'c', "stdout", Arg_parser::no }, + { 'd', "decompress", Arg_parser::no }, + { 'f', "force", Arg_parser::no }, + { 'F', "recompress", Arg_parser::no }, + { 'h', "help", Arg_parser::no }, + { 'k', "keep", Arg_parser::no }, + { 'l', "list", Arg_parser::no }, + { 'm', "match-length", Arg_parser::yes }, + { 'n', "threads", Arg_parser::yes }, + { 'o', "output", Arg_parser::yes }, + { 'q', "quiet", Arg_parser::no }, + { 's', "dictionary-size", Arg_parser::yes }, + { 'S', "volume-size", Arg_parser::yes }, + { 't', "test", Arg_parser::no }, + { 'v', "verbose", Arg_parser::no }, + { 'V', "version", Arg_parser::no }, + { opt_lt, "loose-trailing", Arg_parser::no }, + { 0 , 0, Arg_parser::no } }; const Arg_parser parser( argc, argv, options ); if( parser.error().size() ) // bad option @@ -805,6 +869,7 @@ int main( const int argc, const char * const argv[] ) case 't': set_mode( program_mode, m_test ); break; case 'v': if( verbosity < 4 ) ++verbosity; break; case 'V': show_version(); return 0; + case opt_lt: loose_trailing = true; break; default : internal_error( "uncaught option." ); } } // end process options @@ -823,7 +888,7 @@ int main( const int argc, const char * const argv[] ) if( filenames.empty() ) filenames.push_back("-"); if( program_mode == m_list ) - return list_files( filenames, ignore_trailing ); + return list_files( filenames, ignore_trailing, loose_trailing ); if( program_mode == m_test ) outfd = -1; @@ -837,13 +902,15 @@ int main( const int argc, const char * const argv[] ) ( filenames_given || default_output_filename.size() ) ) set_signals(); - Pretty_print pp( filenames, verbosity ); + Pretty_print pp( filenames ); + int failed_tests = 0; int retval = 0; bool stdin_used = false; for( unsigned i = 0; i < filenames.size(); ++i ) { std::string input_filename; + int infd; struct stat in_stats; output_filename.clear(); @@ -858,12 +925,12 @@ int main( const int argc, const char * const argv[] ) else { if( program_mode == m_compress ) - set_c_outname( default_output_filename, volume_size > 0 ); + set_c_outname( default_output_filename, false, volume_size > 0 ); else output_filename = default_output_filename; if( !open_outstream( force, true ) ) { if( retval < 1 ) retval = 1; - close( infd ); infd = -1; + close( infd ); continue; } } @@ -881,12 +948,12 @@ int main( const int argc, const char * const argv[] ) else { if( program_mode == m_compress ) - set_c_outname( input_filename, volume_size > 0 ); + set_c_outname( input_filename, true, volume_size > 0 ); else set_d_outname( input_filename, eindex ); if( !open_outstream( force, false ) ) { if( retval < 1 ) retval = 1; - close( infd ); infd = -1; + close( infd ); continue; } } @@ -897,33 +964,44 @@ int main( const int argc, const char * const argv[] ) if( !check_tty( pp.name(), infd, program_mode ) ) { if( retval < 1 ) retval = 1; - if( program_mode == m_test ) { close( infd ); infd = -1; continue; } + if( program_mode == m_test ) { close( infd ); continue; } cleanup_and_fail( retval ); } const struct stat * const in_statsp = input_filename.size() ? &in_stats : 0; + const unsigned long long cfile_size = + ( in_statsp && S_ISREG( in_statsp->st_mode ) ) ? + ( in_statsp->st_size + 99 ) / 100 : 0; int tmp; if( program_mode == m_compress ) - tmp = compress( member_size, volume_size, infd, encoder_options, pp, - in_statsp, zero ); + tmp = compress( cfile_size, member_size, volume_size, infd, + encoder_options, pp, in_statsp, zero ); else - tmp = decompress( infd, pp, ignore_trailing, program_mode == m_test ); + tmp = decompress( cfile_size, infd, pp, ignore_trailing, + loose_trailing, program_mode == m_test ); if( tmp > retval ) retval = tmp; - if( tmp && program_mode != m_test ) cleanup_and_fail( retval ); + if( tmp ) + { if( program_mode != m_test ) cleanup_and_fail( retval ); + else ++failed_tests; } if( delete_output_on_interrupt ) close_and_set_permissions( in_statsp ); if( input_filename.size() ) { - close( infd ); infd = -1; - if( !keep_input_files && !to_stdout && program_mode != m_test ) + close( infd ); + if( !keep_input_files && !to_stdout && program_mode != m_test && + ( program_mode != m_compress || volume_size == 0 ) ) std::remove( input_filename.c_str() ); } } if( outfd >= 0 && close( outfd ) != 0 ) { - show_error( "Can't close stdout", errno ); + show_error( "Error closing stdout", errno ); if( retval < 1 ) retval = 1; } + if( failed_tests > 0 && verbosity >= 1 && filenames.size() > 1 ) + std::fprintf( stderr, "%s: warning: %d %s failed the test.\n", + program_name, failed_tests, + ( failed_tests == 1 ) ? "file" : "files" ); return retval; } diff --git a/testsuite/check.sh b/testsuite/check.sh index 79e22eb..6cb1616 100755 --- a/testsuite/check.sh +++ b/testsuite/check.sh @@ -1,6 +1,6 @@ #! /bin/sh # check script for Lzip - LZMA lossless data compressor -# Copyright (C) 2008-2017 Antonio Diaz Diaz. +# Copyright (C) 2008-2018 Antonio Diaz Diaz. # # This script is free software: you have unlimited permission # to copy, distribute and modify it. @@ -53,6 +53,8 @@ done [ $? = 2 ] || test_failed $LINENO "${LZIP}" -cdq < in [ $? = 2 ] || test_failed $LINENO +"${LZIP}" -dq -o in < "${in_lz}" +[ $? = 1 ] || test_failed $LINENO # these are for code coverage "${LZIP}" -lt "${in_lz}" 2> /dev/null [ $? = 1 ] || test_failed $LINENO @@ -99,6 +101,11 @@ printf "to be overwritten" > copy || framework_failure "${LZIP}" -df copy.lz { [ $? = 0 ] && [ ! -e copy.lz ] && cmp in copy ; } || test_failed $LINENO +rm -f copy +cat "${in_lz}" > copy.lz || framework_failure +"${LZIP}" -d -S100k copy.lz +{ [ $? = 0 ] && [ ! -e copy.lz ] && cmp in copy ; } || test_failed $LINENO + printf "to be overwritten" > copy || framework_failure "${LZIP}" -df -o copy < "${in_lz}" || test_failed $LINENO cmp in copy || test_failed $LINENO @@ -145,7 +152,7 @@ cat "${in_lz}" "${in_lz}" > in2.lz || framework_failure "${LZIP}" -cd in2.lz > copy2 || test_failed $LINENO cmp in2 copy2 || test_failed $LINENO -"${LZIP}" --output=copy2 < in2 || test_failed $LINENO +"${LZIP}" --output=copy2.lz < in2 || test_failed $LINENO "${LZIP}" -lq copy2.lz || test_failed $LINENO "${LZIP}" -t copy2.lz || test_failed $LINENO "${LZIP}" -cd copy2.lz > copy2 || test_failed $LINENO @@ -204,10 +211,14 @@ for i in s4Ki 0 1 2 3 4 5 6 7 8 9 ; do done cat in in in in in in in in > in8 || framework_failure -"${LZIP}" -1s12 -S100k -o out < in8 || test_failed $LINENO -"${LZIP}" -t out00001.lz out00002.lz || test_failed $LINENO -"${LZIP}" -cd out00001.lz out00002.lz | cmp in8 - || test_failed $LINENO -rm -f out00001.lz +"${LZIP}" -1s12 -S100k in8 || test_failed $LINENO +"${LZIP}" -t in800001.lz in800002.lz || test_failed $LINENO +"${LZIP}" -cd in800001.lz in800002.lz | cmp in8 - || test_failed $LINENO +rm -f in800001.lz in800002.lz +"${LZIP}" -1s12 -S100k -o out.lz < in8 || test_failed $LINENO +"${LZIP}" -t out.lz00001.lz out.lz00002.lz || test_failed $LINENO +"${LZIP}" -cd out.lz00001.lz out.lz00002.lz | cmp in8 - || test_failed $LINENO +rm -f out.lz00001.lz out.lz00002.lz "${LZIP}" -1ks4Ki -b100000 in8 || test_failed $LINENO "${LZIP}" -t in8.lz || test_failed $LINENO "${LZIP}" -cd in8.lz | cmp in8 - || test_failed $LINENO @@ -215,7 +226,15 @@ rm -f in8 "${LZIP}" -0 -S100k -o out < in8.lz || test_failed $LINENO "${LZIP}" -t out00001.lz out00002.lz || test_failed $LINENO "${LZIP}" -cd out00001.lz out00002.lz | cmp in8.lz - || test_failed $LINENO +rm -f out00001.lz +"${LZIP}" -1 -S100k -o out < in8.lz || test_failed $LINENO +"${LZIP}" -t out00001.lz out00002.lz || test_failed $LINENO +"${LZIP}" -cd out00001.lz out00002.lz | cmp in8.lz - || test_failed $LINENO rm -f out00001.lz out00002.lz +"${LZIP}" -0 -F -S100k in8.lz || test_failed $LINENO +"${LZIP}" -t in8.lz00001.lz in8.lz00002.lz || test_failed $LINENO +"${LZIP}" -cd in8.lz00001.lz in8.lz00002.lz | cmp in8.lz - || test_failed $LINENO +rm -f in8.lz00001.lz in8.lz00002.lz "${LZIP}" -0kF -b100k in8.lz || test_failed $LINENO "${LZIP}" -t in8.lz.lz || test_failed $LINENO "${LZIP}" -cd in8.lz.lz | cmp in8.lz - || test_failed $LINENO @@ -223,6 +242,41 @@ rm -f in8.lz in8.lz.lz printf "\ntesting bad input..." +headers='LZIp LZiP LZip LzIP LzIp LziP lZIP lZIp lZiP lzIP' +body='\001\014\000\203\377\373\377\377\300\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000$\000\000\000\000\000\000\000' +cat "${in_lz}" > in0.lz +printf "LZIP${body}" >> in0.lz +if "${LZIP}" -tq in0.lz ; then + for header in ${headers} ; do + printf "${header}${body}" > in0.lz # first member + "${LZIP}" -lq in0.lz + [ $? = 2 ] || test_failed $LINENO ${header} + "${LZIP}" -tq in0.lz + [ $? = 2 ] || test_failed $LINENO ${header} + "${LZIP}" -lq --loose-trailing in0.lz + [ $? = 2 ] || test_failed $LINENO ${header} + "${LZIP}" -tq --loose-trailing in0.lz + [ $? = 2 ] || test_failed $LINENO ${header} + cat "${in_lz}" > in0.lz + printf "${header}${body}" >> in0.lz # trailing data + "${LZIP}" -lq in0.lz + [ $? = 2 ] || test_failed $LINENO ${header} + "${LZIP}" -tq in0.lz + [ $? = 2 ] || test_failed $LINENO ${header} + "${LZIP}" -lq --loose-trailing in0.lz + [ $? = 0 ] || test_failed $LINENO ${header} + "${LZIP}" -t --loose-trailing in0.lz + [ $? = 0 ] || test_failed $LINENO ${header} + "${LZIP}" -lq --loose-trailing --trailing-error in0.lz + [ $? = 2 ] || test_failed $LINENO ${header} + "${LZIP}" -tq --loose-trailing --trailing-error in0.lz + [ $? = 2 ] || test_failed $LINENO ${header} + done +else + printf "\nwarning: skipping header test: 'printf' does not work on your system." +fi +rm -f in0.lz + cat "${in_lz}" "${in_lz}" "${in_lz}" > in3.lz || framework_failure if dd if=in3.lz of=trunc.lz bs=14752 count=1 2> /dev/null && [ -e trunc.lz ] && cmp in2.lz trunc.lz > /dev/null 2>&1 ; then @@ -230,7 +284,7 @@ if dd if=in3.lz of=trunc.lz bs=14752 count=1 2> /dev/null && dd if=in3.lz of=trunc.lz bs=$i count=1 2> /dev/null "${LZIP}" -lq trunc.lz [ $? = 2 ] || test_failed $LINENO $i - "${LZIP}" -t trunc.lz 2> /dev/null + "${LZIP}" -tq trunc.lz [ $? = 2 ] || test_failed $LINENO $i "${LZIP}" -tq < trunc.lz [ $? = 2 ] || test_failed $LINENO $i @@ -242,6 +296,7 @@ if dd if=in3.lz of=trunc.lz bs=14752 count=1 2> /dev/null && else printf "\nwarning: skipping truncation test: 'dd' does not work on your system." fi +rm -f in3.lz trunc.lz cat "${in_lz}" > ingin.lz || framework_failure printf "g" >> ingin.lz || framework_failure @@ -254,6 +309,7 @@ cmp in copy || test_failed $LINENO "${LZIP}" -t < ingin.lz || test_failed $LINENO "${LZIP}" -d < ingin.lz > copy || test_failed $LINENO cmp in copy || test_failed $LINENO +rm -f ingin.lz echo if [ ${fail} = 0 ] ; then |