diff options
| -rw-r--r-- | ChangeLog | 19 | ||||
| -rw-r--r-- | INSTALL | 26 | ||||
| -rw-r--r-- | Makefile.in | 13 | ||||
| -rw-r--r-- | NEWS | 47 | ||||
| -rw-r--r-- | README | 66 | ||||
| -rw-r--r-- | arg_parser.cc | 2 | ||||
| -rw-r--r-- | arg_parser.h | 2 | ||||
| -rw-r--r-- | compress.cc | 28 | ||||
| -rwxr-xr-x | configure | 25 | ||||
| -rw-r--r-- | dec_stdout.cc | 31 | ||||
| -rw-r--r-- | dec_stream.cc | 26 | ||||
| -rw-r--r-- | decompress.cc | 99 | ||||
| -rw-r--r-- | doc/plzip.1 | 30 | ||||
| -rw-r--r-- | doc/plzip.info | 166 | ||||
| -rw-r--r-- | doc/plzip.texi | 134 | ||||
| -rw-r--r-- | list.cc | 34 | ||||
| -rw-r--r-- | lzip.h | 47 | ||||
| -rw-r--r-- | lzip_index.cc (renamed from file_index.cc) | 75 | ||||
| -rw-r--r-- | lzip_index.h (renamed from file_index.h) | 12 | ||||
| -rw-r--r-- | main.cc | 137 | ||||
| -rwxr-xr-x | testsuite/check.sh | 170 |
21 files changed, 729 insertions, 460 deletions
@@ -1,3 +1,18 @@ +2019-01-05 Antonio Diaz Diaz <antonio@gnu.org> + + * Version 1.8 released. + * File_* renamed to Lzip_*. + * main.cc: Added new options '--in-slots' and '--out-slots'. + * main.cc: Increased default in_slots per worker from 2 to 4. + * main.cc: Increased default out_slots per worker from 32 to 64. + * lzip.h (Lzip_trailer): New function 'verify_consistency'. + * lzip_index.cc: Detect some kinds of corrupt trailers. + * main.cc (main): Check return value of close( infd ). + * plzip.texi: Improved description of '-0..-9', '-m' and '-s'. + * configure: Added new option '--with-mingw'. + * configure: Accept appending to CXXFLAGS, 'CXXFLAGS+=OPTIONS'. + * INSTALL: Document use of CXXFLAGS+='-D __USE_MINGW_ANSI_STDIO'. + 2018-02-07 Antonio Diaz Diaz <antonio@gnu.org> * Version 1.7 released. @@ -25,7 +40,7 @@ * The option '-l, --list' has been ported from lziprecover. * Don't allow mixing different operations (-d, -l or -t). * main.cc: Continue testing if any input file is a terminal. - * file_index.cc: Improve detection of bad dict and trailing data. + * lzip_index.cc: Improve detection of bad dict and trailing data. * lzip.h: Unified messages for bad magic, trailing data, etc. 2016-05-14 Antonio Diaz Diaz <antonio@gnu.org> @@ -169,7 +184,7 @@ until something better appears on the net. -Copyright (C) 2009-2018 Antonio Diaz Diaz. +Copyright (C) 2009-2019 Antonio Diaz Diaz. This file is a collection of facts, and thus it is not copyrightable, but just in case, you have unlimited permission to copy, distribute and @@ -1,15 +1,18 @@ Requirements ------------ You will need a C++ compiler and the lzlib compression library installed. -I use gcc 5.3.0 and 4.1.2, but the code should compile with any -standards compliant compiler. -Lzlib must be version 1.0 or newer, but the fast encoder is only -available in lzlib 1.7 or newer, and the HD = 3 detection of corrupt -headers on non-seekable multimember files is only available in lzlib -1.10 or newer. +I use gcc 5.3.0 and 4.1.2, but the code should compile with any standards +compliant compiler. +Lzlib must be version 1.0 or newer, but the fast encoder is only available +in lzlib 1.7 or newer, and the HD = 3 detection of corrupt headers on +non-seekable multimember files is only available in lzlib 1.10 or newer. Gcc is available at http://gcc.gnu.org. Lzlib is available at http://www.nongnu.org/lzip/lzlib.html. +The operating system must allow signal handlers read access to objects with +static storage duration so that the cleanup handler for Control-C can delete +the partial output file. + Procedure --------- @@ -28,6 +31,15 @@ the main archive. cd plzip[version] ./configure + To link against a lzlib not installed in a standard place, use: + + ./configure CPPFLAGS='-I<dir_of_lzlib.h>' LDFLAGS='-L<dir_of_liblz.a>' + + If you are compiling on MinGW, use --with-mingw (note that the Windows + I/O functions used with MinGW are not guaranteed to be thread safe): + + ./configure --with-mingw CXXFLAGS+='-D __USE_MINGW_ANSI_STDIO' + 3. Run make. make @@ -67,7 +79,7 @@ After running 'configure', you can run 'make' and 'make install' as explained above. -Copyright (C) 2009-2018 Antonio Diaz Diaz. +Copyright (C) 2009-2019 Antonio Diaz Diaz. This file is free documentation: you have unlimited permission to copy, distribute and modify it. diff --git a/Makefile.in b/Makefile.in index a9ead0a..6d40fd1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -8,7 +8,7 @@ LIBS = -llz -lpthread SHELL = /bin/sh CAN_RUN_INSTALLINFO = $(SHELL) -c "install-info --version" > /dev/null 2>&1 -objs = arg_parser.o file_index.o list.o compress.o dec_stdout.o \ +objs = arg_parser.o lzip_index.o list.o compress.o dec_stdout.o \ dec_stream.o decompress.o main.o @@ -24,6 +24,9 @@ all : $(progname) $(progname) : $(objs) $(CXX) $(LDFLAGS) $(CXXFLAGS) -o $@ $(objs) $(LIBS) +decompress.o : decompress.cc + $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(with_mingw) -c -o $@ $< + main.o : main.cc $(CXX) $(CPPFLAGS) $(CXXFLAGS) -DPROGVERSION=\"$(pkgversion)\" -c -o $@ $< @@ -33,11 +36,11 @@ main.o : main.cc $(objs) : Makefile arg_parser.o : arg_parser.h compress.o : lzip.h -dec_stdout.o : lzip.h file_index.h +dec_stdout.o : lzip.h lzip_index.h dec_stream.o : lzip.h -decompress.o : lzip.h file_index.h -file_index.o : lzip.h file_index.h -list.o : lzip.h file_index.h +decompress.o : lzip.h lzip_index.h +list.o : lzip.h lzip_index.h +lzip_index.o : lzip.h lzip_index.h main.o : arg_parser.h lzip.h @@ -1,34 +1,31 @@ -Changes in version 1.7: +Changes in version 1.8: -When compressing on a 32 bit system, plzip now tries to limit the memory -use to under 2.22 GiB (4 worker threads at level -9) by reducing the -number of threads below the system's default. +The new options '--in-slots' and '--out-slots', setting the number of input +and output packets buffered during streamed decompression, have been added. +Increasing the number of packets may increase decompression speed, but +requires more memory. -The option '--loose-trailing', has been added. +The default number of input packets buffered per worker thread when +decompressing from non-seekable input has been increased from 2 to 4. -The test used by plzip to discriminate trailing data from a corrupt -header in multimember regular (seekable) files has been improved to a -Hamming distance (HD) of 3, and the 3 bit flips must happen in different -magic bytes for the test to fail. As a consequence some kinds of files -no longer can be appended to a lzip file as trailing data unless the -'--loose-trailing' option is used when decompressing. -Lzlib 1.10 or newer is required for this test to work on non-seekable -files. -Lziprecover can be used to remove conflicting trailing data from a file. +The default number of output packets buffered per worker thread when +decompressing to non-seekable output has been increased from 32 to 64. -The 'bits/byte' ratio has been replaced with the inverse compression -ratio in the output. +Detection of forbidden combinations of characters in trailing data has been +improved. -The progress of decompression is now shown at verbosity level 2 (-vv) or -higher. +Errors are now also checked when closing the input file. -Progress of (de)compression is only shown if stderr is a terminal. +The descriptions of '-0..-9', '-m' and '-s' in the manual have been +improved. -A second '.lz' extension is no longer added to the argument of '-o' if -it already ends in '.lz' or '.tlz'. +The configure script now accepts the option '--with-mingw' to enable the +compilation of plzip under MS Windows (with the MinGW compiler). Use with +care. The Windows I/O functions used are not guaranteed to be thread safe. +(Code based on a patch by Hannes Domani). -The dictionary size is now shown at verbosity level 4 (-vvvv) when -decompressing or testing. +The configure script now accepts appending options to CXXFLAGS using the +syntax 'CXXFLAGS+=OPTIONS'. -The new chapter "Meaning of plzip's output", and a block diagram of -plzip have been added to the manual. +It has been documented in INSTALL the use of +CXXFLAGS+='-D __USE_MINGW_ANSI_STDIO' when compiling on MinGW. @@ -1,8 +1,15 @@ Description -Plzip is a massively parallel (multi-threaded) lossless data compressor -based on the lzlib compression library, with a user interface similar to -the one of lzip, bzip2 or gzip. +Plzip is a massively parallel (multi-threaded) implementation of lzip, fully +compatible with lzip 1.4 or newer. Plzip uses the lzlib compression library. + +Lzip is a lossless data compressor with a user interface similar to the +one of gzip or bzip2. Lzip can compress about as fast as gzip (lzip -0) +or compress most files more than bzip2 (lzip -9). Decompression speed is +intermediate between gzip and bzip2. Lzip is better than gzip and bzip2 +from a data recovery perspective. Lzip has been designed, written and +tested with great care to replace gzip and bzip2 as the standard +general-purpose compressed format for unix-like systems. Plzip can compress/decompress large files on multiprocessor machines much faster than lzip, at the cost of a slightly reduced compression @@ -21,25 +28,21 @@ be decompressed faster than using lzip (unless the '-b' option was used) because lzip usually produces single-member files, which can't be decompressed in parallel. -Plzip uses the lzip file format; the files produced by plzip are fully -compatible with lzip-1.4 or newer, and can be rescued with lziprecover. - -The lzip file format is designed for data sharing and long-term -archiving, taking into account both data integrity and decoder -availability: +The lzip file format is designed for data sharing and long-term archiving, +taking into account both data integrity and decoder availability: * The lzip format provides very safe integrity checking and some data - recovery means. The lziprecover program can repair bit-flip errors + recovery means. The lziprecover program can repair bit flip errors (one of the most common forms of data corruption) in lzip files, and provides data recovery capabilities, including error-checked merging of damaged copies of a file. * The lzip format is as simple as possible (but not simpler). The - lzip manual provides the source code of a simple decompressor along - with a detailed explanation of how it works, so that with the only - help of the lzip manual it would be possible for a digital - archaeologist to extract the data from a lzip file long after - quantum computers eventually render LZMA obsolete. + lzip manual provides the source code of a simple decompressor + along with a detailed explanation of how it works, so that with + the only help of the lzip manual it would be possible for a + digital archaeologist to extract the data from a lzip file long + after quantum computers eventually render LZMA obsolete. * Additionally the lzip reference implementation is copylefted, which guarantees that it will remain free forever. @@ -49,15 +52,14 @@ repair the nearer it is from the beginning of the file. Therefore, with the help of lziprecover, losing an entire archive just because of a corrupt byte near the beginning is a thing of the past. -Plzip uses the same well-defined exit status values used by lzip and -bzip2, which makes it safer than compressors returning ambiguous warning -values (like gzip) when it is used as a back end for other programs like -tar or zutils. +Plzip uses the same well-defined exit status values used by lzip, which +makes it safer than compressors returning ambiguous warning values (like +gzip) when it is used as a back end for other programs like tar or zutils. -Plzip will automatically use the smallest possible dictionary size for -each file without exceeding the given limit. Keep in mind that the -decompression memory requirement is affected at compression time by the -choice of dictionary size limit. +Plzip will automatically use for each file the largest dictionary size +that does not exceed neither the file size nor the limit given. Keep in +mind that the decompression memory requirement is affected at +compression time by the choice of dictionary size limit. When compressing, plzip replaces every file given in the command line with a compressed version of itself, with the name "original_name.lz". @@ -70,25 +72,29 @@ anyothername becomes anyothername.out (De)compressing a file is much like copying or moving it; therefore plzip preserves the access and modification dates, permissions, and, when -possible, ownership of the file just as "cp -p" does. (If the user ID or +possible, ownership of the file just as 'cp -p' does. (If the user ID or the group ID can't be duplicated, the file permission bits S_ISUID and S_ISGID are cleared). Plzip is able to read from some types of non regular files if the -"--stdout" option is specified. +'--stdout' option is specified. If no file names are specified, plzip compresses (or decompresses) from standard input to standard output. In this case, plzip will decline to write compressed output to a terminal, as this would be entirely incomprehensible and therefore pointless. -Plzip will correctly decompress a file which is the concatenation of two -or more compressed files. The result is the concatenation of the -corresponding decompressed files. Integrity testing of concatenated -compressed files is also supported. +Plzip will correctly decompress a file which is the concatenation of two or +more compressed files. The result is the concatenation of the corresponding +decompressed files. Integrity testing of concatenated compressed files is +also supported. + +LANGUAGE NOTE: Uncompressed = not compressed = plain data; it may never +have been compressed. Decompressed is used to refer to data which have +undergone the process of decompression. -Copyright (C) 2009-2018 Antonio Diaz Diaz. +Copyright (C) 2009-2019 Antonio Diaz Diaz. This file is free documentation: you have unlimited permission to copy, distribute and modify it. diff --git a/arg_parser.cc b/arg_parser.cc index 008ebc8..ea32fde 100644 --- a/arg_parser.cc +++ b/arg_parser.cc @@ -1,5 +1,5 @@ /* Arg_parser - POSIX/GNU command line argument parser. (C++ version) - Copyright (C) 2006-2018 Antonio Diaz Diaz. + Copyright (C) 2006-2019 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided diff --git a/arg_parser.h b/arg_parser.h index f015881..ceb9933 100644 --- a/arg_parser.h +++ b/arg_parser.h @@ -1,5 +1,5 @@ /* Arg_parser - POSIX/GNU command line argument parser. (C++ version) - Copyright (C) 2006-2018 Antonio Diaz Diaz. + Copyright (C) 2006-2019 Antonio Diaz Diaz. This library is free software. Redistribution and use in source and binary forms, with or without modification, are permitted provided diff --git a/compress.cc b/compress.cc index beae59e..af36f95 100644 --- a/compress.cc +++ b/compress.cc @@ -1,6 +1,6 @@ -/* Plzip - Parallel compressor compatible with lzip +/* Plzip - Massively parallel implementation of lzip Copyright (C) 2009 Laszlo Ersek. - Copyright (C) 2009-2018 Antonio Diaz Diaz. + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -25,7 +25,6 @@ #include <cstdio> #include <cstdlib> #include <cstring> -#include <queue> #include <string> #include <vector> #include <pthread.h> @@ -196,7 +195,7 @@ public: ocheck_counter( 0 ), owait_counter( 0 ), receive_id( 0 ), distrib_id( 0 ), deliver_id( 0 ), slot_tally( slots ), circular_ibuffer( slots ), - circular_obuffer( slots, (Packet *) 0 ), + circular_obuffer( slots, (const Packet *) 0 ), num_working( workers ), num_slots( slots ), eof( false ) { xinit_mutex( &imutex ); xinit_cond( &iav_or_eof ); @@ -318,7 +317,7 @@ struct Splitter_arg // courier for packaging and distribution to workers. extern "C" void * csplitter( void * arg ) { - const Splitter_arg & tmp = *(Splitter_arg *)arg; + const Splitter_arg & tmp = *(const Splitter_arg *)arg; Packet_courier & courier = *tmp.courier; const Pretty_print & pp = *tmp.pp; const int infd = tmp.infd; @@ -364,7 +363,7 @@ struct Worker_arg // them to courier. extern "C" void * cworker( void * arg ) { - const Worker_arg & tmp = *(Worker_arg *)arg; + const Worker_arg & tmp = *(const Worker_arg *)arg; Packet_courier & courier = *tmp.courier; const Pretty_print & pp = *tmp.pp; const int dictionary_size = tmp.dictionary_size; @@ -401,18 +400,15 @@ extern "C" void * cworker( void * arg ) int new_pos = 0; while( true ) { - if( LZ_compress_write_size( encoder ) > 0 ) + if( written < packet->size ) { - if( written < packet->size ) - { - const int wr = LZ_compress_write( encoder, - packet->data + offset + written, - packet->size - written ); - if( wr < 0 ) internal_error( "library error (LZ_compress_write)." ); - written += wr; - } - if( written >= packet->size ) LZ_compress_finish( encoder ); + const int wr = LZ_compress_write( encoder, + packet->data + offset + written, + packet->size - written ); + if( wr < 0 ) internal_error( "library error (LZ_compress_write)." ); + written += wr; } + if( written >= packet->size ) LZ_compress_finish( encoder ); const int rd = LZ_compress_read( encoder, packet->data + new_pos, offset + written - new_pos ); if( rd < 0 ) @@ -1,13 +1,14 @@ #! /bin/sh -# configure script for Plzip - Parallel compressor compatible with lzip -# Copyright (C) 2009-2018 Antonio Diaz Diaz. +# configure script for Plzip - Massively parallel implementation of lzip +# Copyright (C) 2009-2019 Antonio Diaz Diaz. # # This configure script is free software: you have unlimited permission # to copy, distribute and modify it. pkgname=plzip -pkgversion=1.7 +pkgversion=1.8 progname=plzip +with_mingw= srctrigger=doc/${pkgname}.texi # clear some things potentially inherited from environment. @@ -67,9 +68,11 @@ while [ $# != 0 ] ; do echo " --datarootdir=DIR base directory for doc and data [${datarootdir}]" echo " --infodir=DIR info files directory [${infodir}]" echo " --mandir=DIR man pages directory [${mandir}]" + echo " --with-mingw use included pread/pwrite functions missing in MinGW" echo " CXX=COMPILER C++ compiler to use [${CXX}]" echo " CPPFLAGS=OPTIONS command line options for the preprocessor [${CPPFLAGS}]" echo " CXXFLAGS=OPTIONS command line options for the C++ compiler [${CXXFLAGS}]" + echo " CXXFLAGS+=OPTIONS append options to the current value of CXXFLAGS" echo " LDFLAGS=OPTIONS command line options for the linker [${LDFLAGS}]" echo exit 0 ;; @@ -92,11 +95,13 @@ while [ $# != 0 ] ; do --infodir=*) infodir=${optarg} ;; --mandir=*) mandir=${optarg} ;; --no-create) no_create=yes ;; + --with-mingw) with_mingw="-DWITH_MINGW" ;; - CXX=*) CXX=${optarg} ;; - CPPFLAGS=*) CPPFLAGS=${optarg} ;; - CXXFLAGS=*) CXXFLAGS=${optarg} ;; - LDFLAGS=*) LDFLAGS=${optarg} ;; + CXX=*) CXX=${optarg} ;; + CPPFLAGS=*) CPPFLAGS=${optarg} ;; + CXXFLAGS=*) CXXFLAGS=${optarg} ;; + CXXFLAGS+=*) CXXFLAGS="${CXXFLAGS} ${optarg}" ;; + LDFLAGS=*) LDFLAGS=${optarg} ;; --*) echo "configure: WARNING: unrecognized option: '${option}'" 1>&2 ;; @@ -154,6 +159,7 @@ EOF fi echo "creating Makefile" +if [ -n "${with_mingw}" ] ; then echo "WITH_MINGW = yes" ; fi echo "VPATH = ${srcdir}" echo "prefix = ${prefix}" echo "exec_prefix = ${exec_prefix}" @@ -167,8 +173,8 @@ echo "CXXFLAGS = ${CXXFLAGS}" echo "LDFLAGS = ${LDFLAGS}" rm -f Makefile cat > Makefile << EOF -# Makefile for Plzip - Parallel compressor compatible with lzip -# Copyright (C) 2009-2018 Antonio Diaz Diaz. +# Makefile for Plzip - Massively parallel implementation of lzip +# Copyright (C) 2009-2019 Antonio Diaz Diaz. # This file was generated automatically by configure. Don't edit. # # This Makefile is free software: you have unlimited permission @@ -177,6 +183,7 @@ cat > Makefile << EOF pkgname = ${pkgname} pkgversion = ${pkgversion} progname = ${progname} +with_mingw = ${with_mingw} VPATH = ${srcdir} prefix = ${prefix} exec_prefix = ${exec_prefix} diff --git a/dec_stdout.cc b/dec_stdout.cc index 27b9f31..2a85009 100644 --- a/dec_stdout.cc +++ b/dec_stdout.cc @@ -1,6 +1,6 @@ -/* Plzip - Parallel compressor compatible with lzip +/* Plzip - Massively parallel implementation of lzip Copyright (C) 2009 Laszlo Ersek. - Copyright (C) 2009-2018 Antonio Diaz Diaz. + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -34,7 +34,7 @@ #include <lzlib.h> #include "lzip.h" -#include "file_index.h" +#include "lzip_index.h" namespace { @@ -147,7 +147,7 @@ public: struct Worker_arg { - const File_index * file_index; + const Lzip_index * lzip_index; Packet_courier * courier; const Pretty_print * pp; int worker_id; @@ -160,8 +160,8 @@ struct Worker_arg // give the produced packets to courier. extern "C" void * dworker_o( void * arg ) { - const Worker_arg & tmp = *(Worker_arg *)arg; - const File_index & file_index = *tmp.file_index; + const Worker_arg & tmp = *(const Worker_arg *)arg; + const Lzip_index & lzip_index = *tmp.lzip_index; Packet_courier & courier = *tmp.courier; const Pretty_print & pp = *tmp.pp; const int worker_id = tmp.worker_id; @@ -177,10 +177,10 @@ extern "C" void * dworker_o( void * arg ) { pp( "Not enough memory." ); cleanup_and_fail(); } int new_pos = 0; - for( long i = worker_id; i < file_index.members(); i += num_workers ) + for( long i = worker_id; i < lzip_index.members(); i += num_workers ) { - long long member_pos = file_index.mblock( i ).pos(); - long long member_rest = file_index.mblock( i ).size(); + long long member_pos = lzip_index.mblock( i ).pos(); + long long member_rest = lzip_index.mblock( i ).size(); while( member_rest > 0 ) { @@ -229,7 +229,7 @@ extern "C" void * dworker_o( void * arg ) if( rd == 0 ) break; } } - show_progress( file_index.mblock( i ).size() ); + show_progress( lzip_index.mblock( i ).size() ); } delete[] ibuffer; delete[] new_data; @@ -265,9 +265,8 @@ void muxer( Packet_courier & courier, const Pretty_print & pp, const int outfd ) // init the courier, then start the workers and call the muxer. int dec_stdout( const int num_workers, const int infd, const int outfd, const Pretty_print & pp, const int debug_level, - const File_index & file_index ) + const int out_slots, const Lzip_index & lzip_index ) { - const int out_slots = 32; Packet_courier courier( num_workers, out_slots ); Worker_arg * worker_args = new( std::nothrow ) Worker_arg[num_workers]; @@ -276,7 +275,7 @@ int dec_stdout( const int num_workers, const int infd, const int outfd, { pp( "Not enough memory." ); cleanup_and_fail(); } for( int i = 0; i < num_workers; ++i ) { - worker_args[i].file_index = &file_index; + worker_args[i].lzip_index = &lzip_index; worker_args[i].courier = &courier; worker_args[i].pp = &pp; worker_args[i].worker_id = i; @@ -301,9 +300,9 @@ int dec_stdout( const int num_workers, const int infd, const int outfd, if( verbosity >= 2 ) { - if( verbosity >= 4 ) show_header( file_index.dictionary_size( 0 ) ); - const unsigned long long in_size = file_index.cdata_size(); - const unsigned long long out_size = file_index.udata_size(); + if( verbosity >= 4 ) show_header( lzip_index.dictionary_size( 0 ) ); + const unsigned long long in_size = lzip_index.cdata_size(); + const unsigned long long out_size = lzip_index.udata_size(); if( out_size == 0 || in_size == 0 ) std::fputs( "no data compressed. ", stderr ); else diff --git a/dec_stream.cc b/dec_stream.cc index 36a0ec0..2e1f752 100644 --- a/dec_stream.cc +++ b/dec_stream.cc @@ -1,6 +1,6 @@ -/* Plzip - Parallel compressor compatible with lzip +/* Plzip - Massively parallel implementation of lzip Copyright (C) 2009 Laszlo Ersek. - Copyright (C) 2009-2018 Antonio Diaz Diaz. + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -253,8 +253,8 @@ extern "C" void * dsplitter_s( void * arg ) Packet_courier & courier = *tmp.courier; const Pretty_print & pp = *tmp.pp; const int infd = tmp.infd; - const int hsize = File_header::size; - const int tsize = File_trailer::size; + const int hsize = Lzip_header::size; + const int tsize = Lzip_trailer::size; const int buffer_size = max_packet_size; const int base_buffer_size = tsize + buffer_size + hsize; uint8_t * const base_buffer = new( std::nothrow ) uint8_t[base_buffer_size]; @@ -268,7 +268,7 @@ extern "C" void * dsplitter_s( void * arg ) if( size + hsize < min_member_size ) { show_file_error( pp.name(), "Input file is too short." ); cleanup_and_fail( 2 ); } - const File_header & header = *(File_header *)buffer; + const Lzip_header & header = *(const Lzip_header *)buffer; if( !header.verify_magic() ) { show_file_error( pp.name(), bad_magic_msg ); cleanup_and_fail( 2 ); } if( !header.verify_version() ) @@ -288,11 +288,12 @@ extern "C" void * dsplitter_s( void * arg ) newpos = find_magic( buffer, newpos, size + 4 - newpos ); if( newpos <= size ) { - const File_trailer & trailer = *(File_trailer *)(buffer + newpos - tsize); + const Lzip_trailer & trailer = + *(const Lzip_trailer *)(buffer + newpos - tsize); const unsigned long long member_size = trailer.member_size(); if( partial_member_size + newpos - pos == member_size ) { // header found - const File_header & header = *(File_header *)(buffer + newpos); + const Lzip_header & header = *(const Lzip_header *)(buffer + newpos); if( !header.verify_version() ) { pp( bad_version( header.version() ) ); cleanup_and_fail( 2 ); } const unsigned dictionary_size = header.dictionary_size(); @@ -354,7 +355,7 @@ struct Worker_arg // if not testing, give the produced packets to courier. extern "C" void * dworker_s( void * arg ) { - const Worker_arg & tmp = *(Worker_arg *)arg; + const Worker_arg & tmp = *(const Worker_arg *)arg; Packet_courier & courier = *tmp.courier; const Pretty_print & pp = *tmp.pp; const int worker_id = tmp.worker_id; @@ -479,15 +480,14 @@ void muxer( Packet_courier & courier, const Pretty_print & pp, const int outfd ) int dec_stream( const unsigned long long cfile_size, const int num_workers, const int infd, const int outfd, const Pretty_print & pp, const int debug_level, + const int in_slots, const int out_slots, const bool ignore_trailing, const bool loose_trailing ) { - const int in_slots_per_worker = 2; - const int out_slots = 32; - const int in_slots = ( INT_MAX / num_workers >= in_slots_per_worker ) ? - num_workers * in_slots_per_worker : INT_MAX; + const int total_in_slots = ( INT_MAX / num_workers >= in_slots ) ? + num_workers * in_slots : INT_MAX; in_size = 0; out_size = 0; - Packet_courier courier( num_workers, in_slots, out_slots ); + Packet_courier courier( num_workers, total_in_slots, out_slots ); Splitter_arg splitter_arg; splitter_arg.cfile_size = cfile_size; diff --git a/decompress.cc b/decompress.cc index ed1ac21..19cb1df 100644 --- a/decompress.cc +++ b/decompress.cc @@ -1,6 +1,6 @@ -/* Plzip - Parallel compressor compatible with lzip +/* Plzip - Massively parallel implementation of lzip Copyright (C) 2009 Laszlo Ersek. - Copyright (C) 2009-2018 Antonio Diaz Diaz. + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -34,7 +34,46 @@ #include <lzlib.h> #include "lzip.h" -#include "file_index.h" +#include "lzip_index.h" + + +// This code is based on a patch by Hannes Domani, ssbssa@yahoo.de +// to be able to compile plzip under MS Windows (with MINGW compiler). +#if defined(__MSVCRT__) && defined(WITH_MINGW) +#include <windows.h> +#warning "Parallel I/O is not guaranteed to work on Windows." + +ssize_t pread( int fd, void *buf, size_t count, uint64_t offset ) + { + OVERLAPPED o = {0,0,0,0,0}; + HANDLE fh = (HANDLE)_get_osfhandle(fd); + DWORD bytes; + BOOL ret; + + if( fh == INVALID_HANDLE_VALUE ) { errno = EBADF; return -1; } + o.Offset = offset & 0xffffffff; + o.OffsetHigh = (offset >> 32) & 0xffffffff; + ret = ReadFile( fh, buf, (DWORD)count, &bytes, &o ); + if( !ret ) { errno = EIO; return -1; } + return (ssize_t)bytes; + } + +ssize_t pwrite( int fd, const void *buf, size_t count, uint64_t offset ) + { + OVERLAPPED o = {0,0,0,0,0}; + HANDLE fh = (HANDLE)_get_osfhandle(fd); + DWORD bytes; + BOOL ret; + + if( fh == INVALID_HANDLE_VALUE ) { errno = EBADF; return -1; } + o.Offset = offset & 0xffffffff; + o.OffsetHigh = (offset >> 32) & 0xffffffff; + ret = WriteFile(fh, buf, (DWORD)count, &bytes, &o); + if( !ret ) { errno = EIO; return -1; } + return (ssize_t)bytes; + } + +#endif // __MSVCRT__ // Returns the number of bytes really read. @@ -95,7 +134,7 @@ namespace { struct Worker_arg { - const File_index * file_index; + const Lzip_index * lzip_index; const Pretty_print * pp; int worker_id; int num_workers; @@ -108,8 +147,8 @@ struct Worker_arg // write the produced data to file. extern "C" void * dworker( void * arg ) { - const Worker_arg & tmp = *(Worker_arg *)arg; - const File_index & file_index = *tmp.file_index; + const Worker_arg & tmp = *(const Worker_arg *)arg; + const Lzip_index & lzip_index = *tmp.lzip_index; const Pretty_print & pp = *tmp.pp; const int worker_id = tmp.worker_id; const int num_workers = tmp.num_workers; @@ -124,12 +163,12 @@ extern "C" void * dworker( void * arg ) LZ_decompress_errno( decoder ) != LZ_ok ) { pp( "Not enough memory." ); cleanup_and_fail(); } - for( long i = worker_id; i < file_index.members(); i += num_workers ) + for( long i = worker_id; i < lzip_index.members(); i += num_workers ) { - long long data_pos = file_index.dblock( i ).pos(); - long long data_rest = file_index.dblock( i ).size(); - long long member_pos = file_index.mblock( i ).pos(); - long long member_rest = file_index.mblock( i ).size(); + long long data_pos = lzip_index.dblock( i ).pos(); + long long data_rest = lzip_index.dblock( i ).size(); + long long member_pos = lzip_index.mblock( i ).pos(); + long long member_rest = lzip_index.mblock( i ).size(); while( member_rest > 0 ) { @@ -180,7 +219,7 @@ extern "C" void * dworker( void * arg ) if( rd == 0 ) break; } } - show_progress( file_index.mblock( i ).size() ); + show_progress( lzip_index.mblock( i ).size() ); } delete[] obuffer; delete[] ibuffer; @@ -197,26 +236,27 @@ extern "C" void * dworker( void * arg ) // start the workers and wait for them to finish. int decompress( const unsigned long long cfile_size, int num_workers, const int infd, const int outfd, const Pretty_print & pp, - const int debug_level, const bool ignore_trailing, + const int debug_level, const int in_slots, + const int out_slots, const bool ignore_trailing, const bool loose_trailing, const bool infd_isreg ) { if( !infd_isreg ) - return dec_stream( cfile_size, num_workers, infd, outfd, pp, - debug_level, ignore_trailing, loose_trailing ); + return dec_stream( cfile_size, num_workers, infd, outfd, pp, debug_level, + in_slots, out_slots, ignore_trailing, loose_trailing ); - const File_index file_index( infd, ignore_trailing, loose_trailing ); - if( file_index.retval() == 1 ) + const Lzip_index lzip_index( infd, ignore_trailing, loose_trailing ); + if( lzip_index.retval() == 1 ) { lseek( infd, 0, SEEK_SET ); - return dec_stream( cfile_size, num_workers, infd, outfd, pp, - debug_level, ignore_trailing, loose_trailing ); + return dec_stream( cfile_size, num_workers, infd, outfd, pp, debug_level, + in_slots, out_slots, ignore_trailing, loose_trailing ); } - if( file_index.retval() != 0 ) - { show_file_error( pp.name(), file_index.error().c_str() ); - return file_index.retval(); } + if( lzip_index.retval() != 0 ) + { show_file_error( pp.name(), lzip_index.error().c_str() ); + return lzip_index.retval(); } - if( num_workers > file_index.members() ) - num_workers = file_index.members(); + if( num_workers > lzip_index.members() ) + num_workers = lzip_index.members(); if( verbosity >= 1 ) pp(); show_progress( 0, cfile_size, &pp ); // init @@ -225,7 +265,8 @@ int decompress( const unsigned long long cfile_size, int num_workers, struct stat st; if( fstat( outfd, &st ) != 0 || !S_ISREG( st.st_mode ) || lseek( outfd, 0, SEEK_CUR ) < 0 ) - return dec_stdout( num_workers, infd, outfd, pp, debug_level, file_index ); + return dec_stdout( num_workers, infd, outfd, pp, debug_level, out_slots, + lzip_index ); } Worker_arg * worker_args = new( std::nothrow ) Worker_arg[num_workers]; @@ -234,7 +275,7 @@ int decompress( const unsigned long long cfile_size, int num_workers, { pp( "Not enough memory." ); cleanup_and_fail(); } for( int i = 0; i < num_workers; ++i ) { - worker_args[i].file_index = &file_index; + worker_args[i].lzip_index = &lzip_index; worker_args[i].pp = &pp; worker_args[i].worker_id = i; worker_args[i].num_workers = num_workers; @@ -257,9 +298,9 @@ int decompress( const unsigned long long cfile_size, int num_workers, if( verbosity >= 2 ) { - if( verbosity >= 4 ) show_header( file_index.dictionary_size( 0 ) ); - const unsigned long long in_size = file_index.cdata_size(); - const unsigned long long out_size = file_index.udata_size(); + if( verbosity >= 4 ) show_header( lzip_index.dictionary_size( 0 ) ); + const unsigned long long in_size = lzip_index.cdata_size(); + const unsigned long long out_size = lzip_index.udata_size(); if( out_size == 0 || in_size == 0 ) std::fputs( "no data compressed. ", stderr ); else diff --git a/doc/plzip.1 b/doc/plzip.1 index 99dfd8b..694a99d 100644 --- a/doc/plzip.1 +++ b/doc/plzip.1 @@ -1,12 +1,28 @@ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.1. -.TH PLZIP "1" "February 2018" "plzip 1.7" "User Commands" +.TH PLZIP "1" "January 2019" "plzip 1.8" "User Commands" .SH NAME plzip \- reduces the size of files .SH SYNOPSIS .B plzip [\fI\,options\/\fR] [\fI\,files\/\fR] .SH DESCRIPTION -Plzip \- Parallel compressor compatible with lzip. +Plzip is a massively parallel (multi\-threaded) implementation of lzip, fully +compatible with lzip 1.4 or newer. Plzip uses the lzlib compression library. +.PP +Lzip is a lossless data compressor with a user interface similar to the +one of gzip or bzip2. Lzip can compress about as fast as gzip (lzip \fB\-0\fR) +or compress most files more than bzip2 (lzip \fB\-9\fR). Decompression speed is +intermediate between gzip and bzip2. Lzip is better than gzip and bzip2 +from a data recovery perspective. Lzip has been designed, written and +tested with great care to replace gzip and bzip2 as the standard +general\-purpose compressed format for unix\-like systems. +.PP +Plzip can compress/decompress large files on multiprocessor machines +much faster than lzip, at the cost of a slightly reduced compression +ratio (0.4 to 2 percent larger compressed files). Note that the number +of usable threads is limited by file size; on files larger than a few GB +plzip can use hundreds of processors, but on files of only a few MB +plzip is no faster than lzip. .SH OPTIONS .TP \fB\-h\fR, \fB\-\-help\fR @@ -71,6 +87,12 @@ alias for \fB\-9\fR .TP \fB\-\-loose\-trailing\fR allow trailing data seeming corrupt header +.TP +\fB\-\-in\-slots=\fR<n> +number of 1 MiB input packets buffered [4] +.TP +\fB\-\-out\-slots=\fR<n> +number of 1 MiB output packets buffered [64] .PP If no file names are given, or if a file is '\-', plzip compresses or decompresses from standard input to standard output. @@ -95,8 +117,8 @@ Plzip home page: http://www.nongnu.org/lzip/plzip.html .SH COPYRIGHT Copyright \(co 2009 Laszlo Ersek. .br -Copyright \(co 2018 Antonio Diaz Diaz. -Using lzlib 1.10 +Copyright \(co 2019 Antonio Diaz Diaz. +Using lzlib 1.11 License GPLv2+: GNU GPL version 2 or later <http://gnu.org/licenses/gpl.html> .br This is free software: you are free to change and redistribute it. diff --git a/doc/plzip.info b/doc/plzip.info index c8d7387..2b7aa52 100644 --- a/doc/plzip.info +++ b/doc/plzip.info @@ -2,7 +2,7 @@ This is plzip.info, produced by makeinfo version 4.13+ from plzip.texi. INFO-DIR-SECTION Data Compression START-INFO-DIR-ENTRY -* Plzip: (plzip). Parallel compressor compatible with lzip +* Plzip: (plzip). Massively parallel implementation of lzip END-INFO-DIR-ENTRY @@ -11,7 +11,7 @@ File: plzip.info, Node: Top, Next: Introduction, Up: (dir) Plzip Manual ************ -This manual is for Plzip (version 1.7, 7 February 2018). +This manual is for Plzip (version 1.8, 5 January 2019). * Menu: @@ -28,7 +28,7 @@ This manual is for Plzip (version 1.7, 7 February 2018). * Concept index:: Index of concepts - Copyright (C) 2009-2018 Antonio Diaz Diaz. + Copyright (C) 2009-2019 Antonio Diaz Diaz. This manual is free documentation: you have unlimited permission to copy, distribute and modify it. @@ -39,20 +39,25 @@ File: plzip.info, Node: Introduction, Next: Output, Prev: Top, Up: Top 1 Introduction ************** -Plzip is a massively parallel (multi-threaded) lossless data compressor -based on the lzlib compression library, with a user interface similar to -the one of lzip, bzip2 or gzip. +Plzip is a massively parallel (multi-threaded) implementation of lzip, +fully compatible with lzip 1.4 or newer. Plzip uses the lzlib +compression library. + + Lzip is a lossless data compressor with a user interface similar to +the one of gzip or bzip2. Lzip can compress about as fast as gzip +(lzip -0) or compress most files more than bzip2 (lzip -9). +Decompression speed is intermediate between gzip and bzip2. Lzip is +better than gzip and bzip2 from a data recovery perspective. Lzip has +been designed, written and tested with great care to replace gzip and +bzip2 as the standard general-purpose compressed format for unix-like +systems. Plzip can compress/decompress large files on multiprocessor machines much faster than lzip, at the cost of a slightly reduced compression ratio (0.4 to 2 percent larger compressed files). Note that the number of usable threads is limited by file size; on files larger than a few GB plzip can use hundreds of processors, but on files of only a few MB -plzip is no faster than lzip (*note Minimum file sizes::). - - Plzip uses the lzip file format; the files produced by plzip are -fully compatible with lzip-1.4 or newer, and can be rescued with -lziprecover. +plzip is no faster than lzip. *Note Minimum file sizes::. The lzip file format is designed for data sharing and long-term archiving, taking into account both data integrity and decoder @@ -80,15 +85,16 @@ repair the nearer it is from the beginning of the file. Therefore, with the help of lziprecover, losing an entire archive just because of a corrupt byte near the beginning is a thing of the past. - Plzip uses the same well-defined exit status values used by lzip and -bzip2, which makes it safer than compressors returning ambiguous warning -values (like gzip) when it is used as a back end for other programs like -tar or zutils. + Plzip uses the same well-defined exit status values used by lzip, +which makes it safer than compressors returning ambiguous warning +values (like gzip) when it is used as a back end for other programs +like tar or zutils. - Plzip will automatically use the smallest possible dictionary size -for each file without exceeding the given limit. Keep in mind that the -decompression memory requirement is affected at compression time by the -choice of dictionary size limit (*note Memory requirements::). + Plzip will automatically use for each file the largest dictionary +size that does not exceed neither the file size nor the limit given. +Keep in mind that the decompression memory requirement is affected at +compression time by the choice of dictionary size limit. *Note Memory +requirements::. When compressing, plzip replaces every file given in the command line with a compressed version of itself, with the name "original_name.lz". @@ -101,7 +107,7 @@ anyothername becomes anyothername.out (De)compressing a file is much like copying or moving it; therefore plzip preserves the access and modification dates, permissions, and, -when possible, ownership of the file just as "cp -p" does. (If the user +when possible, ownership of the file just as 'cp -p' does. (If the user ID or the group ID can't be duplicated, the file permission bits S_ISUID and S_ISGID are cleared). @@ -188,6 +194,7 @@ command line. '-V' '--version' Print the version number of plzip on the standard output and exit. + This version number should be included in all bug reports. '-a' '--trailing-error' @@ -286,12 +293,14 @@ command line. '-s BYTES' '--dictionary-size=BYTES' When compressing, set the dictionary size limit in bytes. Plzip - will use the smallest possible dictionary size for each file - without exceeding this limit. Valid values range from 4 KiB to - 512 MiB. Values 12 to 29 are interpreted as powers of two, meaning - 2^12 to 2^29 bytes. Note that dictionary sizes are quantized. If - the specified size does not match one of the valid sizes, it will - be rounded upwards by adding up to (BYTES / 8) to it. + will use for each file the largest dictionary size that does not + exceed neither the file size nor this limit. Valid values range + from 4 KiB to 512 MiB. Values 12 to 29 are interpreted as powers + of two, meaning 2^12 to 2^29 bytes. Dictionary sizes are quantized + so that they can be coded in just one byte (*note + coded-dict-size::). If the specified size does not match one of + the valid sizes, it will be rounded upwards by adding up to + (BYTES / 8) to it. For maximum compression you should use a dictionary size limit as large as possible, but keep in mind that the decompression memory @@ -320,27 +329,32 @@ command line. except for single-member files. '-0 .. -9' - Set the compression parameters (dictionary size and match length - limit) as shown in the table below. The default compression level - is '-6'. Note that '-9' can be much slower than '-0'. These - options have no effect when decompressing, testing or listing. + Compression level. Set the compression parameters (dictionary size + and match length limit) as shown in the table below. The default + compression level is '-6', equivalent to '-s8MiB -m36'. Note that + '-9' can be much slower than '-0'. These options have no effect + when decompressing, testing or listing. The bidimensional parameter space of LZMA can't be mapped to a linear scale optimal for all files. If your files are large, very repetitive, etc, you may need to use the '--dictionary-size' and '--match-length' options directly to achieve optimal performance. - Level Dictionary size Match length limit - -0 64 KiB 16 bytes - -1 1 MiB 5 bytes - -2 1.5 MiB 6 bytes - -3 2 MiB 8 bytes - -4 3 MiB 12 bytes - -5 4 MiB 20 bytes - -6 8 MiB 36 bytes - -7 16 MiB 68 bytes - -8 24 MiB 132 bytes - -9 32 MiB 273 bytes + If several compression levels or '-s' or '-m' options are given, + the last setting is used. For example '-9 -s64MiB' is equivalent + to '-s64MiB -m273' + + Level Dictionary size (-s) Match length limit (-m) + -0 64 KiB 16 bytes + -1 1 MiB 5 bytes + -2 1.5 MiB 6 bytes + -3 2 MiB 8 bytes + -4 3 MiB 12 bytes + -5 4 MiB 20 bytes + -6 8 MiB 36 bytes + -7 16 MiB 68 bytes + -8 24 MiB 132 bytes + -9 32 MiB 273 bytes '--fast' '--best' @@ -353,6 +367,18 @@ command line. if a file triggers a "corrupt header" error and the cause is not indeed a corrupt header. +'--in-slots=N' + Number of 1 MiB input packets buffered per worker thread when + decompressing from non-seekable input. Increasing the number of + packets may increase decompression speed, but requires more + memory. Valid values range from 1 to 64. The default value is 4. + +'--out-slots=N' + Number of 1 MiB output packets buffered per worker thread when + decompressing to non-seekable output. Increasing the number of + packets may increase decompression speed, but requires more + memory. Valid values range from 1 to 1024. The default value is 64. + Numbers given as arguments to options may be followed by a multiplier and an optional 'B' for "byte". @@ -465,11 +491,11 @@ additional information before, between, or after them. 'DS (coded dictionary size, 1 byte)' The dictionary size is calculated by taking a power of 2 (the base - size) and substracting from it a fraction between 0/16 and 7/16 of + size) and subtracting from it a fraction between 0/16 and 7/16 of the base size. Bits 4-0 contain the base 2 logarithm of the base size (12 to 29). - Bits 7-5 contain the numerator of the fraction (0 to 7) to - substract from the base size to obtain the dictionary size. + Bits 7-5 contain the numerator of the fraction (0 to 7) to subtract + from the base size to obtain the dictionary size. Example: 0xD3 = 2^19 - 6 * 2^15 = 512 KiB - 6 * 32 KiB = 320 KiB Valid values for dictionary size range from 4 KiB to 512 MiB. @@ -497,22 +523,25 @@ File: plzip.info, Node: Memory requirements, Next: Minimum file sizes, Prev: 6 Memory required to compress and decompress ******************************************** -The amount of memory required *per thread* for decompression or testing -is approximately the following: +The amount of memory required *per worker thread* for decompression or +testing is approximately the following: * For decompression of a regular (seekable) file to another regular file, or for testing of a regular file; the dictionary size. * For testing of a non-seekable file or of standard input; the - dictionary size plus up to 5 MiB. + dictionary size plus 1 MiB plus up to the number of 1 MiB input + packets buffered (4 by default). * For decompression of a regular file to a non-seekable file or to - standard output; the dictionary size plus up to 32 MiB. + standard output; the dictionary size plus up to the number of 1 MiB + output packets buffered (64 by default). * For decompression of a non-seekable file or of standard input; the - dictionary size plus up to 35 MiB. + dictionary size plus 1 MiB plus up to the number of 1 MiB input + and output packets buffered (68 by default). -The amount of memory required *per thread* for compression is +The amount of memory required *per worker thread* for compression is approximately the following: * For compression at level -0; 1.5 MiB plus 3.375 times the data size @@ -561,7 +590,7 @@ for full use of N processors at a given compression level, using the default data size for each level: Processors 2 4 8 16 64 256 -------------------------------------------------------------------------- +------------------------------------------------------------------ Level -0 2 MiB 4 MiB 8 MiB 16 MiB 64 MiB 256 MiB -1 4 MiB 8 MiB 16 MiB 32 MiB 128 MiB 512 MiB @@ -633,7 +662,11 @@ compressed file (bugs in the system libraries, memory errors, etc). Therefore, if the data you are going to compress are important, give the '--keep' option to plzip and don't remove the original file until you verify the compressed file with a command like -'plzip -cd file.lz | cmp file -'. +'plzip -cd file.lz | cmp file -'. Most RAM errors happening during +compression can only be detected by comparing the compressed file with +the original because the corruption happens before plzip compresses the +RAM contents, resulting in a valid compressed file containing wrong +data. Example 1: Replace a regular file with its compressed version 'file.lz' @@ -728,21 +761,22 @@ Concept index Tag Table: -Node: Top221 +Node: Top222 Node: Introduction1158 -Node: Output5134 -Node: Invoking plzip6614 -Ref: --trailing-error7177 -Ref: --data-size7420 -Node: Program design14938 -Node: File format17090 -Node: Memory requirements19522 -Node: Minimum file sizes20985 -Node: Trailing data23002 -Node: Examples25285 -Ref: concat-example26450 -Node: Problems27025 -Node: Concept index27553 +Node: Output5456 +Node: Invoking plzip6936 +Ref: --trailing-error7563 +Ref: --data-size7806 +Node: Program design16267 +Node: File format18419 +Ref: coded-dict-size19719 +Node: Memory requirements20849 +Node: Minimum file sizes22531 +Node: Trailing data24540 +Node: Examples26823 +Ref: concat-example28238 +Node: Problems28813 +Node: Concept index29341 End Tag Table diff --git a/doc/plzip.texi b/doc/plzip.texi index 44cff75..b5469b9 100644 --- a/doc/plzip.texi +++ b/doc/plzip.texi @@ -6,19 +6,19 @@ @finalout @c %**end of header -@set UPDATED 7 February 2018 -@set VERSION 1.7 +@set UPDATED 5 January 2019 +@set VERSION 1.8 @dircategory Data Compression @direntry -* Plzip: (plzip). Parallel compressor compatible with lzip +* Plzip: (plzip). Massively parallel implementation of lzip @end direntry @ifnothtml @titlepage @title Plzip -@subtitle Parallel compressor compatible with lzip +@subtitle Massively parallel implementation of lzip @subtitle for Plzip version @value{VERSION}, @value{UPDATED} @author by Antonio Diaz Diaz @@ -49,7 +49,7 @@ This manual is for Plzip (version @value{VERSION}, @value{UPDATED}). @end menu @sp 1 -Copyright @copyright{} 2009-2018 Antonio Diaz Diaz. +Copyright @copyright{} 2009-2019 Antonio Diaz Diaz. This manual is free documentation: you have unlimited permission to copy, distribute and modify it. @@ -59,23 +59,28 @@ to copy, distribute and modify it. @chapter Introduction @cindex introduction -Plzip is a massively parallel (multi-threaded) lossless data compressor -based on the lzlib compression library, with a user interface similar to -the one of lzip, bzip2 or gzip. +@uref{http://www.nongnu.org/lzip/plzip.html,,Plzip} is a massively parallel +(multi-threaded) implementation of lzip, fully compatible with lzip 1.4 or +newer. Plzip uses the lzlib compression library. + +@uref{http://www.nongnu.org/lzip/lzip.html,,Lzip} is a lossless data +compressor with a user interface similar to the one of gzip or bzip2. Lzip +can compress about as fast as gzip @w{(lzip -0)} or compress most files more +than bzip2 @w{(lzip -9)}. Decompression speed is intermediate between gzip +and bzip2. Lzip is better than gzip and bzip2 from a data recovery +perspective. Lzip has been designed, written and tested with great care to +replace gzip and bzip2 as the standard general-purpose compressed format for +unix-like systems. Plzip can compress/decompress large files on multiprocessor machines much faster than lzip, at the cost of a slightly reduced compression ratio (0.4 to 2 percent larger compressed files). Note that the number of usable threads is limited by file size; on files larger than a few GB plzip can use hundreds of processors, but on files of only a few MB -plzip is no faster than lzip (@pxref{Minimum file sizes}). - -Plzip uses the lzip file format; the files produced by plzip are fully -compatible with lzip-1.4 or newer, and can be rescued with lziprecover. +plzip is no faster than lzip. @xref{Minimum file sizes}. -The lzip file format is designed for data sharing and long-term -archiving, taking into account both data integrity and decoder -availability: +The lzip file format is designed for data sharing and long-term archiving, +taking into account both data integrity and decoder availability: @itemize @bullet @item @@ -107,15 +112,14 @@ repair the nearer it is from the beginning of the file. Therefore, with the help of lziprecover, losing an entire archive just because of a corrupt byte near the beginning is a thing of the past. -Plzip uses the same well-defined exit status values used by lzip and -bzip2, which makes it safer than compressors returning ambiguous warning -values (like gzip) when it is used as a back end for other programs like -tar or zutils. +Plzip uses the same well-defined exit status values used by lzip, which +makes it safer than compressors returning ambiguous warning values (like +gzip) when it is used as a back end for other programs like tar or zutils. -Plzip will automatically use the smallest possible dictionary size for -each file without exceeding the given limit. Keep in mind that the -decompression memory requirement is affected at compression time by the -choice of dictionary size limit (@pxref{Memory requirements}). +Plzip will automatically use for each file the largest dictionary size that +does not exceed neither the file size nor the limit given. Keep in mind that +the decompression memory requirement is affected at compression time by the +choice of dictionary size limit. @xref{Memory requirements}. When compressing, plzip replaces every file given in the command line with a compressed version of itself, with the name "original_name.lz". @@ -130,7 +134,7 @@ file from that of the compressed file as follows: (De)compressing a file is much like copying or moving it; therefore plzip preserves the access and modification dates, permissions, and, when -possible, ownership of the file just as "cp -p" does. (If the user ID or +possible, ownership of the file just as @samp{cp -p} does. (If the user ID or the group ID can't be duplicated, the file permission bits S_ISUID and S_ISGID are cleared). @@ -142,10 +146,10 @@ standard input to standard output. In this case, plzip will decline to write compressed output to a terminal, as this would be entirely incomprehensible and therefore pointless. -Plzip will correctly decompress a file which is the concatenation of two -or more compressed files. The result is the concatenation of the -corresponding decompressed files. Integrity testing of concatenated -compressed files is also supported. +Plzip will correctly decompress a file which is the concatenation of two or +more compressed files. The result is the concatenation of the corresponding +decompressed files. Integrity testing of concatenated compressed files is +also supported. @node Output @@ -225,6 +229,7 @@ Print an informative help message describing the options and exit. @item -V @itemx --version Print the version number of plzip on the standard output and exit. +This version number should be included in all bug reports. @anchor{--trailing-error} @item -a @@ -322,12 +327,13 @@ Quiet operation. Suppress all messages. @item -s @var{bytes} @itemx --dictionary-size=@var{bytes} When compressing, set the dictionary size limit in bytes. Plzip will use -the smallest possible dictionary size for each file without exceeding -this limit. Valid values range from @w{4 KiB} to @w{512 MiB}. Values 12 -to 29 are interpreted as powers of two, meaning 2^12 to 2^29 bytes. Note -that dictionary sizes are quantized. If the specified size does not -match one of the valid sizes, it will be rounded upwards by adding up to -@w{(@var{bytes} / 8)} to it. +for each file the largest dictionary size that does not exceed neither +the file size nor this limit. Valid values range from @w{4 KiB} to +@w{512 MiB}. Values 12 to 29 are interpreted as powers of two, meaning +2^12 to 2^29 bytes. Dictionary sizes are quantized so that they can be +coded in just one byte (@pxref{coded-dict-size}). If the specified size +does not match one of the valid sizes, it will be rounded upwards by +adding up to @w{(@var{bytes} / 8)} to it. For maximum compression you should use a dictionary size limit as large as possible, but keep in mind that the decompression memory requirement @@ -354,18 +360,23 @@ Two or more @samp{-v} options show the progress of (de)compression, except for single-member files. @item -0 .. -9 -Set the compression parameters (dictionary size and match length limit) -as shown in the table below. The default compression level is @samp{-6}. -Note that @samp{-9} can be much slower than @samp{-0}. These options -have no effect when decompressing, testing or listing. +Compression level. Set the compression parameters (dictionary size and +match length limit) as shown in the table below. The default compression +level is @samp{-6}, equivalent to @w{@samp{-s8MiB -m36}}. Note that +@samp{-9} can be much slower than @samp{-0}. These options have no +effect when decompressing, testing or listing. The bidimensional parameter space of LZMA can't be mapped to a linear scale optimal for all files. If your files are large, very repetitive, etc, you may need to use the @samp{--dictionary-size} and @samp{--match-length} options directly to achieve optimal performance. -@multitable {Level} {Dictionary size} {Match length limit} -@item Level @tab Dictionary size @tab Match length limit +If several compression levels or @samp{-s} or @samp{-m} options are +given, the last setting is used. For example @w{@samp{-9 -s64MiB}} is +equivalent to @w{@samp{-s64MiB -m273}} + +@multitable {Level} {Dictionary size (-s)} {Match length limit (-m)} +@item Level @tab Dictionary size (-s) @tab Match length limit (-m) @item -0 @tab 64 KiB @tab 16 bytes @item -1 @tab 1 MiB @tab 5 bytes @item -2 @tab 1.5 MiB @tab 6 bytes @@ -388,6 +399,18 @@ bytes are so similar to the magic bytes of a lzip header that they can be confused with a corrupt header. Use this option if a file triggers a "corrupt header" error and the cause is not indeed a corrupt header. +@item --in-slots=@var{n} +Number of @w{1 MiB} input packets buffered per worker thread when +decompressing from non-seekable input. Increasing the number of packets +may increase decompression speed, but requires more memory. Valid values +range from 1 to 64. The default value is 4. + +@item --out-slots=@var{n} +Number of @w{1 MiB} output packets buffered per worker thread when +decompressing to non-seekable output. Increasing the number of packets +may increase decompression speed, but requires more memory. Valid values +range from 1 to 1024. The default value is 64. + @end table Numbers given as arguments to options may be followed by a multiplier @@ -506,12 +529,13 @@ A four byte string, identifying the lzip format, with the value "LZIP" @item VN (version number, 1 byte) Just in case something needs to be modified in the future. 1 for now. +@anchor{coded-dict-size} @item DS (coded dictionary size, 1 byte) The dictionary size is calculated by taking a power of 2 (the base size) -and substracting from it a fraction between 0/16 and 7/16 of the base +and subtracting from it a fraction between 0/16 and 7/16 of the base size.@* Bits 4-0 contain the base 2 logarithm of the base size (12 to 29).@* -Bits 7-5 contain the numerator of the fraction (0 to 7) to substract +Bits 7-5 contain the numerator of the fraction (0 to 7) to subtract from the base size to obtain the dictionary size.@* Example: 0xD3 = 2^19 - 6 * 2^15 = 512 KiB - 6 * 32 KiB = 320 KiB@* Valid values for dictionary size range from 4 KiB to 512 MiB. @@ -546,8 +570,8 @@ facilitates safe recovery of undamaged members from multimember files. @chapter Memory required to compress and decompress @cindex memory requirements -The amount of memory required @strong{per thread} for decompression or -testing is approximately the following: +The amount of memory required @strong{per worker thread} for +decompression or testing is approximately the following: @itemize @bullet @item @@ -556,20 +580,23 @@ or for testing of a regular file; the dictionary size. @item For testing of a non-seekable file or of standard input; the dictionary -size plus up to @w{5 MiB}. +size plus @w{1 MiB} plus up to the number of @w{1 MiB} input packets +buffered (4 by default). @item For decompression of a regular file to a non-seekable file or to -standard output; the dictionary size plus up to @w{32 MiB}. +standard output; the dictionary size plus up to the number of @w{1 MiB} +output packets buffered (64 by default). @item For decompression of a non-seekable file or of standard input; the -dictionary size plus up to @w{35 MiB}. +dictionary size plus @w{1 MiB} plus up to the number of @w{1 MiB} input +and output packets buffered (68 by default). @end itemize @noindent -The amount of memory required @strong{per thread} for compression is -approximately the following: +The amount of memory required @strong{per worker thread} for compression +is approximately the following: @itemize @bullet @item @@ -696,9 +723,12 @@ where a file containing trailing data must be rejected, the option WARNING! Even if plzip is bug-free, other causes may result in a corrupt compressed file (bugs in the system libraries, memory errors, etc). Therefore, if the data you are going to compress are important, give the -@samp{--keep} option to plzip and don't remove the original file until -you verify the compressed file with a command like -@w{@samp{plzip -cd file.lz | cmp file -}}. +@samp{--keep} option to plzip and don't remove the original file until you +verify the compressed file with a command like +@w{@samp{plzip -cd file.lz | cmp file -}}. Most RAM errors happening during +compression can only be detected by comparing the compressed file with the +original because the corruption happens before plzip compresses the RAM +contents, resulting in a valid compressed file containing wrong data. @sp 1 @noindent @@ -1,5 +1,5 @@ -/* Plzip - Parallel compressor compatible with lzip - Copyright (C) 2009-2018 Antonio Diaz Diaz. +/* Plzip - Massively parallel implementation of lzip + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ #include <sys/stat.h> #include "lzip.h" -#include "file_index.h" +#include "lzip_index.h" namespace { @@ -66,18 +66,18 @@ int list_files( const std::vector< std::string > & filenames, open_instream( input_filename, &in_stats, true, true ); if( infd < 0 ) { if( retval < 1 ) retval = 1; continue; } - const File_index file_index( infd, ignore_trailing, loose_trailing ); + const Lzip_index lzip_index( infd, ignore_trailing, loose_trailing ); close( infd ); - if( file_index.retval() != 0 ) + if( lzip_index.retval() != 0 ) { - show_file_error( input_filename, file_index.error().c_str() ); - if( retval < file_index.retval() ) retval = file_index.retval(); + show_file_error( input_filename, lzip_index.error().c_str() ); + if( retval < lzip_index.retval() ) retval = lzip_index.retval(); continue; } if( verbosity >= 0 ) { - const unsigned long long udata_size = file_index.udata_size(); - const unsigned long long cdata_size = file_index.cdata_size(); + const unsigned long long udata_size = lzip_index.udata_size(); + const unsigned long long cdata_size = lzip_index.cdata_size(); total_comp += cdata_size; total_uncomp += udata_size; ++files; if( first_post ) { @@ -88,22 +88,22 @@ int list_files( const std::vector< std::string > & filenames, if( verbosity >= 1 ) { unsigned dictionary_size = 0; - for( long i = 0; i < file_index.members(); ++i ) + for( long i = 0; i < lzip_index.members(); ++i ) dictionary_size = - std::max( dictionary_size, file_index.dictionary_size( i ) ); - const long long trailing_size = file_index.file_size() - cdata_size; + std::max( dictionary_size, lzip_index.dictionary_size( i ) ); + const long long trailing_size = lzip_index.file_size() - cdata_size; std::printf( "%s %5ld %6lld ", format_ds( dictionary_size ), - file_index.members(), trailing_size ); + lzip_index.members(), trailing_size ); } list_line( udata_size, cdata_size, input_filename ); - if( verbosity >= 2 && file_index.members() > 1 ) + if( verbosity >= 2 && lzip_index.members() > 1 ) { std::fputs( " member data_pos data_size member_pos member_size\n", stdout ); - for( long i = 0; i < file_index.members(); ++i ) + for( long i = 0; i < lzip_index.members(); ++i ) { - const Block & db = file_index.dblock( i ); - const Block & mb = file_index.mblock( i ); + const Block & db = lzip_index.dblock( i ); + const Block & mb = lzip_index.mblock( i ); std::printf( "%5ld %15llu %15llu %15llu %15llu\n", i + 1, db.pos(), db.size(), mb.pos(), mb.size() ); } @@ -1,5 +1,5 @@ -/* Plzip - Parallel compressor compatible with lzip - Copyright (C) 2009-2018 Antonio Diaz Diaz. +/* Plzip - Massively parallel implementation of lzip + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -48,7 +48,7 @@ public: { const std::string & s = filenames[i]; const unsigned len = ( s == "-" ) ? stdin_name_len : s.size(); - if( len > longest_name ) longest_name = len; + if( longest_name < len ) longest_name = len; } if( longest_name == 0 ) longest_name = stdin_name_len; } @@ -58,7 +58,7 @@ public: if( filename.size() && filename != "-" ) name_ = filename; else name_ = stdin_name; padded_name = " "; padded_name += name_; padded_name += ": "; - if( name_.size() < longest_name ) + if( longest_name > name_.size() ) padded_name.append( longest_name - name_.size(), ' ' ); first_post = true; } @@ -82,30 +82,30 @@ inline int real_bits( unsigned value ) } -const uint8_t magic_string[4] = { 0x4C, 0x5A, 0x49, 0x50 }; // "LZIP" +const uint8_t lzip_magic[4] = { 0x4C, 0x5A, 0x49, 0x50 }; // "LZIP" -struct File_header +struct Lzip_header { uint8_t data[6]; // 0-3 magic bytes // 4 version // 5 coded_dict_size enum { size = 6 }; - void set_magic() { std::memcpy( data, magic_string, 4 ); data[4] = 1; } + void set_magic() { std::memcpy( data, lzip_magic, 4 ); data[4] = 1; } bool verify_magic() const - { return ( std::memcmp( data, magic_string, 4 ) == 0 ); } + { return ( std::memcmp( data, lzip_magic, 4 ) == 0 ); } bool verify_prefix( const int sz ) const // detect (truncated) header { for( int i = 0; i < sz && i < 4; ++i ) - if( data[i] != magic_string[i] ) return false; + if( data[i] != lzip_magic[i] ) return false; return ( sz > 0 ); } bool verify_corrupt() const // detect corrupt header { int matches = 0; for( int i = 0; i < 4; ++i ) - if( data[i] == magic_string[i] ) ++matches; + if( data[i] == lzip_magic[i] ) ++matches; return ( matches > 1 && matches < 4 ); } @@ -137,12 +137,11 @@ struct File_header }; -struct File_trailer +struct Lzip_trailer { uint8_t data[20]; // 0-3 CRC32 of the uncompressed data // 4-11 size of the uncompressed data // 12-19 member size including header and trailer - enum { size = 20 }; unsigned data_crc() const @@ -174,6 +173,20 @@ struct File_trailer void member_size( unsigned long long sz ) { for( int i = 12; i <= 19; ++i ) { data[i] = (uint8_t)sz; sz >>= 8; } } + + bool verify_consistency() const // check internal consistency + { + const unsigned crc = data_crc(); + const unsigned long long dsize = data_size(); + if( ( crc == 0 ) != ( dsize == 0 ) ) return false; + const unsigned long long msize = member_size(); + if( msize < min_member_size ) return false; + const unsigned long long mlimit = ( 9 * dsize + 7 ) / 8 + min_member_size; + if( mlimit > dsize && msize > mlimit ) return false; + const unsigned long long dlimit = 7090 * ( msize - 26 ) - 1; + if( dlimit > msize && dsize > dlimit ) return false; + return true; + } }; @@ -200,18 +213,19 @@ int compress( const unsigned long long cfile_size, const int infd, const int outfd, const Pretty_print & pp, const int debug_level ); -// defined in file_index.cc -class File_index; +// defined in lzip_index.cc +class Lzip_index; // defined in dec_stdout.cc int dec_stdout( const int num_workers, const int infd, const int outfd, const Pretty_print & pp, const int debug_level, - const File_index & file_index ); + const int out_slots, const Lzip_index & lzip_index ); // defined in dec_stream.cc int dec_stream( const unsigned long long cfile_size, const int num_workers, const int infd, const int outfd, const Pretty_print & pp, const int debug_level, + const int in_slots, const int out_slots, const bool ignore_trailing, const bool loose_trailing ); // defined in decompress.cc @@ -221,7 +235,8 @@ int decompress_read_error( struct LZ_Decoder * const decoder, const Pretty_print & pp, const int worker_id ); int decompress( const unsigned long long cfile_size, int num_workers, const int infd, const int outfd, const Pretty_print & pp, - const int debug_level, const bool ignore_trailing, + const int debug_level, const int in_slots, + const int out_slots, const bool ignore_trailing, const bool loose_trailing, const bool infd_isreg ); // defined in list.cc diff --git a/file_index.cc b/lzip_index.cc index 8238054..d9c810c 100644 --- a/file_index.cc +++ b/lzip_index.cc @@ -1,5 +1,5 @@ -/* Plzip - Parallel compressor compatible with lzip - Copyright (C) 2009-2018 Antonio Diaz Diaz. +/* Plzip - Massively parallel implementation of lzip + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -28,7 +28,7 @@ #include <unistd.h> #include "lzip.h" -#include "file_index.h" +#include "lzip_index.h" namespace { @@ -44,13 +44,13 @@ int seek_read( const int fd, uint8_t * const buf, const int size, } // end namespace -void File_index::set_errno_error( const char * const msg ) +void Lzip_index::set_errno_error( const char * const msg ) { error_ = msg; error_ += std::strerror( errno ); retval_ = 1; } -void File_index::set_num_error( const char * const msg, unsigned long long num ) +void Lzip_index::set_num_error( const char * const msg, unsigned long long num ) { char buf[80]; snprintf( buf, sizeof buf, "%s%llu", msg, num ); @@ -60,11 +60,11 @@ void File_index::set_num_error( const char * const msg, unsigned long long num ) // If successful, push last member and set pos to member header. -bool File_index::skip_trailing_data( const int fd, long long & pos, +bool Lzip_index::skip_trailing_data( const int fd, long long & pos, const bool ignore_trailing, const bool loose_trailing ) { enum { block_size = 16384, - buffer_size = block_size + File_trailer::size - 1 + File_header::size }; + buffer_size = block_size + Lzip_trailer::size - 1 + Lzip_header::size }; uint8_t buffer[buffer_size]; if( pos < min_member_size ) return false; int bsize = pos % block_size; // total bytes in buffer @@ -78,28 +78,28 @@ bool File_index::skip_trailing_data( const int fd, long long & pos, if( seek_read( fd, buffer, rd_size, ipos ) != rd_size ) { set_errno_error( "Error seeking member trailer: " ); return false; } const uint8_t max_msb = ( ipos + search_size ) >> 56; - for( int i = search_size; i >= File_trailer::size; --i ) + for( int i = search_size; i >= Lzip_trailer::size; --i ) if( buffer[i-1] <= max_msb ) // most significant byte of member_size { - File_trailer & trailer = - *(File_trailer *)( buffer + i - File_trailer::size ); + const Lzip_trailer & trailer = + *(const Lzip_trailer *)( buffer + i - Lzip_trailer::size ); const unsigned long long member_size = trailer.member_size(); - if( member_size == 0 ) - { while( i > File_trailer::size && buffer[i-9] == 0 ) --i; continue; } - if( member_size < min_member_size || member_size > ipos + i ) + if( member_size == 0 ) // skip trailing zeros + { while( i > Lzip_trailer::size && buffer[i-9] == 0 ) --i; continue; } + if( member_size > ipos + i || !trailer.verify_consistency() ) continue; - File_header header; - if( seek_read( fd, header.data, File_header::size, - ipos + i - member_size ) != File_header::size ) + Lzip_header header; + if( seek_read( fd, header.data, Lzip_header::size, + ipos + i - member_size ) != Lzip_header::size ) { set_errno_error( "Error reading member header: " ); return false; } const unsigned dictionary_size = header.dictionary_size(); if( !header.verify_magic() || !header.verify_version() || !isvalid_ds( dictionary_size ) ) continue; - if( (*(File_header *)( buffer + i )).verify_prefix( bsize - i ) ) + if( (*(const Lzip_header *)( buffer + i )).verify_prefix( bsize - i ) ) { error_ = "Last member in input file is truncated or corrupt."; retval_ = 2; return false; } - if( !loose_trailing && bsize - i >= File_header::size && - (*(File_header *)( buffer + i )).verify_corrupt() ) + if( !loose_trailing && bsize - i >= Lzip_header::size && + (*(const Lzip_header *)( buffer + i )).verify_corrupt() ) { error_ = corrupt_mm_msg; retval_ = 2; return false; } if( !ignore_trailing ) { error_ = trailing_msg; retval_ = 2; return false; } @@ -109,10 +109,10 @@ bool File_index::skip_trailing_data( const int fd, long long & pos, return true; } if( ipos <= 0 ) - { set_num_error( "Member size in trailer is corrupt at pos ", pos - 8 ); + { set_num_error( "Bad trailer at pos ", pos - Lzip_trailer::size ); return false; } bsize = buffer_size; - search_size = bsize - File_header::size; + search_size = bsize - Lzip_header::size; rd_size = block_size; ipos -= rd_size; std::memcpy( buffer + rd_size, buffer, buffer_size - rd_size ); @@ -120,20 +120,20 @@ bool File_index::skip_trailing_data( const int fd, long long & pos, } -File_index::File_index( const int infd, const bool ignore_trailing, +Lzip_index::Lzip_index( const int infd, const bool ignore_trailing, const bool loose_trailing ) - : isize( lseek( infd, 0, SEEK_END ) ), retval_( 0 ) + : insize( lseek( infd, 0, SEEK_END ) ), retval_( 0 ) { - if( isize < 0 ) + if( insize < 0 ) { set_errno_error( "Input file is not seekable: " ); return; } - if( isize < min_member_size ) + if( insize < min_member_size ) { error_ = "Input file is too short."; retval_ = 2; return; } - if( isize > INT64_MAX ) + if( insize > INT64_MAX ) { error_ = "Input file is too long (2^63 bytes or more)."; retval_ = 2; return; } - File_header header; - if( seek_read( infd, header.data, File_header::size, 0 ) != File_header::size ) + Lzip_header header; + if( seek_read( infd, header.data, Lzip_header::size, 0 ) != Lzip_header::size ) { set_errno_error( "Error reading member header: " ); return; } if( !header.verify_magic() ) { error_ = bad_magic_msg; retval_ = 2; return; } @@ -142,24 +142,24 @@ File_index::File_index( const int infd, const bool ignore_trailing, if( !isvalid_ds( header.dictionary_size() ) ) { error_ = bad_dict_msg; retval_ = 2; return; } - long long pos = isize; // always points to a header or to EOF + long long pos = insize; // always points to a header or to EOF while( pos >= min_member_size ) { - File_trailer trailer; - if( seek_read( infd, trailer.data, File_trailer::size, - pos - File_trailer::size ) != File_trailer::size ) + Lzip_trailer trailer; + if( seek_read( infd, trailer.data, Lzip_trailer::size, + pos - Lzip_trailer::size ) != Lzip_trailer::size ) { set_errno_error( "Error reading member trailer: " ); break; } const unsigned long long member_size = trailer.member_size(); - if( member_size < min_member_size || member_size > (unsigned long long)pos ) + if( member_size > (unsigned long long)pos || !trailer.verify_consistency() ) { if( member_vector.empty() ) { if( skip_trailing_data( infd, pos, ignore_trailing, loose_trailing ) ) continue; else return; } - set_num_error( "Member size in trailer is corrupt at pos ", pos - 8 ); + set_num_error( "Bad trailer at pos ", pos - Lzip_trailer::size ); break; } - if( seek_read( infd, header.data, File_header::size, - pos - member_size ) != File_header::size ) + if( seek_read( infd, header.data, Lzip_header::size, + pos - member_size ) != Lzip_header::size ) { set_errno_error( "Error reading member header: " ); break; } const unsigned dictionary_size = header.dictionary_size(); if( !header.verify_magic() || !header.verify_version() || @@ -182,7 +182,7 @@ File_index::File_index( const int infd, const bool ignore_trailing, return; } std::reverse( member_vector.begin(), member_vector.end() ); - for( unsigned long i = 0; i < member_vector.size() - 1; ++i ) + for( unsigned long i = 0; ; ++i ) { const long long end = member_vector[i].dblock.end(); if( end < 0 || end > INT64_MAX ) @@ -191,6 +191,7 @@ File_index::File_index( const int infd, const bool ignore_trailing, error_ = "Data in input file is too long (2^63 bytes or more)."; retval_ = 2; return; } + if( i + 1 >= member_vector.size() ) break; member_vector[i+1].dblock.pos( end ); } } diff --git a/file_index.h b/lzip_index.h index 7962b99..3775446 100644 --- a/file_index.h +++ b/lzip_index.h @@ -1,5 +1,5 @@ -/* Plzip - Parallel compressor compatible with lzip - Copyright (C) 2009-2018 Antonio Diaz Diaz. +/* Plzip - Massively parallel implementation of lzip + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -36,7 +36,7 @@ public: }; -class File_index +class Lzip_index { struct Member { @@ -50,7 +50,7 @@ class File_index std::vector< Member > member_vector; std::string error_; - const long long isize; + const long long insize; int retval_; void set_errno_error( const char * const msg ); @@ -59,7 +59,7 @@ class File_index const bool ignore_trailing, const bool loose_trailing ); public: - File_index( const int infd, const bool ignore_trailing, + Lzip_index( const int infd, const bool ignore_trailing, const bool loose_trailing ); long members() const { return member_vector.size(); } @@ -76,7 +76,7 @@ public: // total size including trailing data (if any) long long file_size() const - { if( isize >= 0 ) return isize; else return 0; } + { if( insize >= 0 ) return insize; else return 0; } const Block & dblock( const long i ) const { return member_vector[i].dblock; } @@ -1,6 +1,6 @@ -/* Plzip - Parallel compressor compatible with lzip +/* Plzip - Massively parallel implementation of lzip Copyright (C) 2009 Laszlo Ersek. - Copyright (C) 2009-2018 Antonio Diaz Diaz. + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -40,20 +40,21 @@ #include <utime.h> #include <sys/stat.h> #include <lzlib.h> -#if defined(__MSVCRT__) +#if defined(__MSVCRT__) || defined(__OS2__) #include <io.h> +#if defined(__MSVCRT__) #define fchmod(x,y) 0 #define fchown(x,y,z) 0 #define strtoull std::strtoul #define SIGHUP SIGTERM #define S_ISSOCK(x) 0 +#ifndef S_IRGRP #define S_IRGRP 0 #define S_IWGRP 0 #define S_IROTH 0 #define S_IWOTH 0 #endif -#if defined(__OS2__) -#include <io.h> +#endif #endif #include "arg_parser.h" @@ -71,9 +72,8 @@ int verbosity = 0; namespace { -const char * const Program_name = "Plzip"; const char * const program_name = "plzip"; -const char * const program_year = "2018"; +const char * const program_year = "2019"; const char * invocation_name = 0; const struct { const char * from; const char * to; } known_extensions[] = { @@ -89,6 +89,8 @@ struct Lzma_options enum Mode { m_compress, m_decompress, m_list, m_test }; +/* Variables used in signal handler context. + They are not declared volatile because the handler never returns. */ std::string output_filename; int outfd = -1; bool delete_output_on_interrupt = false; @@ -96,8 +98,22 @@ bool delete_output_on_interrupt = false; void show_help( const long num_online ) { - std::printf( "%s - Parallel compressor compatible with lzip.\n", Program_name ); - std::printf( "\nUsage: %s [options] [files]\n", invocation_name ); + std::printf( "Plzip is a massively parallel (multi-threaded) implementation of lzip, fully\n" + "compatible with lzip 1.4 or newer. Plzip uses the lzlib compression library.\n" + "\nLzip is a lossless data compressor with a user interface similar to the\n" + "one of gzip or bzip2. Lzip can compress about as fast as gzip (lzip -0)\n" + "or compress most files more than bzip2 (lzip -9). Decompression speed is\n" + "intermediate between gzip and bzip2. Lzip is better than gzip and bzip2\n" + "from a data recovery perspective. Lzip has been designed, written and\n" + "tested with great care to replace gzip and bzip2 as the standard\n" + "general-purpose compressed format for unix-like systems.\n" + "\nPlzip can compress/decompress large files on multiprocessor machines\n" + "much faster than lzip, at the cost of a slightly reduced compression\n" + "ratio (0.4 to 2 percent larger compressed files). Note that the number\n" + "of usable threads is limited by file size; on files larger than a few GB\n" + "plzip can use hundreds of processors, but on files of only a few MB\n" + "plzip is no faster than lzip.\n" + "\nUsage: %s [options] [files]\n", invocation_name ); std::printf( "\nOptions:\n" " -h, --help display this help and exit\n" " -V, --version output version information and exit\n" @@ -120,6 +136,8 @@ void show_help( const long num_online ) " --fast alias for -0\n" " --best alias for -9\n" " --loose-trailing allow trailing data seeming corrupt header\n" + " --in-slots=<n> number of 1 MiB input packets buffered [4]\n" + " --out-slots=<n> number of 1 MiB output packets buffered [64]\n" , num_online ); if( verbosity >= 1 ) { @@ -263,7 +281,7 @@ int get_dict_size( const char * const arg ) const long bits = std::strtol( arg, &tail, 0 ); if( bits >= LZ_min_dictionary_bits() && bits <= LZ_max_dictionary_bits() && *tail == 0 ) - return ( 1 << bits ); + return 1 << bits; int dictionary_size = getnum( arg, LZ_min_dictionary_size(), LZ_max_dictionary_size() ); if( dictionary_size == 65535 ) ++dictionary_size; // no fast encoder @@ -409,6 +427,14 @@ bool check_tty( const char * const input_filename, const int infd, return true; } + +void set_signals( void (*action)(int) ) + { + std::signal( SIGHUP, action ); + std::signal( SIGINT, action ); + std::signal( SIGTERM, action ); + } + } // end namespace // This can be called from any thread, main thread or sub-threads alike, @@ -420,6 +446,7 @@ void cleanup_and_fail( const int retval ) // only one thread can delete and exit static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + set_signals( SIG_IGN ); // ignore signals pthread_mutex_lock( &mutex ); // ignore errors to avoid loop const int saved_verbosity = verbosity; verbosity = -1; // suppress messages from other threads @@ -440,6 +467,13 @@ void cleanup_and_fail( const int retval ) namespace { +extern "C" void signal_handler( int ) + { + show_error( "Control-C or similar caught, quitting." ); + cleanup_and_fail( 1 ); + } + + // Set permissions, owner and times. void close_and_set_permissions( const struct stat * const in_statsp ) { @@ -473,21 +507,6 @@ void close_and_set_permissions( const struct stat * const in_statsp ) show_error( "Can't change output file attributes." ); } - -extern "C" void signal_handler( int ) - { - show_error( "Control-C or similar caught, quitting." ); - cleanup_and_fail( 1 ); - } - - -void set_signals() - { - std::signal( SIGHUP, signal_handler ); - std::signal( SIGINT, signal_handler ); - std::signal( SIGTERM, signal_handler ); - } - } // end namespace @@ -495,11 +514,9 @@ void show_error( const char * const msg, const int errcode, const bool help ) { if( verbosity < 0 ) return; if( msg && msg[0] ) - { - std::fprintf( stderr, "%s: %s", program_name, msg ); - if( errcode > 0 ) std::fprintf( stderr, ": %s", std::strerror( errcode ) ); - std::fputc( '\n', stderr ); - } + std::fprintf( stderr, "%s: %s%s%s\n", program_name, msg, + ( errcode > 0 ) ? ": " : "", + ( errcode > 0 ) ? std::strerror( errcode ) : "" ); if( help ) std::fprintf( stderr, "Try '%s --help' for more information.\n", invocation_name ); @@ -509,10 +526,10 @@ void show_error( const char * const msg, const int errcode, const bool help ) void show_file_error( const char * const filename, const char * const msg, const int errcode ) { - if( verbosity < 0 ) return; - std::fprintf( stderr, "%s: %s: %s", program_name, filename, msg ); - if( errcode > 0 ) std::fprintf( stderr, ": %s", std::strerror( errcode ) ); - std::fputc( '\n', stderr ); + if( verbosity >= 0 ) + std::fprintf( stderr, "%s: %s: %s%s%s\n", program_name, filename, msg, + ( errcode > 0 ) ? ": " : "", + ( errcode > 0 ) ? std::strerror( errcode ) : "" ); } @@ -554,6 +571,26 @@ void show_progress( const unsigned long long packet_size, } +#if defined(__MSVCRT__) +#include <windows.h> +#define _SC_NPROCESSORS_ONLN 1 +#define _SC_THREAD_THREADS_MAX 2 + +long sysconf( int flag ) + { + if( flag == _SC_NPROCESSORS_ONLN ) + { + SYSTEM_INFO si; + GetSystemInfo( &si ); + return si.dwNumberOfProcessors; + } + if( flag != _SC_THREAD_THREADS_MAX ) errno = EINVAL; + return -1; // unlimited threads or error + } + +#endif // __MSVCRT__ + + int main( const int argc, const char * const argv[] ) { /* Mapping from gzip/bzip2 style 1..9 compression modes @@ -576,6 +613,8 @@ int main( const int argc, const char * const argv[] ) int data_size = 0; int debug_level = 0; int num_workers = 0; // start this many worker threads + int in_slots = 4; + int out_slots = 64; Mode program_mode = m_compress; bool force = false; bool ignore_trailing = true; @@ -589,7 +628,7 @@ int main( const int argc, const char * const argv[] ) { show_error( "Bad library version. At least lzlib 1.0 is required." ); return 1; } - enum { opt_dbg = 256, opt_lt }; + enum { opt_dbg = 256, opt_in, opt_lt, opt_out }; const Arg_parser::Option options[] = { { '0', "fast", Arg_parser::no }, @@ -622,7 +661,9 @@ int main( const int argc, const char * const argv[] ) { 'v', "verbose", Arg_parser::no }, { 'V', "version", Arg_parser::no }, { opt_dbg, "debug", Arg_parser::yes }, + { opt_in, "in-slots", Arg_parser::yes }, { opt_lt, "loose-trailing", Arg_parser::no }, + { opt_out, "out-slots", Arg_parser::yes }, { 0 , 0, Arg_parser::no } }; const Arg_parser parser( argc, argv, options ); @@ -670,7 +711,9 @@ int main( const int argc, const char * const argv[] ) case 'v': if( verbosity < 4 ) ++verbosity; break; case 'V': show_version(); return 0; case opt_dbg: debug_level = getnum( arg, 0, 3 ); break; + case opt_in: in_slots = getnum( arg, 1, 64 ); break; case opt_lt: loose_trailing = true; break; + case opt_out: out_slots = getnum( arg, 1, 1024 ); break; default : internal_error( "uncaught option." ); } } // end process options @@ -707,8 +750,9 @@ int main( const int argc, const char * const argv[] ) if( num_workers <= 0 ) { - if( sizeof (void *) <= 4 ) // use less than 2.22 GiB on 32 bit systems + if( program_mode == m_compress && sizeof (void *) <= 4 ) { + // use less than 2.22 GiB on 32 bit systems const long long limit = ( 27LL << 25 ) + ( 11LL << 27 ); // 4 * 568 MiB const long long mem = ( 27LL * data_size ) / 8 + ( fast ? 3LL << 19 : 11LL * encoder_options.dictionary_size ); @@ -720,10 +764,11 @@ int main( const int argc, const char * const argv[] ) if( !to_stdout && program_mode != m_test && ( filenames_given || default_output_filename.size() ) ) - set_signals(); + set_signals( signal_handler ); Pretty_print pp( filenames ); + int failed_tests = 0; int retval = 0; bool stdin_used = false; for( unsigned i = 0; i < filenames.size(); ++i ) @@ -798,15 +843,23 @@ int main( const int argc, const char * const argv[] ) num_workers, infd, outfd, pp, debug_level ); else tmp = decompress( cfile_size, num_workers, infd, outfd, pp, debug_level, - ignore_trailing, loose_trailing, infd_isreg ); + in_slots, out_slots, ignore_trailing, loose_trailing, + infd_isreg ); + if( close( infd ) != 0 ) + { + show_error( input_filename.size() ? "Error closing input file" : + "Error closing stdin", errno ); + if( tmp < 1 ) tmp = 1; + } if( tmp > retval ) retval = tmp; - if( tmp && program_mode != m_test ) cleanup_and_fail( retval ); + if( tmp ) + { if( program_mode != m_test ) cleanup_and_fail( retval ); + else ++failed_tests; } if( delete_output_on_interrupt ) close_and_set_permissions( in_statsp ); if( input_filename.size() ) { - close( infd ); if( !keep_input_files && !to_stdout && program_mode != m_test ) std::remove( input_filename.c_str() ); } @@ -816,5 +869,9 @@ int main( const int argc, const char * const argv[] ) show_error( "Error closing stdout", errno ); if( retval < 1 ) retval = 1; } + if( failed_tests > 0 && verbosity >= 1 && filenames.size() > 1 ) + std::fprintf( stderr, "%s: warning: %d %s failed the test.\n", + program_name, failed_tests, + ( failed_tests == 1 ) ? "file" : "files" ); return retval; } diff --git a/testsuite/check.sh b/testsuite/check.sh index a4113c3..59fe3f5 100755 --- a/testsuite/check.sh +++ b/testsuite/check.sh @@ -1,6 +1,6 @@ #! /bin/sh -# check script for Plzip - Parallel compressor compatible with lzip -# Copyright (C) 2009-2018 Antonio Diaz Diaz. +# check script for Plzip - Massively parallel implementation of lzip +# Copyright (C) 2009-2019 Antonio Diaz Diaz. # # This script is free software: you have unlimited permission # to copy, distribute and modify it. @@ -31,17 +31,28 @@ cd "${objdir}"/tmp || framework_failure cat "${testdir}"/test.txt > in || framework_failure in_lz="${testdir}"/test.txt.lz fail=0 +lwarn8=0 +lwarn10=0 test_failed() { fail=1 ; printf " $1" ; [ -z "$2" ] || printf "($2)" ; } +lzlib_1_8() { [ ${lwarn8} = 0 ] && + printf "\nwarning: header truncation detection requires lzlib 1.8 or newer" + lwarn8=1 ; } +lzlib_1_10() { [ ${lwarn10} = 0 ] && + printf "\nwarning: header HD=3 detection requires lzlib 1.10 or newer" + lwarn10=1 ; } printf "testing plzip-%s..." "$2" "${LZIP}" -fkqm4 in -{ [ $? = 1 ] && [ ! -e in.lz ] ; } || test_failed $LINENO +[ $? = 1 ] || test_failed $LINENO +[ ! -e in.lz ] || test_failed $LINENO "${LZIP}" -fkqm274 in -{ [ $? = 1 ] && [ ! -e in.lz ] ; } || test_failed $LINENO +[ $? = 1 ] || test_failed $LINENO +[ ! -e in.lz ] || test_failed $LINENO for i in bad_size -1 0 4095 513MiB 1G 1T 1P 1E 1Z 1Y 10KB ; do "${LZIP}" -fkqs $i in - { [ $? = 1 ] && [ ! -e in.lz ] ; } || test_failed $LINENO $i + [ $? = 1 ] || test_failed $LINENO $i + [ ! -e in.lz ] || test_failed $LINENO $i done "${LZIP}" -lq in [ $? = 2 ] || test_failed $LINENO @@ -91,26 +102,34 @@ printf "\ntesting decompression..." "${LZIP}" -cd "${in_lz}" > copy || test_failed $LINENO cmp in copy || test_failed $LINENO -rm -f copy +rm -f copy || framework_failure cat "${in_lz}" > copy.lz || framework_failure "${LZIP}" -dk copy.lz || test_failed $LINENO cmp in copy || test_failed $LINENO printf "to be overwritten" > copy || framework_failure "${LZIP}" -d copy.lz 2> /dev/null [ $? = 1 ] || test_failed $LINENO -"${LZIP}" -df copy.lz -{ [ $? = 0 ] && [ ! -e copy.lz ] && cmp in copy ; } || test_failed $LINENO +"${LZIP}" -df copy.lz || test_failed $LINENO +[ ! -e copy.lz ] || test_failed $LINENO +cmp in copy || test_failed $LINENO + +rm -f copy || framework_failure +cat "${in_lz}" > copy.lz || framework_failure +"${LZIP}" -d -S100k copy.lz || test_failed $LINENO # ignore -S +[ ! -e copy.lz ] || test_failed $LINENO +cmp in copy || test_failed $LINENO printf "to be overwritten" > copy || framework_failure "${LZIP}" -df -o copy < "${in_lz}" || test_failed $LINENO cmp in copy || test_failed $LINENO -rm -f copy +rm -f copy || framework_failure "${LZIP}" < in > anyothername || test_failed $LINENO -"${LZIP}" -dv --output copy - anyothername - < "${in_lz}" 2> /dev/null -{ [ $? = 0 ] && cmp in copy && cmp in anyothername.out ; } || +"${LZIP}" -dv --output copy - anyothername - < "${in_lz}" 2> /dev/null || test_failed $LINENO -rm -f copy anyothername.out +cmp in copy || test_failed $LINENO +cmp in anyothername.out || test_failed $LINENO +rm -f copy anyothername.out || framework_failure "${LZIP}" -lq in "${in_lz}" [ $? = 2 ] || test_failed $LINENO @@ -121,10 +140,12 @@ rm -f copy anyothername.out "${LZIP}" -tq nx_file.lz "${in_lz}" [ $? = 1 ] || test_failed $LINENO "${LZIP}" -cdq in "${in_lz}" > copy -{ [ $? = 2 ] && cat copy in | cmp in - ; } || test_failed $LINENO +[ $? = 2 ] || test_failed $LINENO +cat copy in | cmp in - || test_failed $LINENO "${LZIP}" -cdq nx_file.lz "${in_lz}" > copy -{ [ $? = 1 ] && cmp in copy ; } || test_failed $LINENO -rm -f copy +[ $? = 1 ] || test_failed $LINENO +cmp in copy || test_failed $LINENO +rm -f copy || framework_failure cat "${in_lz}" > copy.lz || framework_failure for i in 1 2 3 4 5 6 7 ; do printf "g" >> copy.lz || framework_failure @@ -134,11 +155,15 @@ for i in 1 2 3 4 5 6 7 ; do [ $? = 2 ] || test_failed $LINENO $i done "${LZIP}" -dq in copy.lz -{ [ $? = 2 ] && [ -e copy.lz ] && [ ! -e copy ] && [ ! -e in.out ] ; } || - test_failed $LINENO +[ $? = 2 ] || test_failed $LINENO +[ -e copy.lz ] || test_failed $LINENO +[ ! -e copy ] || test_failed $LINENO +[ ! -e in.out ] || test_failed $LINENO "${LZIP}" -dq nx_file.lz copy.lz -{ [ $? = 1 ] && [ ! -e copy.lz ] && [ ! -e nx_file ] && cmp in copy ; } || - test_failed $LINENO +[ $? = 1 ] || test_failed $LINENO +[ ! -e copy.lz ] || test_failed $LINENO +[ ! -e nx_file ] || test_failed $LINENO +cmp in copy || test_failed $LINENO cat in in > in2 || framework_failure cat "${in_lz}" "${in_lz}" > in2.lz || framework_failure @@ -155,7 +180,7 @@ cmp in2 copy2 || test_failed $LINENO printf "\ngarbage" >> copy2.lz || framework_failure "${LZIP}" -tvvvv copy2.lz 2> /dev/null || test_failed $LINENO -rm -f copy2 +rm -f copy2 || framework_failure "${LZIP}" -alq copy2.lz [ $? = 2 ] || test_failed $LINENO "${LZIP}" -atq copy2.lz @@ -163,12 +188,15 @@ rm -f copy2 "${LZIP}" -atq < copy2.lz [ $? = 2 ] || test_failed $LINENO "${LZIP}" -adkq copy2.lz -{ [ $? = 2 ] && [ ! -e copy2 ] ; } || test_failed $LINENO +[ $? = 2 ] || test_failed $LINENO +[ ! -e copy2 ] || test_failed $LINENO "${LZIP}" -adkq -o copy2 < copy2.lz -{ [ $? = 2 ] && [ ! -e copy2 ] ; } || test_failed $LINENO +[ $? = 2 ] || test_failed $LINENO +[ ! -e copy2 ] || test_failed $LINENO printf "to be overwritten" > copy2 || framework_failure "${LZIP}" -df copy2.lz || test_failed $LINENO cmp in2 copy2 || test_failed $LINENO +rm -f in2 copy2 || framework_failure printf "\ntesting compression..." @@ -204,24 +232,30 @@ for i in s4Ki 0 1 2 3 4 5 6 7 8 9 ; do "${LZIP}" -df -o copy < out.lz || test_failed $LINENO $i cmp in copy || test_failed $LINENO $i done +rm -f out.lz || framework_failure cat in in in in > in4 || framework_failure for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 ; do + "${LZIP}" -s4Ki -B8Ki -n$i < in4 > out4.lz || test_failed $LINENO $i + printf "g" >> out4.lz || framework_failure + "${LZIP}" -d -n$i < out4.lz > out4 || test_failed $LINENO $i + cmp in4 out4 || test_failed $LINENO $i + "${LZIP}" -d --in-slots=$i < out4.lz > out4 || test_failed $LINENO $i + cmp in4 out4 || test_failed $LINENO $i + "${LZIP}" -d --out-slots=$i < out4.lz > out4 || test_failed $LINENO $i + cmp in4 out4 || test_failed $LINENO $i + "${LZIP}" -c -s4Ki -B8Ki -n$i in4 > out4.lz || test_failed $LINENO $i printf "g" >> out4.lz || framework_failure - "${LZIP}" -cd -n$i out4.lz > copy4 || test_failed $LINENO $i - cmp in4 copy4 || test_failed $LINENO $i + "${LZIP}" -cd -n$i out4.lz > out4 || test_failed $LINENO $i + cmp in4 out4 || test_failed $LINENO $i + "${LZIP}" -cd --out-slots=$i out4.lz > out4 || test_failed $LINENO $i + cmp in4 out4 || test_failed $LINENO $i + rm -f out4 || framework_failure "${LZIP}" -d -n$i out4.lz || test_failed $LINENO $i cmp in4 out4 || test_failed $LINENO $i - rm -f out4 -done - -for i in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 ; do - "${LZIP}" -s4Ki -B8Ki -n$i < in4 > out4 || test_failed $LINENO $i - printf "g" >> out4 || framework_failure - "${LZIP}" -d -n$i < out4 > copy4 || test_failed $LINENO $i - cmp in4 copy4 || test_failed $LINENO $i done +rm -f out4 || framework_failure cat in in in in in in in in | "${LZIP}" -1s4Ki | "${LZIP}" -t || test_failed $LINENO @@ -230,58 +264,58 @@ printf "\ntesting bad input..." headers='LZIp LZiP LZip LzIP LzIp LziP lZIP lZIp lZiP lzIP' body='\001\014\000\203\377\373\377\377\300\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000$\000\000\000\000\000\000\000' -cat "${in_lz}" > in0.lz -printf "LZIP${body}" >> in0.lz -if "${LZIP}" -tq in0.lz ; then +cat "${in_lz}" > int.lz +printf "LZIP${body}" >> int.lz +if "${LZIP}" -tq int.lz ; then for header in ${headers} ; do - printf "${header}${body}" > in0.lz # first member - "${LZIP}" -lq in0.lz + printf "${header}${body}" > int.lz # first member + "${LZIP}" -lq int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -tq in0.lz + "${LZIP}" -tq int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -tq < in0.lz + "${LZIP}" -tq < int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -cdq in0.lz > /dev/null + "${LZIP}" -cdq int.lz > /dev/null [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -lq --loose-trailing in0.lz + "${LZIP}" -lq --loose-trailing int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -tq --loose-trailing in0.lz + "${LZIP}" -tq --loose-trailing int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -tq --loose-trailing < in0.lz + "${LZIP}" -tq --loose-trailing < int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -cdq --loose-trailing in0.lz > /dev/null + "${LZIP}" -cdq --loose-trailing int.lz > /dev/null [ $? = 2 ] || test_failed $LINENO ${header} - cat "${in_lz}" > in0.lz - printf "${header}${body}" >> in0.lz # trailing data - "${LZIP}" -lq in0.lz + cat "${in_lz}" > int.lz + printf "${header}${body}" >> int.lz # trailing data + "${LZIP}" -lq int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -tq in0.lz + "${LZIP}" -tq int.lz [ $? = 2 ] || test_failed $LINENO ${header} -# "${LZIP}" -tq < in0.lz # requires lzlib-1.10 -# [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -cdq in0.lz > /dev/null + "${LZIP}" -tq < int.lz + [ $? = 2 ] || lzlib_1_10 # requires lzlib 1.10 + "${LZIP}" -cdq int.lz > /dev/null [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -lq --loose-trailing in0.lz - [ $? = 0 ] || test_failed $LINENO ${header} - "${LZIP}" -t --loose-trailing in0.lz - [ $? = 0 ] || test_failed $LINENO ${header} - "${LZIP}" -t --loose-trailing < in0.lz - [ $? = 0 ] || test_failed $LINENO ${header} - "${LZIP}" -cd --loose-trailing in0.lz > /dev/null - [ $? = 0 ] || test_failed $LINENO ${header} - "${LZIP}" -lq --loose-trailing --trailing-error in0.lz + "${LZIP}" -lq --loose-trailing int.lz || + test_failed $LINENO ${header} + "${LZIP}" -t --loose-trailing int.lz || + test_failed $LINENO ${header} + "${LZIP}" -t --loose-trailing < int.lz || + test_failed $LINENO ${header} + "${LZIP}" -cd --loose-trailing int.lz > /dev/null || + test_failed $LINENO ${header} + "${LZIP}" -lq --loose-trailing --trailing-error int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -tq --loose-trailing --trailing-error in0.lz + "${LZIP}" -tq --loose-trailing --trailing-error int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -tq --loose-trailing --trailing-error < in0.lz + "${LZIP}" -tq --loose-trailing --trailing-error < int.lz [ $? = 2 ] || test_failed $LINENO ${header} - "${LZIP}" -cdq --loose-trailing --trailing-error in0.lz > /dev/null + "${LZIP}" -cdq --loose-trailing --trailing-error int.lz > /dev/null [ $? = 2 ] || test_failed $LINENO ${header} done else printf "\nwarning: skipping header test: 'printf' does not work on your system." fi -rm -f in0.lz +rm -f int.lz || framework_failure cat "${in_lz}" "${in_lz}" "${in_lz}" > in3.lz || framework_failure if dd if=in3.lz of=trunc.lz bs=14752 count=1 2> /dev/null && @@ -293,16 +327,16 @@ if dd if=in3.lz of=trunc.lz bs=14752 count=1 2> /dev/null && "${LZIP}" -tq trunc.lz [ $? = 2 ] || test_failed $LINENO $i "${LZIP}" -tq < trunc.lz - [ $? = 2 ] || test_failed $LINENO $i + [ $? = 2 ] || lzlib_1_8 # requires lzlib 1.8 "${LZIP}" -cdq trunc.lz > out [ $? = 2 ] || test_failed $LINENO $i "${LZIP}" -dq < trunc.lz > out - [ $? = 2 ] || test_failed $LINENO $i + [ $? = 2 ] || lzlib_1_8 # requires lzlib 1.8 done else printf "\nwarning: skipping truncation test: 'dd' does not work on your system." fi -rm -f in3.lz trunc.lz +rm -f in2.lz in3.lz trunc.lz out || framework_failure cat "${in_lz}" > ingin.lz || framework_failure printf "g" >> ingin.lz || framework_failure @@ -316,7 +350,7 @@ cat "${in_lz}" >> ingin.lz || framework_failure "${LZIP}" -t < ingin.lz || test_failed $LINENO "${LZIP}" -d < ingin.lz > copy || test_failed $LINENO cmp in copy || test_failed $LINENO -rm -f ingin.lz +rm -f copy ingin.lz || framework_failure echo if [ ${fail} = 0 ] ; then |