diff options
Diffstat (limited to 'main.cc')
| -rw-r--r-- | main.cc | 137 |
1 files changed, 97 insertions, 40 deletions
@@ -1,6 +1,6 @@ -/* Plzip - Parallel compressor compatible with lzip +/* Plzip - Massively parallel implementation of lzip Copyright (C) 2009 Laszlo Ersek. - Copyright (C) 2009-2018 Antonio Diaz Diaz. + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -40,20 +40,21 @@ #include <utime.h> #include <sys/stat.h> #include <lzlib.h> -#if defined(__MSVCRT__) +#if defined(__MSVCRT__) || defined(__OS2__) #include <io.h> +#if defined(__MSVCRT__) #define fchmod(x,y) 0 #define fchown(x,y,z) 0 #define strtoull std::strtoul #define SIGHUP SIGTERM #define S_ISSOCK(x) 0 +#ifndef S_IRGRP #define S_IRGRP 0 #define S_IWGRP 0 #define S_IROTH 0 #define S_IWOTH 0 #endif -#if defined(__OS2__) -#include <io.h> +#endif #endif #include "arg_parser.h" @@ -71,9 +72,8 @@ int verbosity = 0; namespace { -const char * const Program_name = "Plzip"; const char * const program_name = "plzip"; -const char * const program_year = "2018"; +const char * const program_year = "2019"; const char * invocation_name = 0; const struct { const char * from; const char * to; } known_extensions[] = { @@ -89,6 +89,8 @@ struct Lzma_options enum Mode { m_compress, m_decompress, m_list, m_test }; +/* Variables used in signal handler context. + They are not declared volatile because the handler never returns. */ std::string output_filename; int outfd = -1; bool delete_output_on_interrupt = false; @@ -96,8 +98,22 @@ bool delete_output_on_interrupt = false; void show_help( const long num_online ) { - std::printf( "%s - Parallel compressor compatible with lzip.\n", Program_name ); - std::printf( "\nUsage: %s [options] [files]\n", invocation_name ); + std::printf( "Plzip is a massively parallel (multi-threaded) implementation of lzip, fully\n" + "compatible with lzip 1.4 or newer. Plzip uses the lzlib compression library.\n" + "\nLzip is a lossless data compressor with a user interface similar to the\n" + "one of gzip or bzip2. Lzip can compress about as fast as gzip (lzip -0)\n" + "or compress most files more than bzip2 (lzip -9). Decompression speed is\n" + "intermediate between gzip and bzip2. Lzip is better than gzip and bzip2\n" + "from a data recovery perspective. Lzip has been designed, written and\n" + "tested with great care to replace gzip and bzip2 as the standard\n" + "general-purpose compressed format for unix-like systems.\n" + "\nPlzip can compress/decompress large files on multiprocessor machines\n" + "much faster than lzip, at the cost of a slightly reduced compression\n" + "ratio (0.4 to 2 percent larger compressed files). Note that the number\n" + "of usable threads is limited by file size; on files larger than a few GB\n" + "plzip can use hundreds of processors, but on files of only a few MB\n" + "plzip is no faster than lzip.\n" + "\nUsage: %s [options] [files]\n", invocation_name ); std::printf( "\nOptions:\n" " -h, --help display this help and exit\n" " -V, --version output version information and exit\n" @@ -120,6 +136,8 @@ void show_help( const long num_online ) " --fast alias for -0\n" " --best alias for -9\n" " --loose-trailing allow trailing data seeming corrupt header\n" + " --in-slots=<n> number of 1 MiB input packets buffered [4]\n" + " --out-slots=<n> number of 1 MiB output packets buffered [64]\n" , num_online ); if( verbosity >= 1 ) { @@ -263,7 +281,7 @@ int get_dict_size( const char * const arg ) const long bits = std::strtol( arg, &tail, 0 ); if( bits >= LZ_min_dictionary_bits() && bits <= LZ_max_dictionary_bits() && *tail == 0 ) - return ( 1 << bits ); + return 1 << bits; int dictionary_size = getnum( arg, LZ_min_dictionary_size(), LZ_max_dictionary_size() ); if( dictionary_size == 65535 ) ++dictionary_size; // no fast encoder @@ -409,6 +427,14 @@ bool check_tty( const char * const input_filename, const int infd, return true; } + +void set_signals( void (*action)(int) ) + { + std::signal( SIGHUP, action ); + std::signal( SIGINT, action ); + std::signal( SIGTERM, action ); + } + } // end namespace // This can be called from any thread, main thread or sub-threads alike, @@ -420,6 +446,7 @@ void cleanup_and_fail( const int retval ) // only one thread can delete and exit static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + set_signals( SIG_IGN ); // ignore signals pthread_mutex_lock( &mutex ); // ignore errors to avoid loop const int saved_verbosity = verbosity; verbosity = -1; // suppress messages from other threads @@ -440,6 +467,13 @@ void cleanup_and_fail( const int retval ) namespace { +extern "C" void signal_handler( int ) + { + show_error( "Control-C or similar caught, quitting." ); + cleanup_and_fail( 1 ); + } + + // Set permissions, owner and times. void close_and_set_permissions( const struct stat * const in_statsp ) { @@ -473,21 +507,6 @@ void close_and_set_permissions( const struct stat * const in_statsp ) show_error( "Can't change output file attributes." ); } - -extern "C" void signal_handler( int ) - { - show_error( "Control-C or similar caught, quitting." ); - cleanup_and_fail( 1 ); - } - - -void set_signals() - { - std::signal( SIGHUP, signal_handler ); - std::signal( SIGINT, signal_handler ); - std::signal( SIGTERM, signal_handler ); - } - } // end namespace @@ -495,11 +514,9 @@ void show_error( const char * const msg, const int errcode, const bool help ) { if( verbosity < 0 ) return; if( msg && msg[0] ) - { - std::fprintf( stderr, "%s: %s", program_name, msg ); - if( errcode > 0 ) std::fprintf( stderr, ": %s", std::strerror( errcode ) ); - std::fputc( '\n', stderr ); - } + std::fprintf( stderr, "%s: %s%s%s\n", program_name, msg, + ( errcode > 0 ) ? ": " : "", + ( errcode > 0 ) ? std::strerror( errcode ) : "" ); if( help ) std::fprintf( stderr, "Try '%s --help' for more information.\n", invocation_name ); @@ -509,10 +526,10 @@ void show_error( const char * const msg, const int errcode, const bool help ) void show_file_error( const char * const filename, const char * const msg, const int errcode ) { - if( verbosity < 0 ) return; - std::fprintf( stderr, "%s: %s: %s", program_name, filename, msg ); - if( errcode > 0 ) std::fprintf( stderr, ": %s", std::strerror( errcode ) ); - std::fputc( '\n', stderr ); + if( verbosity >= 0 ) + std::fprintf( stderr, "%s: %s: %s%s%s\n", program_name, filename, msg, + ( errcode > 0 ) ? ": " : "", + ( errcode > 0 ) ? std::strerror( errcode ) : "" ); } @@ -554,6 +571,26 @@ void show_progress( const unsigned long long packet_size, } +#if defined(__MSVCRT__) +#include <windows.h> +#define _SC_NPROCESSORS_ONLN 1 +#define _SC_THREAD_THREADS_MAX 2 + +long sysconf( int flag ) + { + if( flag == _SC_NPROCESSORS_ONLN ) + { + SYSTEM_INFO si; + GetSystemInfo( &si ); + return si.dwNumberOfProcessors; + } + if( flag != _SC_THREAD_THREADS_MAX ) errno = EINVAL; + return -1; // unlimited threads or error + } + +#endif // __MSVCRT__ + + int main( const int argc, const char * const argv[] ) { /* Mapping from gzip/bzip2 style 1..9 compression modes @@ -576,6 +613,8 @@ int main( const int argc, const char * const argv[] ) int data_size = 0; int debug_level = 0; int num_workers = 0; // start this many worker threads + int in_slots = 4; + int out_slots = 64; Mode program_mode = m_compress; bool force = false; bool ignore_trailing = true; @@ -589,7 +628,7 @@ int main( const int argc, const char * const argv[] ) { show_error( "Bad library version. At least lzlib 1.0 is required." ); return 1; } - enum { opt_dbg = 256, opt_lt }; + enum { opt_dbg = 256, opt_in, opt_lt, opt_out }; const Arg_parser::Option options[] = { { '0', "fast", Arg_parser::no }, @@ -622,7 +661,9 @@ int main( const int argc, const char * const argv[] ) { 'v', "verbose", Arg_parser::no }, { 'V', "version", Arg_parser::no }, { opt_dbg, "debug", Arg_parser::yes }, + { opt_in, "in-slots", Arg_parser::yes }, { opt_lt, "loose-trailing", Arg_parser::no }, + { opt_out, "out-slots", Arg_parser::yes }, { 0 , 0, Arg_parser::no } }; const Arg_parser parser( argc, argv, options ); @@ -670,7 +711,9 @@ int main( const int argc, const char * const argv[] ) case 'v': if( verbosity < 4 ) ++verbosity; break; case 'V': show_version(); return 0; case opt_dbg: debug_level = getnum( arg, 0, 3 ); break; + case opt_in: in_slots = getnum( arg, 1, 64 ); break; case opt_lt: loose_trailing = true; break; + case opt_out: out_slots = getnum( arg, 1, 1024 ); break; default : internal_error( "uncaught option." ); } } // end process options @@ -707,8 +750,9 @@ int main( const int argc, const char * const argv[] ) if( num_workers <= 0 ) { - if( sizeof (void *) <= 4 ) // use less than 2.22 GiB on 32 bit systems + if( program_mode == m_compress && sizeof (void *) <= 4 ) { + // use less than 2.22 GiB on 32 bit systems const long long limit = ( 27LL << 25 ) + ( 11LL << 27 ); // 4 * 568 MiB const long long mem = ( 27LL * data_size ) / 8 + ( fast ? 3LL << 19 : 11LL * encoder_options.dictionary_size ); @@ -720,10 +764,11 @@ int main( const int argc, const char * const argv[] ) if( !to_stdout && program_mode != m_test && ( filenames_given || default_output_filename.size() ) ) - set_signals(); + set_signals( signal_handler ); Pretty_print pp( filenames ); + int failed_tests = 0; int retval = 0; bool stdin_used = false; for( unsigned i = 0; i < filenames.size(); ++i ) @@ -798,15 +843,23 @@ int main( const int argc, const char * const argv[] ) num_workers, infd, outfd, pp, debug_level ); else tmp = decompress( cfile_size, num_workers, infd, outfd, pp, debug_level, - ignore_trailing, loose_trailing, infd_isreg ); + in_slots, out_slots, ignore_trailing, loose_trailing, + infd_isreg ); + if( close( infd ) != 0 ) + { + show_error( input_filename.size() ? "Error closing input file" : + "Error closing stdin", errno ); + if( tmp < 1 ) tmp = 1; + } if( tmp > retval ) retval = tmp; - if( tmp && program_mode != m_test ) cleanup_and_fail( retval ); + if( tmp ) + { if( program_mode != m_test ) cleanup_and_fail( retval ); + else ++failed_tests; } if( delete_output_on_interrupt ) close_and_set_permissions( in_statsp ); if( input_filename.size() ) { - close( infd ); if( !keep_input_files && !to_stdout && program_mode != m_test ) std::remove( input_filename.c_str() ); } @@ -816,5 +869,9 @@ int main( const int argc, const char * const argv[] ) show_error( "Error closing stdout", errno ); if( retval < 1 ) retval = 1; } + if( failed_tests > 0 && verbosity >= 1 && filenames.size() > 1 ) + std::fprintf( stderr, "%s: warning: %d %s failed the test.\n", + program_name, failed_tests, + ( failed_tests == 1 ) ? "file" : "files" ); return retval; } |