From aeab49c877a93051065ab41625d9b13a6017c1d6 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Tue, 13 Feb 2018 08:06:11 +0100 Subject: Merging upstream version 1.7. Signed-off-by: Daniel Baumann --- main.cc | 231 +++++++++++++++++++++++++++++++++++++--------------------------- 1 file changed, 134 insertions(+), 97 deletions(-) (limited to 'main.cc') diff --git a/main.cc b/main.cc index 5e75690..d1f76bc 100644 --- a/main.cc +++ b/main.cc @@ -1,6 +1,6 @@ /* Plzip - Parallel compressor compatible with lzip Copyright (C) 2009 Laszlo Ersek. - Copyright (C) 2009-2017 Antonio Diaz Diaz. + Copyright (C) 2009-2018 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -73,7 +73,7 @@ namespace { const char * const Program_name = "Plzip"; const char * const program_name = "plzip"; -const char * const program_year = "2017"; +const char * const program_year = "2018"; const char * invocation_name = 0; const struct { const char * from; const char * to; } known_extensions[] = { @@ -118,7 +118,9 @@ void show_help( const long num_online ) " -v, --verbose be verbose (a 2nd -v gives more)\n" " -0 .. -9 set compression level [default 6]\n" " --fast alias for -0\n" - " --best alias for -9\n", num_online ); + " --best alias for -9\n" + " --loose-trailing allow trailing data seeming corrupt header\n" + , num_online ); if( verbosity >= 1 ) { std::printf( " --debug= (0-1) print debug statistics to stderr\n" ); @@ -145,8 +147,8 @@ void show_help( const long num_online ) void show_version() { std::printf( "%s %s\n", program_name, PROGVERSION ); - std::printf( "Copyright (C) 2009 Laszlo Ersek.\n" - "Copyright (C) %s Antonio Diaz Diaz.\n", program_year ); + std::printf( "Copyright (C) 2009 Laszlo Ersek.\n" ); + std::printf( "Copyright (C) %s Antonio Diaz Diaz.\n", program_year ); std::printf( "Using lzlib %s\n", LZ_version() ); std::printf( "License GPLv2+: GNU GPL version 2 or later \n" "This is free software: you are free to change and redistribute it.\n" @@ -155,6 +157,21 @@ void show_version() } // end namespace +void Pretty_print::operator()( const char * const msg ) const + { + if( verbosity >= 0 ) + { + if( first_post ) + { + first_post = false; + std::fputs( padded_name.c_str(), stderr ); + if( !msg ) std::fflush( stderr ); + } + if( msg ) std::fprintf( stderr, "%s\n", msg ); + } + } + + const char * bad_version( const unsigned version ) { static char buf[80]; @@ -185,8 +202,7 @@ const char * format_ds( const unsigned dictionary_size ) void show_header( const unsigned dictionary_size ) { - if( verbosity >= 3 ) - std::fprintf( stderr, "dictionary %s. ", format_ds( dictionary_size ) ); + std::fprintf( stderr, "dictionary %s, ", format_ds( dictionary_size ) ); } namespace { @@ -278,6 +294,33 @@ int extension_index( const std::string & name ) return -1; } + +void set_c_outname( const std::string & name, const bool force_ext ) + { + output_filename = name; + if( force_ext || extension_index( output_filename ) < 0 ) + output_filename += known_extensions[0].from; + } + + +void set_d_outname( const std::string & name, const int eindex ) + { + if( eindex >= 0 ) + { + const std::string from( known_extensions[eindex].from ); + if( name.size() > from.size() ) + { + output_filename.assign( name, 0, name.size() - from.size() ); + output_filename += known_extensions[eindex].to; + return; + } + } + output_filename = name; output_filename += ".out"; + if( verbosity >= 1 ) + std::fprintf( stderr, "%s: Can't guess original name for '%s' -- using '%s'\n", + program_name, name.c_str(), output_filename.c_str() ); + } + } // end namespace int open_instream( const char * const name, struct stat * const in_statsp, @@ -325,32 +368,6 @@ int open_instream2( const char * const name, struct stat * const in_statsp, } -void set_c_outname( const std::string & name ) - { - output_filename = name; - output_filename += known_extensions[0].from; - } - - -void set_d_outname( const std::string & name, const int eindex ) - { - if( eindex >= 0 ) - { - const std::string from( known_extensions[eindex].from ); - if( name.size() > from.size() ) - { - output_filename.assign( name, 0, name.size() - from.size() ); - output_filename += known_extensions[eindex].to; - return; - } - } - output_filename = name; output_filename += ".out"; - if( verbosity >= 1 ) - std::fprintf( stderr, "%s: Can't guess original name for '%s' -- using '%s'\n", - program_name, name.c_str(), output_filename.c_str() ); - } - - bool open_outstream( const bool force, const bool from_stdin ) { const mode_t usr_rw = S_IRUSR | S_IWUSR; @@ -404,15 +421,19 @@ void cleanup_and_fail( const int retval ) static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; pthread_mutex_lock( &mutex ); // ignore errors to avoid loop + const int saved_verbosity = verbosity; + verbosity = -1; // suppress messages from other threads if( delete_output_on_interrupt ) { delete_output_on_interrupt = false; - if( verbosity >= 0 ) + if( saved_verbosity >= 0 ) std::fprintf( stderr, "%s: Deleting output file '%s', if it exists.\n", program_name, output_filename.c_str() ); if( outfd >= 0 ) { close( outfd ); outfd = -1; } - if( std::remove( output_filename.c_str() ) != 0 && errno != ENOENT ) - show_error( "WARNING: deletion of output file (apparently) failed." ); + if( std::remove( output_filename.c_str() ) != 0 && errno != ENOENT && + saved_verbosity >= 0 ) + std::fprintf( stderr, "%s: WARNING: deletion of output file " + "(apparently) failed.\n", program_name ); } std::exit( retval ); } @@ -503,25 +524,30 @@ void internal_error( const char * const msg ) } -void show_progress( const int packet_size, - const Pretty_print * const p, - const unsigned long long cfile_size ) +void show_progress( const unsigned long long packet_size, + const unsigned long long cfile_size, + const Pretty_print * const p ) { static unsigned long long csize = 0; // file_size / 100 static unsigned long long pos = 0; static const Pretty_print * pp = 0; static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + static bool enabled = true; - if( verbosity < 2 ) return; + if( !enabled ) return; if( p ) // initialize static vars - { csize = cfile_size; pos = 0; pp = p; } + { + if( verbosity < 2 || !isatty( STDERR_FILENO ) ) { enabled = false; return; } + csize = cfile_size; pos = 0; pp = p; + } if( pp ) { xlock( &mutex ); pos += packet_size; if( csize > 0 ) - std::fprintf( stderr, "%4llu%%", pos / csize ); - std::fprintf( stderr, " %.1f MB\r", pos / 1000000.0 ); + std::fprintf( stderr, "%4llu%% %.1f MB\r", pos / csize, pos / 1000000.0 ); + else + std::fprintf( stderr, " %.1f MB\r", pos / 1000000.0 ); pp->reset(); (*pp)(); // restore cursor position xunlock( &mutex ); } @@ -549,12 +575,12 @@ int main( const int argc, const char * const argv[] ) std::vector< std::string > filenames; int data_size = 0; int debug_level = 0; - int infd = -1; int num_workers = 0; // start this many worker threads Mode program_mode = m_compress; bool force = false; bool ignore_trailing = true; bool keep_input_files = false; + bool loose_trailing = false; bool recompress = false; bool to_stdout = false; invocation_name = argv[0]; @@ -563,50 +589,51 @@ int main( const int argc, const char * const argv[] ) { show_error( "Bad library version. At least lzlib 1.0 is required." ); return 1; } - const long num_online = std::max( 1L, sysconf( _SC_NPROCESSORS_ONLN ) ); - long max_workers = sysconf( _SC_THREAD_THREADS_MAX ); - if( max_workers < 1 || max_workers > INT_MAX / (int)sizeof (pthread_t) ) - max_workers = INT_MAX / sizeof (pthread_t); - - enum Optcode { opt_dbg = 256 }; + enum { opt_dbg = 256, opt_lt }; const Arg_parser::Option options[] = { - { '0', "fast", Arg_parser::no }, - { '1', 0, Arg_parser::no }, - { '2', 0, Arg_parser::no }, - { '3', 0, Arg_parser::no }, - { '4', 0, Arg_parser::no }, - { '5', 0, Arg_parser::no }, - { '6', 0, Arg_parser::no }, - { '7', 0, Arg_parser::no }, - { '8', 0, Arg_parser::no }, - { '9', "best", Arg_parser::no }, - { 'a', "trailing-error", Arg_parser::no }, - { 'b', "member-size", Arg_parser::yes }, - { 'B', "data-size", Arg_parser::yes }, - { 'c', "stdout", Arg_parser::no }, - { 'd', "decompress", Arg_parser::no }, - { 'f', "force", Arg_parser::no }, - { 'F', "recompress", Arg_parser::no }, - { 'h', "help", Arg_parser::no }, - { 'k', "keep", Arg_parser::no }, - { 'l', "list", Arg_parser::no }, - { 'm', "match-length", Arg_parser::yes }, - { 'n', "threads", Arg_parser::yes }, - { 'o', "output", Arg_parser::yes }, - { 'q', "quiet", Arg_parser::no }, - { 's', "dictionary-size", Arg_parser::yes }, - { 'S', "volume-size", Arg_parser::yes }, - { 't', "test", Arg_parser::no }, - { 'v', "verbose", Arg_parser::no }, - { 'V', "version", Arg_parser::no }, - { opt_dbg, "debug", Arg_parser::yes }, - { 0 , 0, Arg_parser::no } }; + { '0', "fast", Arg_parser::no }, + { '1', 0, Arg_parser::no }, + { '2', 0, Arg_parser::no }, + { '3', 0, Arg_parser::no }, + { '4', 0, Arg_parser::no }, + { '5', 0, Arg_parser::no }, + { '6', 0, Arg_parser::no }, + { '7', 0, Arg_parser::no }, + { '8', 0, Arg_parser::no }, + { '9', "best", Arg_parser::no }, + { 'a', "trailing-error", Arg_parser::no }, + { 'b', "member-size", Arg_parser::yes }, + { 'B', "data-size", Arg_parser::yes }, + { 'c', "stdout", Arg_parser::no }, + { 'd', "decompress", Arg_parser::no }, + { 'f', "force", Arg_parser::no }, + { 'F', "recompress", Arg_parser::no }, + { 'h', "help", Arg_parser::no }, + { 'k', "keep", Arg_parser::no }, + { 'l', "list", Arg_parser::no }, + { 'm', "match-length", Arg_parser::yes }, + { 'n', "threads", Arg_parser::yes }, + { 'o', "output", Arg_parser::yes }, + { 'q', "quiet", Arg_parser::no }, + { 's', "dictionary-size", Arg_parser::yes }, + { 'S', "volume-size", Arg_parser::yes }, + { 't', "test", Arg_parser::no }, + { 'v', "verbose", Arg_parser::no }, + { 'V', "version", Arg_parser::no }, + { opt_dbg, "debug", Arg_parser::yes }, + { opt_lt, "loose-trailing", Arg_parser::no }, + { 0 , 0, Arg_parser::no } }; const Arg_parser parser( argc, argv, options ); if( parser.error().size() ) // bad option { show_error( parser.error().c_str(), 0, true ); return 1; } + const long num_online = std::max( 1L, sysconf( _SC_NPROCESSORS_ONLN ) ); + long max_workers = sysconf( _SC_THREAD_THREADS_MAX ); + if( max_workers < 1 || max_workers > INT_MAX / (int)sizeof (pthread_t) ) + max_workers = INT_MAX / sizeof (pthread_t); + int argind = 0; for( ; argind < parser.arguments(); ++argind ) { @@ -643,6 +670,7 @@ int main( const int argc, const char * const argv[] ) case 'v': if( verbosity < 4 ) ++verbosity; break; case 'V': show_version(); return 0; case opt_dbg: debug_level = getnum( arg, 0, 3 ); break; + case opt_lt: loose_trailing = true; break; default : internal_error( "uncaught option." ); } } // end process options @@ -661,7 +689,7 @@ int main( const int argc, const char * const argv[] ) if( filenames.empty() ) filenames.push_back("-"); if( program_mode == m_list ) - return list_files( filenames, ignore_trailing ); + return list_files( filenames, ignore_trailing, loose_trailing ); if( program_mode == m_test ) outfd = -1; @@ -678,19 +706,30 @@ int main( const int argc, const char * const argv[] ) std::max( data_size, LZ_min_dictionary_size() ); if( num_workers <= 0 ) + { + if( sizeof (void *) <= 4 ) // use less than 2.22 GiB on 32 bit systems + { + const long long limit = ( 27LL << 25 ) + ( 11LL << 27 ); // 4 * 568 MiB + const long long mem = ( 27LL * data_size ) / 8 + + ( fast ? 3LL << 19 : 11LL * encoder_options.dictionary_size ); + const int nmax32 = std::max( limit / mem, 1LL ); + if( max_workers > nmax32 ) max_workers = nmax32; + } num_workers = std::min( num_online, max_workers ); + } if( !to_stdout && program_mode != m_test && ( filenames_given || default_output_filename.size() ) ) set_signals(); - Pretty_print pp( filenames, verbosity ); + Pretty_print pp( filenames ); int retval = 0; bool stdin_used = false; for( unsigned i = 0; i < filenames.size(); ++i ) { std::string input_filename; + int infd; struct stat in_stats; output_filename.clear(); @@ -705,12 +744,12 @@ int main( const int argc, const char * const argv[] ) else { if( program_mode == m_compress ) - set_c_outname( default_output_filename ); + set_c_outname( default_output_filename, false ); else output_filename = default_output_filename; if( !open_outstream( force, true ) ) { if( retval < 1 ) retval = 1; - close( infd ); infd = -1; + close( infd ); continue; } } @@ -728,12 +767,12 @@ int main( const int argc, const char * const argv[] ) else { if( program_mode == m_compress ) - set_c_outname( input_filename ); + set_c_outname( input_filename, true ); else set_d_outname( input_filename, eindex ); if( !open_outstream( force, false ) ) { if( retval < 1 ) retval = 1; - close( infd ); infd = -1; + close( infd ); continue; } } @@ -744,24 +783,22 @@ int main( const int argc, const char * const argv[] ) if( !check_tty( pp.name(), infd, program_mode ) ) { if( retval < 1 ) retval = 1; - if( program_mode == m_test ) { close( infd ); infd = -1; continue; } + if( program_mode == m_test ) { close( infd ); continue; } cleanup_and_fail( retval ); } const struct stat * const in_statsp = input_filename.size() ? &in_stats : 0; const bool infd_isreg = in_statsp && S_ISREG( in_statsp->st_mode ); - if( verbosity >= 1 ) pp(); + const unsigned long long cfile_size = + infd_isreg ? ( in_statsp->st_size + 99 ) / 100 : 0; int tmp; if( program_mode == m_compress ) - { - show_progress( 0, &pp, infd_isreg ? in_statsp->st_size / 100 : 0 ); // init - tmp = compress( data_size, encoder_options.dictionary_size, + tmp = compress( cfile_size, data_size, encoder_options.dictionary_size, encoder_options.match_len_limit, num_workers, infd, outfd, pp, debug_level ); - } else - tmp = decompress( num_workers, infd, outfd, pp, debug_level, - ignore_trailing, infd_isreg ); + tmp = decompress( cfile_size, num_workers, infd, outfd, pp, debug_level, + ignore_trailing, loose_trailing, infd_isreg ); if( tmp > retval ) retval = tmp; if( tmp && program_mode != m_test ) cleanup_and_fail( retval ); @@ -769,14 +806,14 @@ int main( const int argc, const char * const argv[] ) close_and_set_permissions( in_statsp ); if( input_filename.size() ) { - close( infd ); infd = -1; + close( infd ); if( !keep_input_files && !to_stdout && program_mode != m_test ) std::remove( input_filename.c_str() ); } } if( outfd >= 0 && close( outfd ) != 0 ) { - show_error( "Can't close stdout", errno ); + show_error( "Error closing stdout", errno ); if( retval < 1 ) retval = 1; } return retval; -- cgit v1.2.3