Merging upstream version 1.8.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 97 insertions, 40 deletions

diff --git a/main.cc b/main.cc
index d1f76bc..5eab9f9 100644
--- a/main.cc
+++ b/main.cc
@@ -1,6 +1,6 @@
-/*  Plzip - Parallel compressor compatible with lzip
+/*  Plzip - Massively parallel implementation of lzip
     Copyright (C) 2009 Laszlo Ersek.
-    Copyright (C) 2009-2018 Antonio Diaz Diaz.
+    Copyright (C) 2009-2019 Antonio Diaz Diaz.
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -40,20 +40,21 @@
 #include <utime.h>
 #include <sys/stat.h>
 #include <lzlib.h>
-#if defined(__MSVCRT__)
+#if defined(__MSVCRT__) || defined(__OS2__)
 #include <io.h>
+#if defined(__MSVCRT__)
 #define fchmod(x,y) 0
 #define fchown(x,y,z) 0
 #define strtoull std::strtoul
 #define SIGHUP SIGTERM
 #define S_ISSOCK(x) 0
+#ifndef S_IRGRP
 #define S_IRGRP 0
 #define S_IWGRP 0
 #define S_IROTH 0
 #define S_IWOTH 0
 #endif
-#if defined(__OS2__)
-#include <io.h>
+#endif
 #endif
 
 #include "arg_parser.h"
@@ -71,9 +72,8 @@ int verbosity = 0;
 
 namespace {
 
-const char * const Program_name = "Plzip";
 const char * const program_name = "plzip";
-const char * const program_year = "2018";
+const char * const program_year = "2019";
 const char * invocation_name = 0;
 
 const struct { const char * from; const char * to; } known_extensions[] = {
@@ -89,6 +89,8 @@ struct Lzma_options
 
 enum Mode { m_compress, m_decompress, m_list, m_test };
 
+/* Variables used in signal handler context.
+   They are not declared volatile because the handler never returns. */
 std::string output_filename;
 int outfd = -1;
 bool delete_output_on_interrupt = false;
@@ -96,8 +98,22 @@ bool delete_output_on_interrupt = false;
 
 void show_help( const long num_online )
   {
-  std::printf( "%s - Parallel compressor compatible with lzip.\n", Program_name );
-  std::printf( "\nUsage: %s [options] [files]\n", invocation_name );
+  std::printf( "Plzip is a massively parallel (multi-threaded) implementation of lzip, fully\n"
+               "compatible with lzip 1.4 or newer. Plzip uses the lzlib compression library.\n"
+               "\nLzip is a lossless data compressor with a user interface similar to the\n"
+               "one of gzip or bzip2. Lzip can compress about as fast as gzip (lzip -0)\n"
+               "or compress most files more than bzip2 (lzip -9). Decompression speed is\n"
+               "intermediate between gzip and bzip2. Lzip is better than gzip and bzip2\n"
+               "from a data recovery perspective. Lzip has been designed, written and\n"
+               "tested with great care to replace gzip and bzip2 as the standard\n"
+               "general-purpose compressed format for unix-like systems.\n"
+               "\nPlzip can compress/decompress large files on multiprocessor machines\n"
+               "much faster than lzip, at the cost of a slightly reduced compression\n"
+               "ratio (0.4 to 2 percent larger compressed files). Note that the number\n"
+               "of usable threads is limited by file size; on files larger than a few GB\n"
+               "plzip can use hundreds of processors, but on files of only a few MB\n"
+               "plzip is no faster than lzip.\n"
+               "\nUsage: %s [options] [files]\n", invocation_name );
   std::printf( "\nOptions:\n"
                "  -h, --help                     display this help and exit\n"
                "  -V, --version                  output version information and exit\n"
@@ -120,6 +136,8 @@ void show_help( const long num_online )
                "      --fast                     alias for -0\n"
                "      --best                     alias for -9\n"
                "      --loose-trailing           allow trailing data seeming corrupt header\n"
+               "      --in-slots=<n>             number of 1 MiB input packets buffered [4]\n"
+               "      --out-slots=<n>            number of 1 MiB output packets buffered [64]\n"
                , num_online );
   if( verbosity >= 1 )
     {
@@ -263,7 +281,7 @@ int get_dict_size( const char * const arg )
   const long bits = std::strtol( arg, &tail, 0 );
   if( bits >= LZ_min_dictionary_bits() &&
       bits <= LZ_max_dictionary_bits() && *tail == 0 )
-    return ( 1 << bits );
+    return 1 << bits;
   int dictionary_size = getnum( arg, LZ_min_dictionary_size(),
                                      LZ_max_dictionary_size() );
   if( dictionary_size == 65535 ) ++dictionary_size;	// no fast encoder
@@ -409,6 +427,14 @@ bool check_tty( const char * const input_filename, const int infd,
   return true;
   }
 
+
+void set_signals( void (*action)(int) )
+  {
+  std::signal( SIGHUP, action );
+  std::signal( SIGINT, action );
+  std::signal( SIGTERM, action );
+  }
+
 } // end namespace
 
 // This can be called from any thread, main thread or sub-threads alike,
@@ -420,6 +446,7 @@ void cleanup_and_fail( const int retval )
   // only one thread can delete and exit
   static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
 
+  set_signals( SIG_IGN );			// ignore signals
   pthread_mutex_lock( &mutex );		// ignore errors to avoid loop
   const int saved_verbosity = verbosity;
   verbosity = -1;		// suppress messages from other threads
@@ -440,6 +467,13 @@ void cleanup_and_fail( const int retval )
 
 namespace {
 
+extern "C" void signal_handler( int )
+  {
+  show_error( "Control-C or similar caught, quitting." );
+  cleanup_and_fail( 1 );
+  }
+
+
      // Set permissions, owner and times.
 void close_and_set_permissions( const struct stat * const in_statsp )
   {
@@ -473,21 +507,6 @@ void close_and_set_permissions( const struct stat * const in_statsp )
     show_error( "Can't change output file attributes." );
   }
 
-
-extern "C" void signal_handler( int )
-  {
-  show_error( "Control-C or similar caught, quitting." );
-  cleanup_and_fail( 1 );
-  }
-
-
-void set_signals()
-  {
-  std::signal( SIGHUP, signal_handler );
-  std::signal( SIGINT, signal_handler );
-  std::signal( SIGTERM, signal_handler );
-  }
-
 } // end namespace
 
 
@@ -495,11 +514,9 @@ void show_error( const char * const msg, const int errcode, const bool help )
   {
   if( verbosity < 0 ) return;
   if( msg && msg[0] )
-    {
-    std::fprintf( stderr, "%s: %s", program_name, msg );
-    if( errcode > 0 ) std::fprintf( stderr, ": %s", std::strerror( errcode ) );
-    std::fputc( '\n', stderr );
-    }
+    std::fprintf( stderr, "%s: %s%s%s\n", program_name, msg,
+                  ( errcode > 0 ) ? ": " : "",
+                  ( errcode > 0 ) ? std::strerror( errcode ) : "" );
   if( help )
     std::fprintf( stderr, "Try '%s --help' for more information.\n",
                   invocation_name );
@@ -509,10 +526,10 @@ void show_error( const char * const msg, const int errcode, const bool help )
 void show_file_error( const char * const filename, const char * const msg,
                       const int errcode )
   {
-  if( verbosity < 0 ) return;
-  std::fprintf( stderr, "%s: %s: %s", program_name, filename, msg );
-  if( errcode > 0 ) std::fprintf( stderr, ": %s", std::strerror( errcode ) );
-  std::fputc( '\n', stderr );
+  if( verbosity >= 0 )
+    std::fprintf( stderr, "%s: %s: %s%s%s\n", program_name, filename, msg,
+                  ( errcode > 0 ) ? ": " : "",
+                  ( errcode > 0 ) ? std::strerror( errcode ) : "" );
   }
 
 
@@ -554,6 +571,26 @@ void show_progress( const unsigned long long packet_size,
   }
 
 
+#if defined(__MSVCRT__)
+#include <windows.h>
+#define _SC_NPROCESSORS_ONLN   1
+#define _SC_THREAD_THREADS_MAX 2
+
+long sysconf( int flag )
+  {
+  if( flag == _SC_NPROCESSORS_ONLN )
+    {
+    SYSTEM_INFO si;
+    GetSystemInfo( &si );
+    return si.dwNumberOfProcessors;
+    }
+  if( flag != _SC_THREAD_THREADS_MAX ) errno = EINVAL;
+  return -1;		// unlimited threads or error
+  }
+
+#endif	// __MSVCRT__
+
+
 int main( const int argc, const char * const argv[] )
   {
   /* Mapping from gzip/bzip2 style 1..9 compression modes
@@ -576,6 +613,8 @@ int main( const int argc, const char * const argv[] )
   int data_size = 0;
   int debug_level = 0;
   int num_workers = 0;		// start this many worker threads
+  int in_slots = 4;
+  int out_slots = 64;
   Mode program_mode = m_compress;
   bool force = false;
   bool ignore_trailing = true;
@@ -589,7 +628,7 @@ int main( const int argc, const char * const argv[] )
     { show_error( "Bad library version. At least lzlib 1.0 is required." );
       return 1; }
 
-  enum { opt_dbg = 256, opt_lt };
+  enum { opt_dbg = 256, opt_in, opt_lt, opt_out };
   const Arg_parser::Option options[] =
     {
     { '0', "fast",              Arg_parser::no  },
@@ -622,7 +661,9 @@ int main( const int argc, const char * const argv[] )
     { 'v', "verbose",           Arg_parser::no  },
     { 'V', "version",           Arg_parser::no  },
     { opt_dbg, "debug",         Arg_parser::yes },
+    { opt_in, "in-slots",       Arg_parser::yes },
     { opt_lt, "loose-trailing", Arg_parser::no  },
+    { opt_out, "out-slots",     Arg_parser::yes },
     {  0 , 0,                   Arg_parser::no  } };
 
   const Arg_parser parser( argc, argv, options );
@@ -670,7 +711,9 @@ int main( const int argc, const char * const argv[] )
       case 'v': if( verbosity < 4 ) ++verbosity; break;
       case 'V': show_version(); return 0;
       case opt_dbg: debug_level = getnum( arg, 0, 3 ); break;
+      case opt_in: in_slots = getnum( arg, 1, 64 ); break;
       case opt_lt: loose_trailing = true; break;
+      case opt_out: out_slots = getnum( arg, 1, 1024 ); break;
       default : internal_error( "uncaught option." );
       }
     } // end process options
@@ -707,8 +750,9 @@ int main( const int argc, const char * const argv[] )
 
   if( num_workers <= 0 )
     {
-    if( sizeof (void *) <= 4 )	// use less than 2.22 GiB on 32 bit systems
+    if( program_mode == m_compress && sizeof (void *) <= 4 )
       {
+      // use less than 2.22 GiB on 32 bit systems
       const long long limit = ( 27LL << 25 ) + ( 11LL << 27 );	// 4 * 568 MiB
       const long long mem = ( 27LL * data_size ) / 8 +
         ( fast ? 3LL << 19 : 11LL * encoder_options.dictionary_size );
@@ -720,10 +764,11 @@ int main( const int argc, const char * const argv[] )
 
   if( !to_stdout && program_mode != m_test &&
       ( filenames_given || default_output_filename.size() ) )
-    set_signals();
+    set_signals( signal_handler );
 
   Pretty_print pp( filenames );
 
+  int failed_tests = 0;
   int retval = 0;
   bool stdin_used = false;
   for( unsigned i = 0; i < filenames.size(); ++i )
@@ -798,15 +843,23 @@ int main( const int argc, const char * const argv[] )
                       num_workers, infd, outfd, pp, debug_level );
     else
       tmp = decompress( cfile_size, num_workers, infd, outfd, pp, debug_level,
-                        ignore_trailing, loose_trailing, infd_isreg );
+                        in_slots, out_slots, ignore_trailing, loose_trailing,
+                        infd_isreg );
+    if( close( infd ) != 0 )
+      {
+      show_error( input_filename.size() ? "Error closing input file" :
+                                          "Error closing stdin", errno );
+      if( tmp < 1 ) tmp = 1;
+      }
     if( tmp > retval ) retval = tmp;
-    if( tmp && program_mode != m_test ) cleanup_and_fail( retval );
+    if( tmp )
+      { if( program_mode != m_test ) cleanup_and_fail( retval );
+        else ++failed_tests; }
 
     if( delete_output_on_interrupt )
       close_and_set_permissions( in_statsp );
     if( input_filename.size() )
       {
-      close( infd );
       if( !keep_input_files && !to_stdout && program_mode != m_test )
         std::remove( input_filename.c_str() );
       }
@@ -816,5 +869,9 @@ int main( const int argc, const char * const argv[] )
     show_error( "Error closing stdout", errno );
     if( retval < 1 ) retval = 1;
     }
+  if( failed_tests > 0 && verbosity >= 1 && filenames.size() > 1 )
+    std::fprintf( stderr, "%s: warning: %d %s failed the test.\n",
+                  program_name, failed_tests,
+                  ( failed_tests == 1 ) ? "file" : "files" );
   return retval;
   }