summaryrefslogtreecommitdiffstats
path: root/main.cc
diff options
context:
space:
mode:
Diffstat (limited to 'main.cc')
-rw-r--r--main.cc137
1 files changed, 97 insertions, 40 deletions
diff --git a/main.cc b/main.cc
index d1f76bc..5eab9f9 100644
--- a/main.cc
+++ b/main.cc
@@ -1,6 +1,6 @@
-/* Plzip - Parallel compressor compatible with lzip
+/* Plzip - Massively parallel implementation of lzip
Copyright (C) 2009 Laszlo Ersek.
- Copyright (C) 2009-2018 Antonio Diaz Diaz.
+ Copyright (C) 2009-2019 Antonio Diaz Diaz.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -40,20 +40,21 @@
#include <utime.h>
#include <sys/stat.h>
#include <lzlib.h>
-#if defined(__MSVCRT__)
+#if defined(__MSVCRT__) || defined(__OS2__)
#include <io.h>
+#if defined(__MSVCRT__)
#define fchmod(x,y) 0
#define fchown(x,y,z) 0
#define strtoull std::strtoul
#define SIGHUP SIGTERM
#define S_ISSOCK(x) 0
+#ifndef S_IRGRP
#define S_IRGRP 0
#define S_IWGRP 0
#define S_IROTH 0
#define S_IWOTH 0
#endif
-#if defined(__OS2__)
-#include <io.h>
+#endif
#endif
#include "arg_parser.h"
@@ -71,9 +72,8 @@ int verbosity = 0;
namespace {
-const char * const Program_name = "Plzip";
const char * const program_name = "plzip";
-const char * const program_year = "2018";
+const char * const program_year = "2019";
const char * invocation_name = 0;
const struct { const char * from; const char * to; } known_extensions[] = {
@@ -89,6 +89,8 @@ struct Lzma_options
enum Mode { m_compress, m_decompress, m_list, m_test };
+/* Variables used in signal handler context.
+ They are not declared volatile because the handler never returns. */
std::string output_filename;
int outfd = -1;
bool delete_output_on_interrupt = false;
@@ -96,8 +98,22 @@ bool delete_output_on_interrupt = false;
void show_help( const long num_online )
{
- std::printf( "%s - Parallel compressor compatible with lzip.\n", Program_name );
- std::printf( "\nUsage: %s [options] [files]\n", invocation_name );
+ std::printf( "Plzip is a massively parallel (multi-threaded) implementation of lzip, fully\n"
+ "compatible with lzip 1.4 or newer. Plzip uses the lzlib compression library.\n"
+ "\nLzip is a lossless data compressor with a user interface similar to the\n"
+ "one of gzip or bzip2. Lzip can compress about as fast as gzip (lzip -0)\n"
+ "or compress most files more than bzip2 (lzip -9). Decompression speed is\n"
+ "intermediate between gzip and bzip2. Lzip is better than gzip and bzip2\n"
+ "from a data recovery perspective. Lzip has been designed, written and\n"
+ "tested with great care to replace gzip and bzip2 as the standard\n"
+ "general-purpose compressed format for unix-like systems.\n"
+ "\nPlzip can compress/decompress large files on multiprocessor machines\n"
+ "much faster than lzip, at the cost of a slightly reduced compression\n"
+ "ratio (0.4 to 2 percent larger compressed files). Note that the number\n"
+ "of usable threads is limited by file size; on files larger than a few GB\n"
+ "plzip can use hundreds of processors, but on files of only a few MB\n"
+ "plzip is no faster than lzip.\n"
+ "\nUsage: %s [options] [files]\n", invocation_name );
std::printf( "\nOptions:\n"
" -h, --help display this help and exit\n"
" -V, --version output version information and exit\n"
@@ -120,6 +136,8 @@ void show_help( const long num_online )
" --fast alias for -0\n"
" --best alias for -9\n"
" --loose-trailing allow trailing data seeming corrupt header\n"
+ " --in-slots=<n> number of 1 MiB input packets buffered [4]\n"
+ " --out-slots=<n> number of 1 MiB output packets buffered [64]\n"
, num_online );
if( verbosity >= 1 )
{
@@ -263,7 +281,7 @@ int get_dict_size( const char * const arg )
const long bits = std::strtol( arg, &tail, 0 );
if( bits >= LZ_min_dictionary_bits() &&
bits <= LZ_max_dictionary_bits() && *tail == 0 )
- return ( 1 << bits );
+ return 1 << bits;
int dictionary_size = getnum( arg, LZ_min_dictionary_size(),
LZ_max_dictionary_size() );
if( dictionary_size == 65535 ) ++dictionary_size; // no fast encoder
@@ -409,6 +427,14 @@ bool check_tty( const char * const input_filename, const int infd,
return true;
}
+
+void set_signals( void (*action)(int) )
+ {
+ std::signal( SIGHUP, action );
+ std::signal( SIGINT, action );
+ std::signal( SIGTERM, action );
+ }
+
} // end namespace
// This can be called from any thread, main thread or sub-threads alike,
@@ -420,6 +446,7 @@ void cleanup_and_fail( const int retval )
// only one thread can delete and exit
static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;
+ set_signals( SIG_IGN ); // ignore signals
pthread_mutex_lock( &mutex ); // ignore errors to avoid loop
const int saved_verbosity = verbosity;
verbosity = -1; // suppress messages from other threads
@@ -440,6 +467,13 @@ void cleanup_and_fail( const int retval )
namespace {
+extern "C" void signal_handler( int )
+ {
+ show_error( "Control-C or similar caught, quitting." );
+ cleanup_and_fail( 1 );
+ }
+
+
// Set permissions, owner and times.
void close_and_set_permissions( const struct stat * const in_statsp )
{
@@ -473,21 +507,6 @@ void close_and_set_permissions( const struct stat * const in_statsp )
show_error( "Can't change output file attributes." );
}
-
-extern "C" void signal_handler( int )
- {
- show_error( "Control-C or similar caught, quitting." );
- cleanup_and_fail( 1 );
- }
-
-
-void set_signals()
- {
- std::signal( SIGHUP, signal_handler );
- std::signal( SIGINT, signal_handler );
- std::signal( SIGTERM, signal_handler );
- }
-
} // end namespace
@@ -495,11 +514,9 @@ void show_error( const char * const msg, const int errcode, const bool help )
{
if( verbosity < 0 ) return;
if( msg && msg[0] )
- {
- std::fprintf( stderr, "%s: %s", program_name, msg );
- if( errcode > 0 ) std::fprintf( stderr, ": %s", std::strerror( errcode ) );
- std::fputc( '\n', stderr );
- }
+ std::fprintf( stderr, "%s: %s%s%s\n", program_name, msg,
+ ( errcode > 0 ) ? ": " : "",
+ ( errcode > 0 ) ? std::strerror( errcode ) : "" );
if( help )
std::fprintf( stderr, "Try '%s --help' for more information.\n",
invocation_name );
@@ -509,10 +526,10 @@ void show_error( const char * const msg, const int errcode, const bool help )
void show_file_error( const char * const filename, const char * const msg,
const int errcode )
{
- if( verbosity < 0 ) return;
- std::fprintf( stderr, "%s: %s: %s", program_name, filename, msg );
- if( errcode > 0 ) std::fprintf( stderr, ": %s", std::strerror( errcode ) );
- std::fputc( '\n', stderr );
+ if( verbosity >= 0 )
+ std::fprintf( stderr, "%s: %s: %s%s%s\n", program_name, filename, msg,
+ ( errcode > 0 ) ? ": " : "",
+ ( errcode > 0 ) ? std::strerror( errcode ) : "" );
}
@@ -554,6 +571,26 @@ void show_progress( const unsigned long long packet_size,
}
+#if defined(__MSVCRT__)
+#include <windows.h>
+#define _SC_NPROCESSORS_ONLN 1
+#define _SC_THREAD_THREADS_MAX 2
+
+long sysconf( int flag )
+ {
+ if( flag == _SC_NPROCESSORS_ONLN )
+ {
+ SYSTEM_INFO si;
+ GetSystemInfo( &si );
+ return si.dwNumberOfProcessors;
+ }
+ if( flag != _SC_THREAD_THREADS_MAX ) errno = EINVAL;
+ return -1; // unlimited threads or error
+ }
+
+#endif // __MSVCRT__
+
+
int main( const int argc, const char * const argv[] )
{
/* Mapping from gzip/bzip2 style 1..9 compression modes
@@ -576,6 +613,8 @@ int main( const int argc, const char * const argv[] )
int data_size = 0;
int debug_level = 0;
int num_workers = 0; // start this many worker threads
+ int in_slots = 4;
+ int out_slots = 64;
Mode program_mode = m_compress;
bool force = false;
bool ignore_trailing = true;
@@ -589,7 +628,7 @@ int main( const int argc, const char * const argv[] )
{ show_error( "Bad library version. At least lzlib 1.0 is required." );
return 1; }
- enum { opt_dbg = 256, opt_lt };
+ enum { opt_dbg = 256, opt_in, opt_lt, opt_out };
const Arg_parser::Option options[] =
{
{ '0', "fast", Arg_parser::no },
@@ -622,7 +661,9 @@ int main( const int argc, const char * const argv[] )
{ 'v', "verbose", Arg_parser::no },
{ 'V', "version", Arg_parser::no },
{ opt_dbg, "debug", Arg_parser::yes },
+ { opt_in, "in-slots", Arg_parser::yes },
{ opt_lt, "loose-trailing", Arg_parser::no },
+ { opt_out, "out-slots", Arg_parser::yes },
{ 0 , 0, Arg_parser::no } };
const Arg_parser parser( argc, argv, options );
@@ -670,7 +711,9 @@ int main( const int argc, const char * const argv[] )
case 'v': if( verbosity < 4 ) ++verbosity; break;
case 'V': show_version(); return 0;
case opt_dbg: debug_level = getnum( arg, 0, 3 ); break;
+ case opt_in: in_slots = getnum( arg, 1, 64 ); break;
case opt_lt: loose_trailing = true; break;
+ case opt_out: out_slots = getnum( arg, 1, 1024 ); break;
default : internal_error( "uncaught option." );
}
} // end process options
@@ -707,8 +750,9 @@ int main( const int argc, const char * const argv[] )
if( num_workers <= 0 )
{
- if( sizeof (void *) <= 4 ) // use less than 2.22 GiB on 32 bit systems
+ if( program_mode == m_compress && sizeof (void *) <= 4 )
{
+ // use less than 2.22 GiB on 32 bit systems
const long long limit = ( 27LL << 25 ) + ( 11LL << 27 ); // 4 * 568 MiB
const long long mem = ( 27LL * data_size ) / 8 +
( fast ? 3LL << 19 : 11LL * encoder_options.dictionary_size );
@@ -720,10 +764,11 @@ int main( const int argc, const char * const argv[] )
if( !to_stdout && program_mode != m_test &&
( filenames_given || default_output_filename.size() ) )
- set_signals();
+ set_signals( signal_handler );
Pretty_print pp( filenames );
+ int failed_tests = 0;
int retval = 0;
bool stdin_used = false;
for( unsigned i = 0; i < filenames.size(); ++i )
@@ -798,15 +843,23 @@ int main( const int argc, const char * const argv[] )
num_workers, infd, outfd, pp, debug_level );
else
tmp = decompress( cfile_size, num_workers, infd, outfd, pp, debug_level,
- ignore_trailing, loose_trailing, infd_isreg );
+ in_slots, out_slots, ignore_trailing, loose_trailing,
+ infd_isreg );
+ if( close( infd ) != 0 )
+ {
+ show_error( input_filename.size() ? "Error closing input file" :
+ "Error closing stdin", errno );
+ if( tmp < 1 ) tmp = 1;
+ }
if( tmp > retval ) retval = tmp;
- if( tmp && program_mode != m_test ) cleanup_and_fail( retval );
+ if( tmp )
+ { if( program_mode != m_test ) cleanup_and_fail( retval );
+ else ++failed_tests; }
if( delete_output_on_interrupt )
close_and_set_permissions( in_statsp );
if( input_filename.size() )
{
- close( infd );
if( !keep_input_files && !to_stdout && program_mode != m_test )
std::remove( input_filename.c_str() );
}
@@ -816,5 +869,9 @@ int main( const int argc, const char * const argv[] )
show_error( "Error closing stdout", errno );
if( retval < 1 ) retval = 1;
}
+ if( failed_tests > 0 && verbosity >= 1 && filenames.size() > 1 )
+ std::fprintf( stderr, "%s: warning: %d %s failed the test.\n",
+ program_name, failed_tests,
+ ( failed_tests == 1 ) ? "file" : "files" );
return retval;
}