path: root/main.cc

/* Tarlz - Archiver with multimember lzip compression
   Copyright (C) 2013-2021 Antonio Diaz Diaz.

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation, either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
/*
   Exit status: 0 for a normal exit, 1 for environmental problems (file not
   found, files differ, invalid flags, I/O errors, etc), 2 to indicate a
   corrupt or invalid input file, 3 for an internal consistency error
   (eg, bug) which caused tarlz to panic.
*/

#define _FILE_OFFSET_BITS 64

#include <cctype>
#include <cerrno>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <ctime>
#include <string>
#include <vector>
#include <fcntl.h>
#include <pthread.h>
#include <stdint.h>
#include <unistd.h>
#include <sys/stat.h>
#include <grp.h>
#include <pwd.h>
#include <lzlib.h>
#if defined(__OS2__)
#include <io.h>
#endif

#include "arg_parser.h"
#include "tarlz.h"

#ifndef O_BINARY
#define O_BINARY 0
#endif

#if CHAR_BIT != 8
#error "Environments where CHAR_BIT != 8 are not supported."
#endif

int verbosity = 0;

namespace {

const char * const program_name = "tarlz";
const char * const program_year = "2021";
const char * invocation_name = program_name;		// default value


void show_help( const long num_online )
  {
  std::printf( "Tarlz is a massively parallel (multi-threaded) combined implementation of\n"
               "the tar archiver and the lzip compressor. Tarlz uses the compression library\n"
               "lzlib.\n"
               "\nTarlz creates, lists, and extracts archives in a simplified and safer\n"
               "variant of the POSIX pax format compressed in lzip format, keeping the\n"
               "alignment between tar members and lzip members. The resulting multimember\n"
               "tar.lz archive is fully backward compatible with standard tar tools like GNU\n"
               "tar, which treat it like any other tar.lz archive. Tarlz can append files to\n"
               "the end of such compressed archives.\n"
               "\nKeeping the alignment between tar members and lzip members has two\n"
               "advantages. It adds an indexed lzip layer on top of the tar archive, making\n"
               "it possible to decode the archive safely in parallel. It also minimizes the\n"
               "amount of data lost in case of corruption.\n"
               "\nThe tarlz file format is a safe POSIX-style backup format. In case of\n"
               "corruption, tarlz can extract all the undamaged members from the tar.lz\n"
               "archive, skipping over the damaged members, just like the standard\n"
               "(uncompressed) tar. Moreover, the option '--keep-damaged' can be used to\n"
               "recover as much data as possible from each damaged member, and lziprecover\n"
               "can be used to recover some of the damaged members.\n"
               "\nUsage: %s [options] [files]\n", invocation_name );
  std::printf( "\nOptions:\n"
               "      --help                  display this help and exit\n"
               "  -V, --version               output version information and exit\n"
               "  -A, --concatenate           append archives to the end of an archive\n"
               "  -B, --data-size=<bytes>     set target size of input data blocks [2x8=16 MiB]\n"
               "  -c, --create                create a new archive\n"
               "  -C, --directory=<dir>       change to directory <dir>\n"
               "  -d, --diff                  find differences between archive and file system\n"
               "      --ignore-ids            ignore differences in owner and group IDs\n"
               "      --delete                delete files/directories from an archive\n"
               "      --exclude=<pattern>     exclude files matching a shell pattern\n"
               "  -f, --file=<archive>        use archive file <archive>\n"
               "  -h, --dereference           follow symlinks; archive the files they point to\n"
               "      --mtime=<date>          use <date> as mtime for files added to archive\n"
               "  -n, --threads=<n>           set number of (de)compression threads [%ld]\n"
               "  -p, --preserve-permissions  don't subtract the umask on extraction\n"
               "  -q, --quiet                 suppress all messages\n"
               "  -r, --append                append files to the end of an archive\n"
               "  -t, --list                  list the contents of an archive\n"
               "  -v, --verbose               verbosely list files processed\n"
               "  -x, --extract               extract files/directories from an archive\n"
               "  -0 .. -9                    set compression level [default 6]\n"
               "      --uncompressed          don't compress the archive created\n"
               "      --asolid                create solidly compressed appendable archive\n"
               "      --bsolid                create per block compressed archive (default)\n"
               "      --dsolid                create per directory compressed archive\n"
               "      --no-solid              create per file compressed archive\n"
               "      --solid                 create solidly compressed archive\n"
               "      --anonymous             equivalent to '--owner=root --group=root'\n"
               "      --owner=<owner>         use <owner> name/ID for files added to archive\n"
               "      --group=<group>         use <group> name/ID for files added to archive\n"
               "      --keep-damaged          don't delete partially extracted files\n"
               "      --missing-crc           exit with error status if missing extended CRC\n"
               "      --out-slots=<n>         number of 1 MiB output packets buffered [64]\n"
               "      --check-lib             compare version of lzlib.h with liblz.{a,so}\n"
/*              "      --permissive            allow repeated extended headers and records\n"*/,
               num_online );
  if( verbosity >= 1 )
    {
    std::printf( "      --debug=<level>         (0-1) print debug statistics to stderr\n" );
    }
  std::printf( "\nExit status: 0 for a normal exit, 1 for environmental problems (file not\n"
               "found, files differ, invalid flags, I/O errors, etc), 2 to indicate a\n"
               "corrupt or invalid input file, 3 for an internal consistency error (eg, bug)\n"
               "which caused tarlz to panic.\n"
               "\nReport bugs to lzip-bug@nongnu.org\n"
               "Tarlz home page: http://www.nongnu.org/lzip/tarlz.html\n" );
  }


void show_version()
  {
  std::printf( "%s %s\n", program_name, PROGVERSION );
  std::printf( "Copyright (C) %s Antonio Diaz Diaz.\n", program_year );
  std::printf( "Using lzlib %s\n", LZ_version() );
  std::printf( "License GPLv2+: GNU GPL version 2 or later <http://gnu.org/licenses/gpl.html>\n"
               "This is free software: you are free to change and redistribute it.\n"
               "There is NO WARRANTY, to the extent permitted by law.\n" );
  }


int check_lib()
  {
  bool warning = false;
  if( std::strcmp( LZ_version_string, LZ_version() ) != 0 )
    { warning = true;
      if( verbosity >= 0 )
        std::printf( "warning: LZ_version_string != LZ_version() (%s vs %s)\n",
                     LZ_version_string, LZ_version() ); }
#if defined LZ_API_VERSION && LZ_API_VERSION >= 1012
  if( LZ_API_VERSION != LZ_api_version() )
    { warning = true;
      if( verbosity >= 0 )
        std::printf( "warning: LZ_API_VERSION != LZ_api_version() (%u vs %u)\n",
                     LZ_API_VERSION, LZ_api_version() ); }
#endif
  if( verbosity >= 1 )
    {
    std::printf( "Using lzlib %s\n", LZ_version() );
#if !defined LZ_API_VERSION
    std::fputs( "LZ_API_VERSION is not defined.\n", stdout );
#elif LZ_API_VERSION >= 1012
    std::printf( "Using LZ_API_VERSION = %u\n", LZ_api_version() );
#else
    std::printf( "Compiled with LZ_API_VERSION = %u. "
                 "Using an unknown LZ_API_VERSION\n", LZ_API_VERSION );
#endif
    }
  return warning;
  }


unsigned long long getnum( const char * const ptr,
                           const unsigned long long llimit,
                           const unsigned long long ulimit )
  {
  char * tail;
  errno = 0;
  unsigned long long result = strtoull( ptr, &tail, 0 );
  if( tail == ptr )
    {
    show_error( "Bad or missing numerical argument.", 0, true );
    std::exit( 1 );
    }

  if( !errno && tail[0] )
    {
    const unsigned factor = ( tail[1] == 'i' ) ? 1024 : 1000;
    int exponent = 0;				// 0 = bad multiplier
    switch( tail[0] )
      {
      case 'Y': exponent = 8; break;
      case 'Z': exponent = 7; break;
      case 'E': exponent = 6; break;
      case 'P': exponent = 5; break;
      case 'T': exponent = 4; break;
      case 'G': exponent = 3; break;
      case 'M': exponent = 2; break;
      case 'K': if( factor == 1024 ) exponent = 1; break;
      case 'k': if( factor == 1000 ) exponent = 1; break;
      }
    if( exponent <= 0 )
      {
      show_error( "Bad multiplier in numerical argument.", 0, true );
      std::exit( 1 );
      }
    for( int i = 0; i < exponent; ++i )
      {
      if( ulimit / factor >= result ) result *= factor;
      else { errno = ERANGE; break; }
      }
    }
  if( !errno && ( result < llimit || result > ulimit ) ) errno = ERANGE;
  if( errno )
    {
    show_error( "Numerical argument out of limits." );
    std::exit( 1 );
    }
  return result;
  }


void set_archive_name( std::string & archive_name, const std::string & new_name )
  {
  static bool first_call = true;

  if( first_call ) { if( new_name != "-" ) archive_name = new_name;
                     first_call = false; return; }
  show_error( "Only one archive can be specified.", 0, true );
  std::exit( 1 );
  }


void set_mode( Program_mode & program_mode, const Program_mode new_mode )
  {
  if( program_mode != m_none && program_mode != new_mode )
    {
    show_error( "Only one operation can be specified.", 0, true );
    std::exit( 1 );
    }
  program_mode = new_mode;
  }


void set_mtime( long long & mtime, const char * arg )
  {
  if( *arg == '@' )
    { mtime = getnum( arg + 1, 0, ( 1ULL << 33 ) - 1 ); return; }
  else if( *arg == '.' || *arg == '/' )
    {
    struct stat st;
    if( stat( arg, &st ) == 0 ) { mtime = st.st_mtime; return; }
    show_file_error( arg, "Can't stat", errno ); std::exit( 1 );
    }
  else		// format 'YYYY-MM-DD HH:MM:SS'
    {
    unsigned y, mo, d, h, m, s;
    const int n = std::sscanf( arg, "%u-%u-%u %u:%u:%u",
                               &y, &mo, &d, &h, &m, &s );
    if( n == 6 && y >= 1970 && mo >= 1 )
      {
      struct tm t;
      t.tm_year = y - 1900; t.tm_mon = mo - 1; t.tm_mday = d;
      t.tm_hour = h; t.tm_min = m; t.tm_sec = s; t.tm_isdst = -1;
      mtime = std::mktime( &t ); if( mtime >= 0 ) return;
      }
    }
  show_error( "Invalid mtime.", 0, true ); std::exit( 1 );
  }


void set_owner( int & owner, const char * const arg )
  {
  const struct passwd * const pw = getpwnam( arg );
  if( pw ) owner = pw->pw_uid;
  else if( std::isdigit( (unsigned char)arg[0] ) )
    owner = getnum( arg, 0, INT_MAX );
  else if( std::strcmp( arg, "root" ) == 0 ) owner = 0;
  else { show_file_error( arg, "Invalid owner" ); std::exit( 1 ); }
  }

void set_group( int & group, const char * const arg )
  {
  const struct group * const gr = getgrnam( arg );
  if( gr ) group = gr->gr_gid;
  else if( std::isdigit( (unsigned char)arg[0] ) )
    group = getnum( arg, 0, INT_MAX );
  else if( std::strcmp( arg, "root" ) == 0 ) group = 0;
  else { show_file_error( arg, "Invalid group" ); std::exit( 1 ); }
  }

} // end namespace


int hstat( const char * const filename, struct stat * const st,
           const bool dereference )
  { return dereference ? stat( filename, st ) : lstat( filename, st ); }


int open_instream( const std::string & name )
  {
  const int infd = open( name.c_str(), O_RDONLY | O_BINARY );
  if( infd < 0 )
    show_file_error( name.c_str(), "Can't open for reading", errno );
  return infd;
  }


int open_outstream( const std::string & name, const bool create,
                    Resizable_buffer * const rbufp )
  {
  const int flags = (create ? O_CREAT | O_WRONLY | O_TRUNC : O_RDWR) | O_BINARY;
  const mode_t outfd_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;

  const int outfd = open( name.c_str(), flags, outfd_mode );
  if( outfd < 0 )
    {
    const char * msg = create ? "Can't create file" : "Error opening file";
    if( !rbufp ) show_file_error( name.c_str(), msg, errno );
    else
      snprintf( (*rbufp)(), (*rbufp).size(), "%s: %s: %s\n", name.c_str(),
                msg, std::strerror( errno ) );
    }
  return outfd;
  }


/* This can be called from any thread, main thread or sub-threads alike,
   since they all call common helper functions that call cleanup_and_fail()
   in case of an error.
*/
void cleanup_and_fail( const int retval )
  {
  // calling 'exit' more than once results in undefined behavior
  static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;

  pthread_mutex_lock( &mutex );		// ignore errors to avoid loop
  std::exit( retval );
  }


void show_error( const char * const msg, const int errcode, const bool help )
  {
  if( verbosity < 0 ) return;
  if( msg && msg[0] )
    std::fprintf( stderr, "%s: %s%s%s\n", program_name, msg,
                  ( errcode > 0 ) ? ": " : "",
                  ( errcode > 0 ) ? std::strerror( errcode ) : "" );
  if( help )
    std::fprintf( stderr, "Try '%s --help' for more information.\n",
                  invocation_name );
  }


void format_file_error( std::string & estr, const char * const filename,
                        const char * const msg, const int errcode )
  {
  if( verbosity < 0 ) return;
  estr += program_name; estr += ": "; estr += filename; estr += ": ";
  estr += msg;
  if( errcode > 0 ) { estr += ": "; estr += std::strerror( errcode ); }
  estr += '\n';
  }


void show_file_error( const char * const filename, const char * const msg,
                      const int errcode )
  {
  if( verbosity >= 0 && msg && msg[0] )
    std::fprintf( stderr, "%s: %s: %s%s%s\n", program_name, filename, msg,
                  ( errcode > 0 ) ? ": " : "",
                  ( errcode > 0 ) ? std::strerror( errcode ) : "" );
  }


void internal_error( const char * const msg )
  {
  if( verbosity >= 0 )
    std::fprintf( stderr, "%s: internal error: %s\n", program_name, msg );
  std::exit( 3 );
  }


int main( const int argc, const char * const argv[] )
  {
  if( argc > 0 ) invocation_name = argv[0];

  enum { opt_ano = 256, opt_aso, opt_bso, opt_chk, opt_crc, opt_dbg, opt_del,
         opt_dso, opt_exc, opt_grp, opt_hlp, opt_id, opt_kd, opt_mti, opt_nso,
         opt_out, opt_own, opt_per, opt_sol, opt_un };
  const Arg_parser::Option options[] =
    {
    { '0', 0,                      Arg_parser::no  },
    { '1', 0,                      Arg_parser::no  },
    { '2', 0,                      Arg_parser::no  },
    { '3', 0,                      Arg_parser::no  },
    { '4', 0,                      Arg_parser::no  },
    { '5', 0,                      Arg_parser::no  },
    { '6', 0,                      Arg_parser::no  },
    { '7', 0,                      Arg_parser::no  },
    { '8', 0,                      Arg_parser::no  },
    { '9', 0,                      Arg_parser::no  },
    { 'A', "concatenate",          Arg_parser::no  },
    { 'B', "data-size",            Arg_parser::yes },
    { 'c', "create",               Arg_parser::no  },
    { 'C', "directory",            Arg_parser::yes },
    { 'd', "diff",                 Arg_parser::no  },
    { 'f', "file",                 Arg_parser::yes },
    { 'h', "dereference",          Arg_parser::no  },
    { 'H', "format",               Arg_parser::yes },
    { 'n', "threads",              Arg_parser::yes },
    { 'p', "preserve-permissions", Arg_parser::no  },
    { 'q', "quiet",                Arg_parser::no  },
    { 'r', "append",               Arg_parser::no  },
    { 't', "list",                 Arg_parser::no  },
    { 'v', "verbose",              Arg_parser::no  },
    { 'V', "version",              Arg_parser::no  },
    { 'x', "extract",              Arg_parser::no  },
    { opt_ano, "anonymous",        Arg_parser::no  },
    { opt_aso, "asolid",           Arg_parser::no  },
    { opt_bso, "bsolid",           Arg_parser::no  },
    { opt_chk, "check-lib",        Arg_parser::no  },
    { opt_dbg, "debug",            Arg_parser::yes },
    { opt_del, "delete",           Arg_parser::no  },
    { opt_dso, "dsolid",           Arg_parser::no  },
    { opt_exc, "exclude",          Arg_parser::yes },
    { opt_grp, "group",            Arg_parser::yes },
    { opt_hlp, "help",             Arg_parser::no  },
    { opt_id,  "ignore-ids",       Arg_parser::no  },
    { opt_kd,  "keep-damaged",     Arg_parser::no  },
    { opt_crc, "missing-crc",      Arg_parser::no  },
    { opt_mti, "mtime",            Arg_parser::yes },
    { opt_nso, "no-solid",         Arg_parser::no  },
    { opt_out, "out-slots",        Arg_parser::yes },
    { opt_own, "owner",            Arg_parser::yes },
    { opt_per, "permissive",       Arg_parser::no  },
    { opt_sol, "solid",            Arg_parser::no  },
    { opt_un,  "uncompressed",     Arg_parser::no  },
    {  0, 0,                       Arg_parser::no  } };

  const Arg_parser parser( argc, argv, options, true );	// in_order
  if( parser.error().size() )				// bad option
    { show_error( parser.error().c_str(), 0, true ); return 1; }
  Cl_options cl_opts( parser );

  const long num_online = std::max( 1L, sysconf( _SC_NPROCESSORS_ONLN ) );
  long max_workers = sysconf( _SC_THREAD_THREADS_MAX );
  if( max_workers < 1 || max_workers > INT_MAX / (int)sizeof (pthread_t) )
    max_workers = INT_MAX / sizeof (pthread_t);

  for( int argind = 0; argind < parser.arguments(); ++argind )
    {
    const int code = parser.code( argind );
    if( !code )						// skip non-options
      {
      if( parser.argument( argind ).empty() )
        { show_error( "Empty non-option argument." ); return 1; }
      ++cl_opts.filenames; continue;
      }
    const std::string & sarg = parser.argument( argind );
    const char * const arg = sarg.c_str();
    switch( code )
      {
      case '0': case '1': case '2': case '3': case '4':
      case '5': case '6': case '7': case '8': case '9':
                cl_opts.level = code - '0'; break;
      case 'A': set_mode( cl_opts.program_mode, m_concatenate ); break;
      case 'B': cl_opts.data_size = getnum( arg, min_data_size, max_data_size );
                break;
      case 'c': set_mode( cl_opts.program_mode, m_create ); break;
      case 'C': break;					// skip chdir
      case 'd': set_mode( cl_opts.program_mode, m_diff ); break;
      case 'f': set_archive_name( cl_opts.archive_name, sarg ); break;
      case 'h': cl_opts.dereference = true; break;
      case 'H': break;					// ignore format
      case 'n': cl_opts.num_workers = getnum( arg, 0, max_workers ); break;
      case 'p': cl_opts.preserve_permissions = true; break;
      case 'q': verbosity = -1; break;
      case 'r': set_mode( cl_opts.program_mode, m_append ); break;
      case 't': set_mode( cl_opts.program_mode, m_list ); break;
      case 'v': if( verbosity < 4 ) ++verbosity; break;
      case 'V': show_version(); return 0;
      case 'x': set_mode( cl_opts.program_mode, m_extract ); break;
      case opt_ano: set_owner( cl_opts.owner, "root" );
                    set_group( cl_opts.group, "root" ); break;
      case opt_aso: cl_opts.solidity = asolid; break;
      case opt_bso: cl_opts.solidity = bsolid; break;
      case opt_crc: cl_opts.missing_crc = true; break;
      case opt_chk: return check_lib();
      case opt_dbg: cl_opts.debug_level = getnum( arg, 0, 3 ); break;
      case opt_del: set_mode( cl_opts.program_mode, m_delete ); break;
      case opt_dso: cl_opts.solidity = dsolid; break;
      case opt_exc: Exclude::add_pattern( sarg ); break;
      case opt_grp: set_group( cl_opts.group, arg ); break;
      case opt_hlp: show_help( num_online ); return 0;
      case opt_id:  cl_opts.ignore_ids = true; break;
      case opt_kd:  cl_opts.keep_damaged = true; break;
      case opt_mti: set_mtime( cl_opts.mtime, arg ); break;
      case opt_nso: cl_opts.solidity = no_solid; break;
      case opt_out: cl_opts.out_slots = getnum( arg, 1, 1024 ); break;
      case opt_own: set_owner( cl_opts.owner, arg ); break;
      case opt_per: cl_opts.permissive = true; break;
      case opt_sol: cl_opts.solidity = solid; break;
      case opt_un:  cl_opts.level = -1; break;
      default : internal_error( "uncaught option" );
      }
    } // end process options

#if !defined LZ_API_VERSION || LZ_API_VERSION < 1	// compile-time test
#error "lzlib 1.8 or newer needed."
#endif

#if defined(__MSVCRT__) || defined(__OS2__)
  setmode( STDIN_FILENO, O_BINARY );
  setmode( STDOUT_FILENO, O_BINARY );
#endif

  if( cl_opts.num_workers < 0 )			// 0 disables multi-threading
    cl_opts.num_workers = std::min( num_online, max_workers );

  switch( cl_opts.program_mode )
    {
    case m_none:   show_error( "Missing operation.", 0, true ); return 1;
    case m_append:
    case m_create:      return encode( cl_opts );
    case m_concatenate: return concatenate( cl_opts );
    case m_delete:      return delete_members( cl_opts );
    case m_diff:
    case m_extract:
    case m_list:        return decode( cl_opts );
    }
  }