summaryrefslogtreecommitdiffstats
path: root/main.cc
blob: 631c6f4050e38a66c4d4050b2650c30a56b7ac65 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
/*  Tarlz - Archiver with multimember lzip compression
    Copyright (C) 2013-2019 Antonio Diaz Diaz.

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 2 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/
/*
    Exit status: 0 for a normal exit, 1 for environmental problems
    (file not found, invalid flags, I/O errors, etc), 2 to indicate a
    corrupt or invalid input file, 3 for an internal consistency error
    (eg, bug) which caused tarlz to panic.
*/

#define _FILE_OFFSET_BITS 64

#include <cctype>
#include <cerrno>
#include <climits>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <string>
#include <vector>
#include <fcntl.h>
#include <pthread.h>
#include <stdint.h>
#include <unistd.h>
#include <sys/stat.h>
#include <grp.h>
#include <pwd.h>
#include <lzlib.h>
#if defined(__OS2__)
#include <io.h>
#endif

#include "arg_parser.h"
#include "tarlz.h"

#ifndef O_BINARY
#define O_BINARY 0
#endif

#if CHAR_BIT != 8
#error "Environments where CHAR_BIT != 8 are not supported."
#endif

int verbosity = 0;

namespace {

const char * const program_name = "tarlz";
const char * const program_year = "2019";
const char * invocation_name = program_name;		// default value
bool dereference = false;


void show_help( const long num_online )
  {
  std::printf( "Tarlz is a massively parallel (multi-threaded) combined implementation of\n"
               "the tar archiver and the lzip compressor. Tarlz creates, lists and extracts\n"
               "archives in a simplified and safer variant of the POSIX pax format\n"
               "compressed with lzip, keeping the alignment between tar members and lzip\n"
               "members. The resulting multimember tar.lz archive is fully backward\n"
               "compatible with standard tar tools like GNU tar, which treat it like any\n"
               "other tar.lz archive. Tarlz can append files to the end of such compressed\n"
               "archives.\n"
               "\nKeeping the alignment between tar members and lzip members has two\n"
               "advantages. It adds an indexed lzip layer on top of the tar archive, making\n"
               "it possible to decode the archive safely in parallel. It also minimizes the\n"
               "amount of data lost in case of corruption.\n"
               "\nThe tarlz file format is a safe POSIX-style backup format. In case of\n"
               "corruption, tarlz can extract all the undamaged members from the tar.lz\n"
               "archive, skipping over the damaged members, just like the standard\n"
               "(uncompressed) tar. Moreover, the option '--keep-damaged' can be used to\n"
               "recover as much data as possible from each damaged member, and lziprecover\n"
               "can be used to recover some of the damaged members.\n"
               "\nUsage: %s [options] [files]\n", invocation_name );
  std::printf( "\nOptions:\n"
               "      --help                 display this help and exit\n"
               "  -V, --version              output version information and exit\n"
               "  -A, --concatenate          append archives to the end of an archive\n"
               "  -B, --data-size=<bytes>    set target size of input data blocks [2x8=16 MiB]\n"
               "  -c, --create               create a new archive\n"
               "  -C, --directory=<dir>      change to directory <dir>\n"
               "  -d, --diff                 find differences between archive and file system\n"
               "      --ignore-ids           ignore differences in owner and group IDs\n"
               "      --delete               delete files/directories from an archive\n"
               "      --exclude=<pattern>    exclude files matching a shell pattern\n"
               "  -f, --file=<archive>       use archive file <archive>\n"
               "  -h, --dereference          follow symlinks; archive the files they point to\n"
               "  -n, --threads=<n>          set number of (de)compression threads [%ld]\n"
               "  -q, --quiet                suppress all messages\n"
               "  -r, --append               append files to the end of an archive\n"
               "  -t, --list                 list the contents of an archive\n"
               "  -v, --verbose              verbosely list files processed\n"
               "  -x, --extract              extract files/directories from an archive\n"
               "  -0 .. -9                   set compression level [default 6]\n"
               "      --uncompressed         don't compress the archive created\n"
               "      --asolid               create solidly compressed appendable archive\n"
               "      --bsolid               create per block compressed archive (default)\n"
               "      --dsolid               create per directory compressed archive\n"
               "      --no-solid             create per file compressed archive\n"
               "      --solid                create solidly compressed archive\n"
               "      --anonymous            equivalent to '--owner=root --group=root'\n"
               "      --owner=<owner>        use <owner> name/ID for files added\n"
               "      --group=<group>        use <group> name/ID for files added\n"
               "      --keep-damaged         don't delete partially extracted files\n"
               "      --missing-crc          exit with error status if missing extended CRC\n"
               "      --out-slots=<n>        number of 1 MiB output packets buffered [64]\n"
/*               "      --permissive           allow repeated extended headers and records\n"*/,
               num_online );
  if( verbosity >= 1 )
    {
    std::printf( "      --debug=<level>        (0-1) print debug statistics to stderr\n" );
    }
  std::printf( "\nExit status: 0 for a normal exit, 1 for environmental problems (file\n"
               "not found, invalid flags, I/O errors, etc), 2 to indicate a corrupt or\n"
               "invalid input file, 3 for an internal consistency error (eg, bug) which\n"
               "caused tarlz to panic.\n"
               "\nReport bugs to lzip-bug@nongnu.org\n"
               "Tarlz home page: http://www.nongnu.org/lzip/tarlz.html\n" );
  }


void show_version()
  {
  std::printf( "%s %s\n", program_name, PROGVERSION );
  std::printf( "Copyright (C) %s Antonio Diaz Diaz.\n", program_year );
  std::printf( "Using lzlib %s\n", LZ_version() );
  std::printf( "License GPLv2+: GNU GPL version 2 or later <http://gnu.org/licenses/gpl.html>\n"
               "This is free software: you are free to change and redistribute it.\n"
               "There is NO WARRANTY, to the extent permitted by law.\n" );
  }


unsigned long long getnum( const char * const ptr,
                           const unsigned long long llimit,
                           const unsigned long long ulimit )
  {
  char * tail;
  errno = 0;
  unsigned long long result = strtoull( ptr, &tail, 0 );
  if( tail == ptr )
    {
    show_error( "Bad or missing numerical argument.", 0, true );
    std::exit( 1 );
    }

  if( !errno && tail[0] )
    {
    const unsigned factor = ( tail[1] == 'i' ) ? 1024 : 1000;
    int exponent = 0;				// 0 = bad multiplier
    switch( tail[0] )
      {
      case 'Y': exponent = 8; break;
      case 'Z': exponent = 7; break;
      case 'E': exponent = 6; break;
      case 'P': exponent = 5; break;
      case 'T': exponent = 4; break;
      case 'G': exponent = 3; break;
      case 'M': exponent = 2; break;
      case 'K': if( factor == 1024 ) exponent = 1; break;
      case 'k': if( factor == 1000 ) exponent = 1; break;
      }
    if( exponent <= 0 )
      {
      show_error( "Bad multiplier in numerical argument.", 0, true );
      std::exit( 1 );
      }
    for( int i = 0; i < exponent; ++i )
      {
      if( ulimit / factor >= result ) result *= factor;
      else { errno = ERANGE; break; }
      }
    }
  if( !errno && ( result < llimit || result > ulimit ) ) errno = ERANGE;
  if( errno )
    {
    show_error( "Numerical argument out of limits." );
    std::exit( 1 );
    }
  return result;
  }


void set_mode( Program_mode & program_mode, const Program_mode new_mode )
  {
  if( program_mode != m_none && program_mode != new_mode )
    {
    show_error( "Only one operation can be specified.", 0, true );
    std::exit( 1 );
    }
  program_mode = new_mode;
  }


void set_owner( const char * const arg )
  {
  const struct passwd * const pw = getpwnam( arg );
  if( pw ) cl_owner = pw->pw_uid;
  else if( std::isdigit( (unsigned char)arg[0] ) )
    cl_owner = getnum( arg, 0, INT_MAX );
  else { show_file_error( arg, "Invalid owner" ); std::exit( 1 ); }
  }

void set_group( const char * const arg )
  {
  const struct group * const gr = getgrnam( arg );
  if( gr ) cl_group = gr->gr_gid;
  else if( std::isdigit( (unsigned char)arg[0] ) )
    cl_group = getnum( arg, 0, INT_MAX );
  else { show_file_error( arg, "Invalid group" ); std::exit( 1 ); }
  }

} // end namespace


int hstat( const char * const filename, struct stat * const st )
  { return dereference ? stat( filename, st ) : lstat( filename, st ); }


int open_instream( const std::string & name )
  {
  const int infd = open( name.c_str(), O_RDONLY | O_BINARY );
  if( infd < 0 )
    show_file_error( name.c_str(), "Can't open for reading", errno );
  return infd;
  }


int open_outstream( const std::string & name, const bool create )
  {
  const int flags = (create ? O_CREAT | O_WRONLY | O_TRUNC : O_RDWR) | O_BINARY;
  const mode_t outfd_mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;

  const int outfd = open( name.c_str(), flags, outfd_mode );
  if( outfd < 0 )
    show_file_error( name.c_str(), create ?
                     "Can't create file" : "Error opening file", errno );
  return outfd;
  }


// This can be called from any thread, main thread or sub-threads alike,
// since they all call common helper functions that call cleanup_and_fail()
// in case of an error.
//
void cleanup_and_fail( const int retval )
  {
  // calling 'exit' more than once results in undefined behavior
  static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER;

  pthread_mutex_lock( &mutex );		// ignore errors to avoid loop
  std::exit( retval );
  }


void show_error( const char * const msg, const int errcode, const bool help )
  {
  if( verbosity < 0 ) return;
  if( msg && msg[0] )
    std::fprintf( stderr, "%s: %s%s%s\n", program_name, msg,
                  ( errcode > 0 ) ? ": " : "",
                  ( errcode > 0 ) ? std::strerror( errcode ) : "" );
  if( help )
    std::fprintf( stderr, "Try '%s --help' for more information.\n",
                  invocation_name );
  }


void show_file_error( const char * const filename, const char * const msg,
                      const int errcode )
  {
  if( verbosity >= 0 )
    std::fprintf( stderr, "%s: %s: %s%s%s\n", program_name, filename, msg,
                  ( errcode > 0 ) ? ": " : "",
                  ( errcode > 0 ) ? std::strerror( errcode ) : "" );
  }


void internal_error( const char * const msg )
  {
  if( verbosity >= 0 )
    std::fprintf( stderr, "%s: internal error: %s\n", program_name, msg );
  std::exit( 3 );
  }


int main( const int argc, const char * const argv[] )
  {
  std::string archive_name;
  int debug_level = 0;
  int level = 6;		// compression level, < 0 means uncompressed
  int num_workers = -1;		// start this many worker threads
  int out_slots = 64;
  Program_mode program_mode = m_none;
  bool ignore_ids = false;
  bool keep_damaged = false;
  bool missing_crc = false;
  bool permissive = false;
  if( argc > 0 ) invocation_name = argv[0];

  if( LZ_version()[0] < '1' )
    { show_error( "Bad library version. At least lzlib 1.0 is required." );
      return 1; }

  enum { opt_ano = 256, opt_aso, opt_bso, opt_crc, opt_dbg, opt_del, opt_dso,
         opt_exc, opt_grp, opt_hlp, opt_id, opt_kd, opt_nso, opt_out, opt_own,
         opt_per, opt_sol, opt_un };
  const Arg_parser::Option options[] =
    {
    { '0',  0,                 Arg_parser::no  },
    { '1',  0,                 Arg_parser::no  },
    { '2',  0,                 Arg_parser::no  },
    { '3',  0,                 Arg_parser::no  },
    { '4',  0,                 Arg_parser::no  },
    { '5',  0,                 Arg_parser::no  },
    { '6',  0,                 Arg_parser::no  },
    { '7',  0,                 Arg_parser::no  },
    { '8',  0,                 Arg_parser::no  },
    { '9',  0,                 Arg_parser::no  },
    { 'A', "concatenate",      Arg_parser::no  },
    { 'B', "data-size",        Arg_parser::yes },
    { 'c', "create",           Arg_parser::no  },
    { 'C', "directory",        Arg_parser::yes },
    { 'd', "diff",             Arg_parser::no  },
    { 'f', "file",             Arg_parser::yes },
    { 'h', "dereference",      Arg_parser::no  },
    { 'H', "format",           Arg_parser::yes },
    { 'n', "threads",          Arg_parser::yes },
    { 'q', "quiet",            Arg_parser::no  },
    { 'r', "append",           Arg_parser::no  },
    { 't', "list",             Arg_parser::no  },
    { 'v', "verbose",          Arg_parser::no  },
    { 'V', "version",          Arg_parser::no  },
    { 'x', "extract",          Arg_parser::no  },
    { opt_ano, "anonymous",    Arg_parser::no  },
    { opt_aso, "asolid",       Arg_parser::no  },
    { opt_bso, "bsolid",       Arg_parser::no  },
    { opt_dbg, "debug",        Arg_parser::yes },
    { opt_del, "delete",       Arg_parser::no  },
    { opt_dso, "dsolid",       Arg_parser::no  },
    { opt_exc, "exclude",      Arg_parser::yes },
    { opt_grp, "group",        Arg_parser::yes },
    { opt_hlp, "help",         Arg_parser::no  },
    { opt_id,  "ignore-ids",   Arg_parser::no  },
    { opt_kd,  "keep-damaged", Arg_parser::no  },
    { opt_crc, "missing-crc",  Arg_parser::no  },
    { opt_nso, "no-solid",     Arg_parser::no  },
    { opt_out, "out-slots",    Arg_parser::yes },
    { opt_own, "owner",        Arg_parser::yes },
    { opt_per, "permissive",   Arg_parser::no  },
    { opt_sol, "solid",        Arg_parser::no  },
    { opt_un,  "uncompressed", Arg_parser::no  },
    {  0 ,  0,                 Arg_parser::no  } };

  const Arg_parser parser( argc, argv, options, true );
  if( parser.error().size() )				// bad option
    { show_error( parser.error().c_str(), 0, true ); return 1; }

  const long num_online = std::max( 1L, sysconf( _SC_NPROCESSORS_ONLN ) );
  long max_workers = sysconf( _SC_THREAD_THREADS_MAX );
  if( max_workers < 1 || max_workers > INT_MAX / (int)sizeof (pthread_t) )
    max_workers = INT_MAX / sizeof (pthread_t);

  int filenames = 0;
  for( int argind = 0; argind < parser.arguments(); ++argind )
    {
    const int code = parser.code( argind );
    if( !code )						// skip non-options
      { if( parser.argument( argind ).size() ) ++filenames; continue; }
    const std::string & sarg = parser.argument( argind );
    const char * const arg = sarg.c_str();
    switch( code )
      {
      case '0': case '1': case '2': case '3': case '4':
      case '5': case '6': case '7': case '8': case '9':
                level = code - '0'; break;
      case 'A': set_mode( program_mode, m_concatenate ); break;
      case 'B': cl_data_size = getnum( arg, min_data_size, max_data_size );
                break;
      case 'c': set_mode( program_mode, m_create ); break;
      case 'C': break;					// skip chdir
      case 'd': set_mode( program_mode, m_diff ); break;
      case 'f': if( sarg != "-" ) archive_name = sarg; break;
      case 'h': dereference = true; break;
      case 'H': break;					// ignore format
      case 'n': num_workers = getnum( arg, 0, max_workers ); break;
      case 'q': verbosity = -1; break;
      case 'r': set_mode( program_mode, m_append ); break;
      case 't': set_mode( program_mode, m_list ); break;
      case 'v': if( verbosity < 4 ) ++verbosity; break;
      case 'V': show_version(); return 0;
      case 'x': set_mode( program_mode, m_extract ); break;
      case opt_ano: set_owner( "root" ); set_group( "root" ); break;
      case opt_aso: solidity = asolid; break;
      case opt_bso: solidity = bsolid; break;
      case opt_crc: missing_crc = true; break;
      case opt_dbg: debug_level = getnum( arg, 0, 3 ); break;
      case opt_del: set_mode( program_mode, m_delete ); break;
      case opt_dso: solidity = dsolid; break;
      case opt_exc: Exclude::add_pattern( sarg ); break;
      case opt_grp: set_group( arg ); break;
      case opt_hlp: show_help( num_online ); return 0;
      case opt_id:  ignore_ids = true; break;
      case opt_kd:  keep_damaged = true; break;
      case opt_nso: solidity = no_solid; break;
      case opt_out: out_slots = getnum( arg, 1, 1024 ); break;
      case opt_own: set_owner( arg ); break;
      case opt_per: permissive = true; break;
      case opt_sol: solidity = solid; break;
      case opt_un:  level = -1; break;
      default : internal_error( "uncaught option" );
      }
    } // end process options

#if defined(__MSVCRT__) || defined(__OS2__)
  setmode( STDIN_FILENO, O_BINARY );
  setmode( STDOUT_FILENO, O_BINARY );
#endif

  if( num_workers < 0 ) num_workers = std::min( num_online, max_workers );

  switch( program_mode )
    {
    case m_none:   show_error( "Missing operation.", 0, true ); return 2;
    case m_append:
    case m_create: return encode( archive_name, parser, filenames, level,
                                  num_workers, out_slots, debug_level,
                                  program_mode == m_append, dereference );
    case m_concatenate: return concatenate( archive_name, parser, filenames );
    case m_delete: return delete_members( archive_name, parser, filenames,
                                          missing_crc, permissive );
    case m_diff:
    case m_extract:
    case m_list:   return decode( archive_name, parser, filenames,
                          num_workers, debug_level, program_mode,
                          ignore_ids, keep_damaged, missing_crc, permissive );
    }
  }