diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-15 09:41:34 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-15 09:41:34 +0000 |
commit | 25e43e389c26d56f90f2f20e3cee19c808b2a18c (patch) | |
tree | fdadca249ffd70901103b3f06a98aeab695c6193 /src/xz | |
parent | Adding debian version 5.6.1+really5.4.5-1. (diff) | |
download | xz-utils-25e43e389c26d56f90f2f20e3cee19c808b2a18c.tar.xz xz-utils-25e43e389c26d56f90f2f20e3cee19c808b2a18c.zip |
Merging upstream version 5.6.2.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
37 files changed, 2049 insertions, 630 deletions
diff --git a/src/xz/Makefile.am b/src/xz/Makefile.am index 4bc64f3..4ff061f 100644 --- a/src/xz/Makefile.am +++ b/src/xz/Makefile.am @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## bin_PROGRAMS = xz @@ -25,6 +21,8 @@ xz_SOURCES = \ options.c \ options.h \ private.h \ + sandbox.c \ + sandbox.h \ signals.c \ signals.h \ suffix.c \ @@ -53,7 +51,7 @@ xz_CPPFLAGS = \ -I$(top_srcdir)/src/liblzma/api \ -I$(top_builddir)/lib -xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(CAPSICUM_LIB) +xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la if COND_GNULIB xz_LDADD += $(top_builddir)/lib/libgnu.a diff --git a/src/xz/Makefile.in b/src/xz/Makefile.in index e6422c5..18e0151 100644 --- a/src/xz/Makefile.in +++ b/src/xz/Makefile.in @@ -98,8 +98,8 @@ bin_PROGRAMS = xz$(EXEEXT) @COND_LZMALINKS_TRUE@am__append_4 = lzma unlzma lzcat subdir = src/xz ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \ + $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \ $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -127,23 +127,25 @@ PROGRAMS = $(bin_PROGRAMS) am__xz_SOURCES_DIST = args.c args.h coder.c coder.h file_io.c \ file_io.h hardware.c hardware.h main.c main.h message.c \ message.h mytime.c mytime.h options.c options.h private.h \ - signals.c signals.h suffix.c suffix.h util.c util.h \ - ../common/tuklib_open_stdxxx.c ../common/tuklib_progname.c \ - ../common/tuklib_exit.c ../common/tuklib_mbstr_width.c \ - ../common/tuklib_mbstr_fw.c list.c list.h xz_w32res.rc + sandbox.c sandbox.h signals.c signals.h suffix.c suffix.h \ + util.c util.h ../common/tuklib_open_stdxxx.c \ + ../common/tuklib_progname.c ../common/tuklib_exit.c \ + ../common/tuklib_mbstr_width.c ../common/tuklib_mbstr_fw.c \ + list.c list.h xz_w32res.rc @COND_MAIN_DECODER_TRUE@am__objects_1 = xz-list.$(OBJEXT) @COND_W32_TRUE@am__objects_2 = xz_w32res.$(OBJEXT) am_xz_OBJECTS = xz-args.$(OBJEXT) xz-coder.$(OBJEXT) \ xz-file_io.$(OBJEXT) xz-hardware.$(OBJEXT) xz-main.$(OBJEXT) \ xz-message.$(OBJEXT) xz-mytime.$(OBJEXT) xz-options.$(OBJEXT) \ - xz-signals.$(OBJEXT) xz-suffix.$(OBJEXT) xz-util.$(OBJEXT) \ - xz-tuklib_open_stdxxx.$(OBJEXT) xz-tuklib_progname.$(OBJEXT) \ - xz-tuklib_exit.$(OBJEXT) xz-tuklib_mbstr_width.$(OBJEXT) \ - xz-tuklib_mbstr_fw.$(OBJEXT) $(am__objects_1) $(am__objects_2) + xz-sandbox.$(OBJEXT) xz-signals.$(OBJEXT) xz-suffix.$(OBJEXT) \ + xz-util.$(OBJEXT) xz-tuklib_open_stdxxx.$(OBJEXT) \ + xz-tuklib_progname.$(OBJEXT) xz-tuklib_exit.$(OBJEXT) \ + xz-tuklib_mbstr_width.$(OBJEXT) xz-tuklib_mbstr_fw.$(OBJEXT) \ + $(am__objects_1) $(am__objects_2) xz_OBJECTS = $(am_xz_OBJECTS) am__DEPENDENCIES_1 = xz_DEPENDENCIES = $(top_builddir)/src/liblzma/liblzma.la \ - $(am__DEPENDENCIES_1) $(am__append_3) $(am__DEPENDENCIES_1) + $(am__append_3) $(am__DEPENDENCIES_1) AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) am__v_lt_0 = --silent @@ -167,8 +169,9 @@ am__depfiles_remade = ./$(DEPDIR)/xz-args.Po ./$(DEPDIR)/xz-coder.Po \ ./$(DEPDIR)/xz-file_io.Po ./$(DEPDIR)/xz-hardware.Po \ ./$(DEPDIR)/xz-list.Po ./$(DEPDIR)/xz-main.Po \ ./$(DEPDIR)/xz-message.Po ./$(DEPDIR)/xz-mytime.Po \ - ./$(DEPDIR)/xz-options.Po ./$(DEPDIR)/xz-signals.Po \ - ./$(DEPDIR)/xz-suffix.Po ./$(DEPDIR)/xz-tuklib_exit.Po \ + ./$(DEPDIR)/xz-options.Po ./$(DEPDIR)/xz-sandbox.Po \ + ./$(DEPDIR)/xz-signals.Po ./$(DEPDIR)/xz-suffix.Po \ + ./$(DEPDIR)/xz-tuklib_exit.Po \ ./$(DEPDIR)/xz-tuklib_mbstr_fw.Po \ ./$(DEPDIR)/xz-tuklib_mbstr_width.Po \ ./$(DEPDIR)/xz-tuklib_open_stdxxx.Po \ @@ -259,7 +262,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CAPSICUM_LIB = @CAPSICUM_LIB@ CC = @CC@ CCAS = @CCAS@ CCASDEPMODE = @CCASDEPMODE@ @@ -388,6 +390,8 @@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ +localedir_c = @localedir_c@ +localedir_c_make = @localedir_c_make@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ @@ -408,8 +412,8 @@ top_srcdir = @top_srcdir@ xz = @xz@ xz_SOURCES = args.c args.h coder.c coder.h file_io.c file_io.h \ hardware.c hardware.h main.c main.h message.c message.h \ - mytime.c mytime.h options.c options.h private.h signals.c \ - signals.h suffix.c suffix.h util.c util.h \ + mytime.c mytime.h options.c options.h private.h sandbox.c \ + sandbox.h signals.c signals.h suffix.c suffix.h util.c util.h \ ../common/tuklib_open_stdxxx.c ../common/tuklib_progname.c \ ../common/tuklib_exit.c ../common/tuklib_mbstr_width.c \ ../common/tuklib_mbstr_fw.c $(am__append_1) $(am__append_2) @@ -421,8 +425,8 @@ xz_CPPFLAGS = \ # libgnu.a may need these libs, so this must be after libgnu.a. -xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(CAPSICUM_LIB) \ - $(am__append_3) $(LTLIBINTL) +xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(am__append_3) \ + $(LTLIBINTL) dist_man_MANS = xz.1 xzlinks = unxz xzcat $(am__append_4) all: all-am @@ -527,6 +531,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-message.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-mytime.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-options.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-sandbox.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-signals.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-suffix.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-tuklib_exit.Po@am__quote@ # am--include-marker @@ -675,6 +680,20 @@ xz-options.obj: options.c @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-options.obj `if test -f 'options.c'; then $(CYGPATH_W) 'options.c'; else $(CYGPATH_W) '$(srcdir)/options.c'; fi` +xz-sandbox.o: sandbox.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-sandbox.o -MD -MP -MF $(DEPDIR)/xz-sandbox.Tpo -c -o xz-sandbox.o `test -f 'sandbox.c' || echo '$(srcdir)/'`sandbox.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/xz-sandbox.Tpo $(DEPDIR)/xz-sandbox.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='sandbox.c' object='xz-sandbox.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-sandbox.o `test -f 'sandbox.c' || echo '$(srcdir)/'`sandbox.c + +xz-sandbox.obj: sandbox.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-sandbox.obj -MD -MP -MF $(DEPDIR)/xz-sandbox.Tpo -c -o xz-sandbox.obj `if test -f 'sandbox.c'; then $(CYGPATH_W) 'sandbox.c'; else $(CYGPATH_W) '$(srcdir)/sandbox.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/xz-sandbox.Tpo $(DEPDIR)/xz-sandbox.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='sandbox.c' object='xz-sandbox.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-sandbox.obj `if test -f 'sandbox.c'; then $(CYGPATH_W) 'sandbox.c'; else $(CYGPATH_W) '$(srcdir)/sandbox.c'; fi` + xz-signals.o: signals.c @am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-signals.o -MD -MP -MF $(DEPDIR)/xz-signals.Tpo -c -o xz-signals.o `test -f 'signals.c' || echo '$(srcdir)/'`signals.c @am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/xz-signals.Tpo $(DEPDIR)/xz-signals.Po @@ -985,6 +1004,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/xz-message.Po -rm -f ./$(DEPDIR)/xz-mytime.Po -rm -f ./$(DEPDIR)/xz-options.Po + -rm -f ./$(DEPDIR)/xz-sandbox.Po -rm -f ./$(DEPDIR)/xz-signals.Po -rm -f ./$(DEPDIR)/xz-suffix.Po -rm -f ./$(DEPDIR)/xz-tuklib_exit.Po @@ -1049,6 +1069,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/xz-message.Po -rm -f ./$(DEPDIR)/xz-mytime.Po -rm -f ./$(DEPDIR)/xz-options.Po + -rm -f ./$(DEPDIR)/xz-sandbox.Po -rm -f ./$(DEPDIR)/xz-signals.Po -rm -f ./$(DEPDIR)/xz-suffix.Po -rm -f ./$(DEPDIR)/xz-tuklib_exit.Po diff --git a/src/xz/args.c b/src/xz/args.c index 17e778c..eba1b97 100644 --- a/src/xz/args.c +++ b/src/xz/args.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file args.c @@ -5,10 +7,8 @@ /// /// \note Filter-specific options parsing is in options.c. // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -83,14 +83,20 @@ parse_block_list(const char *str_const) ++count; // Prevent an unlikely integer overflow. - if (count > SIZE_MAX / sizeof(uint64_t) - 1) + if (count > SIZE_MAX / sizeof(block_list_entry) - 1) message_fatal(_("%s: Too many arguments to --block-list"), str); // Allocate memory to hold all the sizes specified. // If --block-list was specified already, its value is forgotten. free(opt_block_list); - opt_block_list = xmalloc((count + 1) * sizeof(uint64_t)); + opt_block_list = xmalloc((count + 1) * sizeof(block_list_entry)); + + // Clear the bitmask of filter chains in use. + block_list_chain_mask = 0; + + // Reset the largest Block size found in --block-list. + block_list_largest = 0; for (size_t i = 0; i < count; ++i) { // Locate the next comma and replace it with \0. @@ -98,6 +104,43 @@ parse_block_list(const char *str_const) if (p != NULL) *p = '\0'; + // Use the default filter chain unless overridden. + opt_block_list[i].chain_num = 0; + + // To specify a filter chain, the block list entry may be + // prepended with "[filter-chain-number]:". The size is + // still required for every block. + // For instance: + // --block-list=2:10MiB,1:5MiB,,8MiB,0:0 + // + // Translates to: + // 1. Block of 10 MiB using filter chain 2 + // 2. Block of 5 MiB using filter chain 1 + // 3. Block of 5 MiB using filter chain 1 + // 4. Block of 8 MiB using the default filter chain + // 5. The last block uses the default filter chain + // + // The block list: + // --block-list=2:MiB,1:,0 + // + // Is not allowed because the second block does not specify + // the block size, only the filter chain. + if (str[0] >= '0' && str[0] <= '9' && str[1] == ':') { + if (str[2] == '\0') + message_fatal(_("In --block-list, block " + "size is missing after " + "filter chain number '%c:'"), + str[0]); + + const unsigned chain_num = (unsigned)(str[0] - '0'); + opt_block_list[i].chain_num = chain_num; + block_list_chain_mask |= 1U << chain_num; + str += 2; + } else { + // This Block uses the default filter chain. + block_list_chain_mask |= 1U << 0; + } + if (str[0] == '\0') { // There is no string, that is, a comma follows // another comma. Use the previous value. @@ -107,25 +150,39 @@ parse_block_list(const char *str_const) assert(i > 0); opt_block_list[i] = opt_block_list[i - 1]; } else { - opt_block_list[i] = str_to_uint64("block-list", str, - 0, UINT64_MAX); + opt_block_list[i].size = str_to_uint64("block-list", + str, 0, UINT64_MAX); // Zero indicates no more new Blocks. - if (opt_block_list[i] == 0) { + if (opt_block_list[i].size == 0) { if (i + 1 != count) message_fatal(_("0 can only be used " "as the last element " "in --block-list")); - opt_block_list[i] = UINT64_MAX; + opt_block_list[i].size = UINT64_MAX; } + + // Remember the largest Block size in the list. + // + // NOTE: Do this after handling the special value 0 + // because when 0 is used, we don't want to reduce + // the Block size of the multithreaded encoder. + if (block_list_largest < opt_block_list[i].size) + block_list_largest = opt_block_list[i].size; } - str = p + 1; + // Be standards compliant: p + 1 is undefined behavior + // if p == NULL. That occurs on the last iteration of + // the loop when we won't care about the value of str + // anymore anyway. That is, this is done conditionally + // solely for standard conformance reasons. + if (p != NULL) + str = p + 1; } // Terminate the array. - opt_block_list[count] = 0; + opt_block_list[count].size = 0; free(str_start); return; @@ -136,13 +193,26 @@ static void parse_real(args_info *args, int argc, char **argv) { enum { - OPT_X86 = INT_MIN, + OPT_FILTERS = INT_MIN, + OPT_FILTERS1, + OPT_FILTERS2, + OPT_FILTERS3, + OPT_FILTERS4, + OPT_FILTERS5, + OPT_FILTERS6, + OPT_FILTERS7, + OPT_FILTERS8, + OPT_FILTERS9, + OPT_FILTERS_HELP, + + OPT_X86, OPT_POWERPC, OPT_IA64, OPT_ARM, OPT_ARMTHUMB, OPT_ARM64, OPT_SPARC, + OPT_RISCV, OPT_DELTA, OPT_LZMA1, OPT_LZMA2, @@ -191,7 +261,7 @@ parse_real(args_info *args, int argc, char **argv) { "check", required_argument, NULL, 'C' }, { "ignore-check", no_argument, NULL, OPT_IGNORE_CHECK }, { "block-size", required_argument, NULL, OPT_BLOCK_SIZE }, - { "block-list", required_argument, NULL, OPT_BLOCK_LIST }, + { "block-list", required_argument, NULL, OPT_BLOCK_LIST }, { "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS }, { "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS }, { "memlimit-mt-decompress", required_argument, NULL, OPT_MEM_MT_DECOMPRESS }, @@ -206,6 +276,18 @@ parse_real(args_info *args, int argc, char **argv) { "best", no_argument, NULL, '9' }, // Filters + { "filters", optional_argument, NULL, OPT_FILTERS}, + { "filters1", optional_argument, NULL, OPT_FILTERS1}, + { "filters2", optional_argument, NULL, OPT_FILTERS2}, + { "filters3", optional_argument, NULL, OPT_FILTERS3}, + { "filters4", optional_argument, NULL, OPT_FILTERS4}, + { "filters5", optional_argument, NULL, OPT_FILTERS5}, + { "filters6", optional_argument, NULL, OPT_FILTERS6}, + { "filters7", optional_argument, NULL, OPT_FILTERS7}, + { "filters8", optional_argument, NULL, OPT_FILTERS8}, + { "filters9", optional_argument, NULL, OPT_FILTERS9}, + { "filters-help", optional_argument, NULL, OPT_FILTERS_HELP}, + { "lzma1", optional_argument, NULL, OPT_LZMA1 }, { "lzma2", optional_argument, NULL, OPT_LZMA2 }, { "x86", optional_argument, NULL, OPT_X86 }, @@ -215,6 +297,7 @@ parse_real(args_info *args, int argc, char **argv) { "armthumb", optional_argument, NULL, OPT_ARMTHUMB }, { "arm64", optional_argument, NULL, OPT_ARM64 }, { "sparc", optional_argument, NULL, OPT_SPARC }, + { "riscv", optional_argument, NULL, OPT_RISCV }, { "delta", optional_argument, NULL, OPT_DELTA }, // Other options @@ -372,7 +455,30 @@ parse_real(args_info *args, int argc, char **argv) opt_mode = MODE_COMPRESS; break; - // Filter setup + // --filters + case OPT_FILTERS: + coder_add_filters_from_str(optarg); + break; + + // --filters1...--filters9 + case OPT_FILTERS1: + case OPT_FILTERS2: + case OPT_FILTERS3: + case OPT_FILTERS4: + case OPT_FILTERS5: + case OPT_FILTERS6: + case OPT_FILTERS7: + case OPT_FILTERS8: + case OPT_FILTERS9: + coder_add_block_filters(optarg, + (size_t)(c - OPT_FILTERS)); + break; + + // --filters-help + case OPT_FILTERS_HELP: + // This doesn't return. + message_filters_help(); + break; case OPT_X86: coder_add_filter(LZMA_FILTER_X86, @@ -409,6 +515,11 @@ parse_real(args_info *args, int argc, char **argv) options_bcj(optarg)); break; + case OPT_RISCV: + coder_add_filter(LZMA_FILTER_RISCV, + options_bcj(optarg)); + break; + case OPT_DELTA: coder_add_filter(LZMA_FILTER_DELTA, options_delta(optarg)); @@ -516,8 +627,8 @@ parse_real(args_info *args, int argc, char **argv) case OPT_FILES0: if (args->files_name != NULL) message_fatal(_("Only one file can be " - "specified with `--files' " - "or `--files0'.")); + "specified with '--files' " + "or '--files0'.")); if (optarg == NULL) { args->files_name = stdin_filename; @@ -718,6 +829,39 @@ args_parse(args_info *args, int argc, char **argv) if (opt_mode == MODE_COMPRESS && opt_format == FORMAT_AUTO) opt_format = FORMAT_XZ; + // Set opt_block_list to NULL if we are not compressing to the .xz + // format. This option cannot be used outside of this case, and + // simplifies the implementation later. + if ((opt_mode != MODE_COMPRESS || opt_format != FORMAT_XZ) + && opt_block_list != NULL) { + message(V_WARNING, _("--block-list is ignored unless " + "compressing to the .xz format")); + free(opt_block_list); + opt_block_list = NULL; + } + + // If raw format is used and a custom suffix is not provided, + // then only stdout mode can be used when compressing or + // decompressing. + if (opt_format == FORMAT_RAW && !suffix_is_set() && !opt_stdout + && (opt_mode == MODE_COMPRESS + || opt_mode == MODE_DECOMPRESS)) { + if (args->files_name != NULL) + message_fatal(_("With --format=raw, " + "--suffix=.SUF is required " + "unless writing to stdout")); + + // If all of the filenames provided are "-" (more than one + // "-" could be specified) or no filenames are provided, + // then we are only going to be writing to standard out. + for (int i = optind; i < argc; i++) { + if (strcmp(argv[i], "-") != 0) + message_fatal(_("With --format=raw, " + "--suffix=.SUF is required " + "unless writing to stdout")); + } + } + // Compression settings need to be validated (options themselves and // their memory usage) when compressing to any file format. It has to // be done also when uncompressing raw data, since for raw decoding @@ -727,14 +871,6 @@ args_parse(args_info *args, int argc, char **argv) && opt_mode != MODE_LIST)) coder_set_compression_settings(); - // If raw format is used and a custom suffix is not provided, - // then only stdout mode can be used when compressing or decompressing. - if (opt_format == FORMAT_RAW && !suffix_is_set() && !opt_stdout - && (opt_mode == MODE_COMPRESS - || opt_mode == MODE_DECOMPRESS)) - message_fatal(_("With --format=raw, --suffix=.SUF is " - "required unless writing to stdout")); - // If no filenames are given, use stdin. if (argv[optind] == NULL && args->files_name == NULL) { // We don't modify or free() the "-" constant. The caller diff --git a/src/xz/args.h b/src/xz/args.h index a1a5930..e693ecd 100644 --- a/src/xz/args.h +++ b/src/xz/args.h @@ -1,12 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file args.h /// \brief Argument parsing // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// diff --git a/src/xz/coder.c b/src/xz/coder.c index 91d40ed..5e41f0d 100644 --- a/src/xz/coder.c +++ b/src/xz/coder.c @@ -1,16 +1,17 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file coder.c /// \brief Compresses or uncompresses a file // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// #include "private.h" +#include "tuklib_integer.h" /// Return value type for coder_init(). @@ -26,25 +27,47 @@ enum format_type opt_format = FORMAT_AUTO; bool opt_auto_adjust = true; bool opt_single_stream = false; uint64_t opt_block_size = 0; -uint64_t *opt_block_list = NULL; - +block_list_entry *opt_block_list = NULL; +uint64_t block_list_largest; +uint32_t block_list_chain_mask; /// Stream used to communicate with liblzma static lzma_stream strm = LZMA_STREAM_INIT; -/// Filters needed for all encoding all formats, and also decoding in raw data -static lzma_filter filters[LZMA_FILTERS_MAX + 1]; +/// Maximum number of filter chains. The first filter chain is the default, +/// and 9 other filter chains can be specified with --filtersX. +#define NUM_FILTER_CHAIN_MAX 10 + +/// The default filter chain is in chains[0]. It is used for encoding +/// in all supported formats and also for decdoing raw streams. The other +/// filter chains are set by --filtersX to support changing filters with +/// the --block-list option. +static lzma_filter chains[NUM_FILTER_CHAIN_MAX][LZMA_FILTERS_MAX + 1]; + +/// Bitmask indicating which filter chains are actually used when encoding +/// in the .xz format. This is needed since the filter chains specified using +/// --filtersX (or the default filter chain) might in reality be unneeded +/// if they are never used in --block-list. When --block-list isn't +/// specified, only the default filter chain is used, thus the initial +/// value of this variable is 1U << 0 (the number of the default chain is 0). +static uint32_t chains_used_mask = 1U << 0; /// Input and output buffers static io_buf in_buf; static io_buf out_buf; -/// Number of filters. Zero indicates that we are using a preset. +/// Number of filters in the default filter chain. Zero indicates that +/// we are using a preset. static uint32_t filters_count = 0; /// Number of the preset (0-9) static uint32_t preset_number = LZMA_PRESET_DEFAULT; +/// True if the current default filter chain was set using the --filters +/// option. The filter chain is reset if a preset option (like -9) or an +/// old-style filter option (like --lzma2) is used after a --filters option. +static bool string_to_filter_used = false; + /// Integrity check type static lzma_check check; @@ -60,7 +83,6 @@ static bool allow_trailing_input; static lzma_mt mt_options = { .flags = 0, .timeout = 300, - .filters = filters, }; #endif @@ -77,14 +99,14 @@ coder_set_check(lzma_check new_check) static void forget_filter_chain(void) { - // Setting a preset makes us forget a possibly defined custom - // filter chain. - while (filters_count > 0) { - --filters_count; - free(filters[filters_count].options); - filters[filters_count].options = NULL; + // Setting a preset or using --filters makes us forget + // the earlier custom filter chain (if any). + if (filters_count > 0) { + lzma_filters_free(chains[0], NULL); + filters_count = 0; } + string_to_filter_used = false; return; } @@ -114,9 +136,15 @@ coder_add_filter(lzma_vli id, void *options) if (filters_count == LZMA_FILTERS_MAX) message_fatal(_("Maximum number of filters is four")); - filters[filters_count].id = id; - filters[filters_count].options = options; - ++filters_count; + if (string_to_filter_used) + forget_filter_chain(); + + chains[0][filters_count].id = id; + chains[0][filters_count].options = options; + + // Terminate the filter chain with LZMA_VLI_UNKNOWN to simplify + // implementation of forget_filter_chain(). + chains[0][++filters_count].id = LZMA_VLI_UNKNOWN; // Setting a custom filter chain makes us forget the preset options. // This makes a difference if one specifies e.g. "xz -9 --lzma2 -e" @@ -128,6 +156,69 @@ coder_add_filter(lzma_vli id, void *options) } +static void +str_to_filters(const char *str, uint32_t index, uint32_t flags) +{ + int error_pos; + const char *err = lzma_str_to_filters(str, &error_pos, + chains[index], flags, NULL); + + if (err != NULL) { + char filter_num[2] = ""; + if (index > 0) + filter_num[0] = '0' + index; + + // FIXME? The message in err isn't translated. + // Including the translations in the xz translations is + // slightly ugly but possible. Creating a new domain for + // liblzma might not be worth it especially since on some + // OSes it adds extra dependencies to translation libraries. + message(V_ERROR, _("Error in --filters%s=FILTERS option:"), + filter_num); + message(V_ERROR, "%s", str); + message(V_ERROR, "%*s^", error_pos, ""); + message_fatal("%s", err); + } +} + + +extern void +coder_add_filters_from_str(const char *filter_str) +{ + // Forget presets and previously defined filter chain. See + // coder_add_filter() above for why preset_number must be reset too. + forget_filter_chain(); + preset_number = LZMA_PRESET_DEFAULT; + + string_to_filter_used = true; + + // Include LZMA_STR_ALL_FILTERS so this can be used with --format=raw. + str_to_filters(filter_str, 0, LZMA_STR_ALL_FILTERS); + + // Set the filters_count to be the number of filters converted from + // the string. + for (filters_count = 0; chains[0][filters_count].id + != LZMA_VLI_UNKNOWN; + ++filters_count) ; + + assert(filters_count > 0); + return; +} + + +extern void +coder_add_block_filters(const char *str, size_t slot) +{ + // Free old filters first, if they were previously allocated. + if (chains_used_mask & (1U << slot)) + lzma_filters_free(chains[slot], NULL); + + str_to_filters(str, slot, 0); + + chains_used_mask |= 1U << slot; +} + + tuklib_attr_noreturn static void memlimit_too_small(uint64_t memory_usage) @@ -139,6 +230,67 @@ memlimit_too_small(uint64_t memory_usage) } +#ifdef HAVE_ENCODERS +/// \brief Calculate the memory usage of each filter chain. +/// +/// \param chains_memusages If non-NULL, the memusage of the encoder +/// or decoder for each chain is stored in +/// this array. +/// \param mt If non-NULL, calculate memory usage of +/// multithreaded encoder. +/// \param encode Whether to calculate encoder or decoder +/// memory usage. This must be true if +/// mt != NULL. +/// +/// \return Return the highest memory usage of all of the filter chains. +static uint64_t +get_chains_memusage(uint64_t *chains_memusages, const lzma_mt *mt, bool encode) +{ + uint64_t max_memusage = 0; + +#ifdef MYTHREAD_ENABLED + // Copy multithreading options to a temporary struct since the + // "filters" member needs to be changed. + lzma_mt mt_local; + if (mt != NULL) + mt_local = *mt; +#else + (void)mt; +#endif + + for (uint32_t i = 0; i < ARRAY_SIZE(chains); i++) { + if (!(chains_used_mask & (1U << i))) + continue; + + uint64_t memusage = UINT64_MAX; +#ifdef MYTHREAD_ENABLED + if (mt != NULL) { + assert(encode); + mt_local.filters = chains[i]; + memusage = lzma_stream_encoder_mt_memusage(&mt_local); + } else +#endif + if (encode) { + memusage = lzma_raw_encoder_memusage(chains[i]); + } +#ifdef HAVE_DECODERS + else { + memusage = lzma_raw_decoder_memusage(chains[i]); + } +#endif + + if (chains_memusages != NULL) + chains_memusages[i] = memusage; + + if (memusage > max_memusage) + max_memusage = memusage; + } + + return max_memusage; +} +#endif + + extern void coder_set_compression_settings(void) { @@ -156,10 +308,56 @@ coder_set_compression_settings(void) check = LZMA_CHECK_CRC32; } +#ifdef HAVE_ENCODERS + if (opt_block_list != NULL) { + // args.c ensures these. + assert(opt_mode == MODE_COMPRESS); + assert(opt_format == FORMAT_XZ); + + // Find out if block_list_chain_mask has a bit set that + // isn't set in chains_used_mask. + const uint32_t missing_chains_mask + = (block_list_chain_mask ^ chains_used_mask) + & block_list_chain_mask; + + // If a filter chain was specified in --block-list but no + // matching --filtersX option was used, exit with an error. + if (missing_chains_mask != 0) { + // Get the number of the first missing filter chain + // and show it in the error message. + const unsigned first_missing + = (unsigned)ctz32(missing_chains_mask); + + message_fatal(_("filter chain %u used by " + "--block-list but not specified " + "with --filters%u="), + first_missing, first_missing); + } + + // Omit the unused filter chains from mask of used chains. + // + // (FIXME? When built with debugging, coder_free() will free() + // the filter chains (except the default chain) which makes + // Valgrind show fewer reachable allocations. But coder_free() + // uses this mask to determine which chains to free. Thus it + // won't free the ones that are cleared here from the mask. + // In practice this doesn't matter.) + chains_used_mask &= block_list_chain_mask; + } else { + // Reset filters used mask in case --block-list is not + // used, but --filtersX is used. + chains_used_mask = 1U << 0; + } +#endif + // Options for LZMA1 or LZMA2 in case we are using a preset. static lzma_options_lzma opt_lzma; - if (filters_count == 0) { + // The first filter in the chains[] array is for the default + // filter chain. + lzma_filter *default_filters = chains[0]; + + if (filters_count == 0 && chains_used_mask & 1) { // We are using a preset. This is not a good idea in raw mode // except when playing around with things. Different versions // of this software may use different options in presets, and @@ -179,46 +377,61 @@ coder_set_compression_settings(void) message_bug(); // Use LZMA2 except with --format=lzma we use LZMA1. - filters[0].id = opt_format == FORMAT_LZMA + default_filters[0].id = opt_format == FORMAT_LZMA ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2; - filters[0].options = &opt_lzma; + default_filters[0].options = &opt_lzma; + filters_count = 1; - } - // Terminate the filter options array. - filters[filters_count].id = LZMA_VLI_UNKNOWN; + // Terminate the filter options array. + default_filters[1].id = LZMA_VLI_UNKNOWN; + } // If we are using the .lzma format, allow exactly one filter - // which has to be LZMA1. + // which has to be LZMA1. There is no need to check if the default + // filter chain is being used since it can only be disabled if + // --block-list is used, which is incompatible with FORMAT_LZMA. if (opt_format == FORMAT_LZMA && (filters_count != 1 - || filters[0].id != LZMA_FILTER_LZMA1)) + || default_filters[0].id != LZMA_FILTER_LZMA1)) message_fatal(_("The .lzma format supports only " "the LZMA1 filter")); // If we are using the .xz format, make sure that there is no LZMA1 - // filter to prevent LZMA_PROG_ERROR. - if (opt_format == FORMAT_XZ) + // filter to prevent LZMA_PROG_ERROR. With the chains from --filtersX + // we have already ensured this by calling lzma_str_to_filters() + // without setting the flags that would allow non-.xz filters. + if (opt_format == FORMAT_XZ && chains_used_mask & 1) for (size_t i = 0; i < filters_count; ++i) - if (filters[i].id == LZMA_FILTER_LZMA1) + if (default_filters[i].id == LZMA_FILTER_LZMA1) message_fatal(_("LZMA1 cannot be used " "with the .xz format")); - // Print the selected filter chain. - message_filters_show(V_DEBUG, filters); + if (chains_used_mask & 1) { + // Print the selected default filter chain. + message_filters_show(V_DEBUG, default_filters); + } // The --flush-timeout option requires LZMA_SYNC_FLUSH support - // from the filter chain. Currently threaded encoder doesn't support - // LZMA_SYNC_FLUSH so single-threaded mode must be used. + // from the filter chain. Currently the threaded encoder doesn't + // support LZMA_SYNC_FLUSH so single-threaded mode must be used. if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) { - for (size_t i = 0; i < filters_count; ++i) { - switch (filters[i].id) { - case LZMA_FILTER_LZMA2: - case LZMA_FILTER_DELTA: - break; + for (unsigned i = 0; i < ARRAY_SIZE(chains); ++i) { + if (!(chains_used_mask & (1U << i))) + continue; + + const lzma_filter *fc = chains[i]; + for (size_t j = 0; fc[j].id != LZMA_VLI_UNKNOWN; j++) { + switch (fc[j].id) { + case LZMA_FILTER_LZMA2: + case LZMA_FILTER_DELTA: + break; - default: - message_fatal(_("The filter chain is " - "incompatible with --flush-timeout")); + default: + message_fatal(_("Filter chain %u is " + "incompatible with " + "--flush-timeout"), + i); + } } } @@ -229,23 +442,72 @@ coder_set_compression_settings(void) } } - // Get the memory usage. Note that if --format=raw was used, - // we can be decompressing. + // Get memory limit and the memory usage of the used filter chains. + // Note that if --format=raw was used, we can be decompressing + // using the default filter chain. // - // If multithreaded .xz compression is done, this value will be - // replaced. + // If multithreaded .xz compression is done, the memory limit + // will be replaced. uint64_t memory_limit = hardware_memlimit_get(opt_mode); uint64_t memory_usage = UINT64_MAX; + +#ifdef HAVE_ENCODERS + // Memory usage for each encoder filter chain (default + // or --filtersX). The encoder options may need to be + // scaled down depending on the memory usage limit. + uint64_t encoder_memusages[ARRAY_SIZE(chains)]; +#endif + if (opt_mode == MODE_COMPRESS) { #ifdef HAVE_ENCODERS # ifdef MYTHREAD_ENABLED if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) { memory_limit = hardware_memlimit_mtenc_get(); mt_options.threads = hardware_threads_get(); - mt_options.block_size = opt_block_size; + + uint64_t block_size = opt_block_size; + + // If opt_block_size is not set, find the maximum + // recommended Block size based on the filter chains + if (block_size == 0) { + for (unsigned i = 0; i < ARRAY_SIZE(chains); + i++) { + if (!(chains_used_mask & (1U << i))) + continue; + + uint64_t size = lzma_mt_block_size( + chains[i]); + + // If this returns an error, then one + // of the filter chains in use is + // invalid, so there is no point in + // progressing further. + if (size == UINT64_MAX) + message_fatal(_("Unsupported " + "options in filter " + "chain %u"), i); + + if (size > block_size) + block_size = size; + } + + // If --block-list was used and our current + // Block size exceeds the largest size + // in --block-list, reduce the Block size of + // the multithreaded encoder. The extra size + // would only be a waste of RAM. With a + // smaller Block size we might even be able + // to use more threads in some cases. + if (block_list_largest > 0 && block_size + > block_list_largest) + block_size = block_list_largest; + } + + mt_options.block_size = block_size; mt_options.check = check; - memory_usage = lzma_stream_encoder_mt_memusage( - &mt_options); + + memory_usage = get_chains_memusage(encoder_memusages, + &mt_options, true); if (memory_usage != UINT64_MAX) message(V_DEBUG, _("Using up to %" PRIu32 " threads."), @@ -253,12 +515,13 @@ coder_set_compression_settings(void) } else # endif { - memory_usage = lzma_raw_encoder_memusage(filters); + memory_usage = get_chains_memusage(encoder_memusages, + NULL, true); } #endif } else { #ifdef HAVE_DECODERS - memory_usage = lzma_raw_decoder_memusage(filters); + memory_usage = lzma_raw_decoder_memusage(default_filters); #endif } @@ -268,12 +531,13 @@ coder_set_compression_settings(void) // Print memory usage info before possible dictionary // size auto-adjusting. // - // NOTE: If only encoder support was built, we cannot show the + // NOTE: If only encoder support was built, we cannot show // what the decoder memory usage will be. message_mem_needed(V_DEBUG, memory_usage); -#ifdef HAVE_DECODERS - if (opt_mode == MODE_COMPRESS) { - const uint64_t decmem = lzma_raw_decoder_memusage(filters); + +#if defined(HAVE_ENCODERS) && defined(HAVE_DECODERS) + if (opt_mode == MODE_COMPRESS && message_verbosity_get() >= V_DEBUG) { + const uint64_t decmem = get_chains_memusage(NULL, NULL, false); if (decmem != UINT64_MAX) message(V_DEBUG, _("Decompression will need " "%s MiB of memory."), uint64_to_str( @@ -300,14 +564,21 @@ coder_set_compression_settings(void) // Reduce the number of threads by one and check // the memory usage. --mt_options.threads; - memory_usage = lzma_stream_encoder_mt_memusage( - &mt_options); + memory_usage = get_chains_memusage(encoder_memusages, + &mt_options, true); if (memory_usage == UINT64_MAX) message_bug(); if (memory_usage <= memory_limit) { // The memory usage is now low enough. - message(V_WARNING, _("Reduced the number of " + // + // Since 5.6.1: This is only shown at + // V_DEBUG instead of V_WARNING because + // changing the number of threads doesn't + // affect the output. On some systems this + // message would be too common now that + // multithreaded compression is the default. + message(V_DEBUG, _("Reduced the number of " "threads from %s to %s to not exceed " "the memory usage limit of %s MiB"), uint64_to_str( @@ -326,8 +597,11 @@ coder_set_compression_settings(void) // way -T0 won't use insane amount of memory but at the same // time the soft limit will never make xz fail and never make // xz change settings that would affect the compressed output. + // + // Since 5.6.1: Like above, this is now shown at V_DEBUG + // instead of V_WARNING. if (hardware_memlimit_mtenc_is_default()) { - message(V_WARNING, _("Reduced the number of threads " + message(V_DEBUG, _("Reduced the number of threads " "from %s to one. The automatic memory usage " "limit of %s MiB is still being exceeded. " "%s MiB of memory is required. " @@ -353,7 +627,8 @@ coder_set_compression_settings(void) // the multithreaded mode but the output // is also different. hardware_threads_set(1); - memory_usage = lzma_raw_encoder_memusage(filters); + memory_usage = get_chains_memusage(encoder_memusages, + NULL, true); message(V_WARNING, _("Switching to single-threaded mode " "to not exceed the memory usage limit of %s MiB"), uint64_to_str(round_up_to_mib(memory_limit), 0)); @@ -368,55 +643,85 @@ coder_set_compression_settings(void) if (!opt_auto_adjust) memlimit_too_small(memory_usage); - // Look for the last filter if it is LZMA2 or LZMA1, so we can make - // it use less RAM. With other filters we don't know what to do. - size_t i = 0; - while (filters[i].id != LZMA_FILTER_LZMA2 - && filters[i].id != LZMA_FILTER_LZMA1) { - if (filters[i].id == LZMA_VLI_UNKNOWN) - memlimit_too_small(memory_usage); + // Adjust each filter chain that is exceeding the memory usage limit. + for (unsigned i = 0; i < ARRAY_SIZE(chains); i++) { + // Skip unused chains. + if (!(chains_used_mask & (1U << i))) + continue; + + // Skip chains that already meet the memory usage limit. + if (encoder_memusages[i] <= memory_limit) + continue; + + // Look for the last filter if it is LZMA2 or LZMA1, so we + // can make it use less RAM. We cannot adjust other filters. + unsigned j = 0; + while (chains[i][j].id != LZMA_FILTER_LZMA2 + && chains[i][j].id != LZMA_FILTER_LZMA1) { + // NOTE: This displays the too high limit of this + // particular filter chain. If multiple chains are + // specified and another one would need more then + // this message could be confusing. As long as LZMA2 + // is the only memory hungry filter in .xz this + // doesn't matter at all in practice. + // + // FIXME? However, it's sort of odd still if we had + // switched from multithreaded mode to single-threaded + // mode because single-threaded produces different + // output. So the messages could perhaps be clearer. + // Another case of this is a few lines below. + if (chains[i][j].id == LZMA_VLI_UNKNOWN) + memlimit_too_small(encoder_memusages[i]); + + ++j; + } - ++i; - } + // Decrease the dictionary size until we meet the memory + // usage limit. First round down to full mebibytes. + lzma_options_lzma *opt = chains[i][j].options; + const uint32_t orig_dict_size = opt->dict_size; + opt->dict_size &= ~((UINT32_C(1) << 20) - 1); - // Decrease the dictionary size until we meet the memory - // usage limit. First round down to full mebibytes. - lzma_options_lzma *opt = filters[i].options; - const uint32_t orig_dict_size = opt->dict_size; - opt->dict_size &= ~((UINT32_C(1) << 20) - 1); - while (true) { - // If it is below 1 MiB, auto-adjusting failed. We could be - // more sophisticated and scale it down even more, but let's - // see if many complain about this version. - // - // FIXME: Displays the scaled memory usage instead - // of the original. - if (opt->dict_size < (UINT32_C(1) << 20)) - memlimit_too_small(memory_usage); + while (true) { + // If it is below 1 MiB, auto-adjusting failed. + // + // FIXME? See the FIXME a few lines above. + if (opt->dict_size < (UINT32_C(1) << 20)) + memlimit_too_small(encoder_memusages[i]); - memory_usage = lzma_raw_encoder_memusage(filters); - if (memory_usage == UINT64_MAX) - message_bug(); + encoder_memusages[i] + = lzma_raw_encoder_memusage(chains[i]); + if (encoder_memusages[i] == UINT64_MAX) + message_bug(); - // Accept it if it is low enough. - if (memory_usage <= memory_limit) - break; + // Accept it if it is low enough. + if (encoder_memusages[i] <= memory_limit) + break; - // Otherwise 1 MiB down and try again. I hope this - // isn't too slow method for cases where the original - // dict_size is very big. - opt->dict_size -= UINT32_C(1) << 20; - } + // Otherwise adjust it 1 MiB down and try again. + opt->dict_size -= UINT32_C(1) << 20; + } - // Tell the user that we decreased the dictionary size. - message(V_WARNING, _("Adjusted LZMA%c dictionary size " - "from %s MiB to %s MiB to not exceed " - "the memory usage limit of %s MiB"), - filters[i].id == LZMA_FILTER_LZMA2 - ? '2' : '1', - uint64_to_str(orig_dict_size >> 20, 0), - uint64_to_str(opt->dict_size >> 20, 1), - uint64_to_str(round_up_to_mib(memory_limit), 2)); + // Tell the user that we decreased the dictionary size. + // The message is slightly different between the default + // filter chain (0) or and chains from --filtersX. + const char lzma_num = chains[i][j].id == LZMA_FILTER_LZMA2 + ? '2' : '1'; + const char *from_size = uint64_to_str(orig_dict_size >> 20, 0); + const char *to_size = uint64_to_str(opt->dict_size >> 20, 1); + const char *limit_size = uint64_to_str(round_up_to_mib( + memory_limit), 2); + if (i == 0) + message(V_WARNING, _("Adjusted LZMA%c dictionary " + "size from %s MiB to %s MiB to not exceed the " + "memory usage limit of %s MiB"), + lzma_num, from_size, to_size, limit_size); + else + message(V_WARNING, _("Adjusted LZMA%c dictionary size " + "for --filters%u from %s MiB to %s MiB to not " + "exceed the memory usage limit of %s MiB"), + lzma_num, i, from_size, to_size, limit_size); + } #endif return; @@ -512,6 +817,13 @@ coder_init(file_pair *pair) // These will be handled later in this function. allow_trailing_input = false; + // Set the first filter chain. If the --block-list option is not + // used then use the default filter chain (chains[0]). + // Otherwise, use first filter chain from the block list. + lzma_filter *active_filters = opt_block_list == NULL + ? chains[0] + : chains[opt_block_list[0].chain_num]; + if (opt_mode == MODE_COMPRESS) { #ifdef HAVE_ENCODERS switch (opt_format) { @@ -522,17 +834,19 @@ coder_init(file_pair *pair) case FORMAT_XZ: # ifdef MYTHREAD_ENABLED + mt_options.filters = active_filters; if (hardware_threads_is_mt()) ret = lzma_stream_encoder_mt( &strm, &mt_options); else # endif ret = lzma_stream_encoder( - &strm, filters, check); + &strm, active_filters, check); break; case FORMAT_LZMA: - ret = lzma_alone_encoder(&strm, filters[0].options); + ret = lzma_alone_encoder(&strm, + active_filters[0].options); break; # ifdef HAVE_LZIP_DECODER @@ -544,7 +858,7 @@ coder_init(file_pair *pair) # endif case FORMAT_RAW: - ret = lzma_raw_encoder(&strm, filters); + ret = lzma_raw_encoder(&strm, active_filters); break; } #endif @@ -668,7 +982,7 @@ coder_init(file_pair *pair) case FORMAT_RAW: // Memory usage has already been checked in // coder_set_compression_settings(). - ret = lzma_raw_decoder(&strm, filters); + ret = lzma_raw_decoder(&strm, active_filters); break; } @@ -716,6 +1030,7 @@ coder_init(file_pair *pair) } +#ifdef HAVE_ENCODERS /// Resolve conflicts between opt_block_size and opt_block_list in single /// threaded mode. We want to default to opt_block_list, except when it is /// larger than opt_block_size. If this is the case for the current Block @@ -746,12 +1061,39 @@ split_block(uint64_t *block_remaining, } else { // The Block at *list_pos has been finished. Go to the next - // entry in the list. If the end of the list has been reached, - // reuse the size of the last Block. - if (opt_block_list[*list_pos + 1] != 0) + // entry in the list. If the end of the list has been + // reached, reuse the size and filters of the last Block. + if (opt_block_list[*list_pos + 1].size != 0) { ++*list_pos; - *block_remaining = opt_block_list[*list_pos]; + // Update the filters if needed. + if (opt_block_list[*list_pos - 1].chain_num + != opt_block_list[*list_pos].chain_num) { + const unsigned chain_num + = opt_block_list[*list_pos].chain_num; + const lzma_filter *next = chains[chain_num]; + const lzma_ret ret = lzma_filters_update( + &strm, next); + + if (ret != LZMA_OK) { + // This message is only possible if + // the filter chain has unsupported + // options since the filter chain is + // validated using + // lzma_raw_encoder_memusage() or + // lzma_stream_encoder_mt_memusage(). + // Some options are not validated until + // the encoders are initialized. + message_fatal( + _("Error changing to " + "filter chain %u: %s"), + chain_num, + message_strm(ret)); + } + } + } + + *block_remaining = opt_block_list[*list_pos].size; // If in single-threaded mode, split up the Block if needed. // This is not needed in multi-threaded mode because liblzma @@ -764,6 +1106,7 @@ split_block(uint64_t *block_remaining, } } } +#endif static bool @@ -796,6 +1139,7 @@ coder_normal(file_pair *pair) // Assume that something goes wrong. bool success = false; +#ifdef HAVE_ENCODERS // block_remaining indicates how many input bytes to encode before // finishing the current .xz Block. The Block size is set with // --block-size=SIZE and --block-list. They have an effect only when @@ -829,15 +1173,18 @@ coder_normal(file_pair *pair) // output is still not identical because in single-threaded // mode the size info isn't written into Block Headers. if (opt_block_list != NULL) { - if (block_remaining < opt_block_list[list_pos]) { + if (block_remaining < opt_block_list[list_pos].size) { assert(!hardware_threads_is_mt()); - next_block_remaining = opt_block_list[list_pos] + next_block_remaining = + opt_block_list[list_pos].size - block_remaining; } else { - block_remaining = opt_block_list[list_pos]; + block_remaining = + opt_block_list[list_pos].size; } } } +#endif strm.next_out = out_buf.u8; strm.avail_out = IO_BUFFER_SIZE; @@ -847,17 +1194,22 @@ coder_normal(file_pair *pair) // flushing or finishing. if (strm.avail_in == 0 && action == LZMA_RUN) { strm.next_in = in_buf.u8; - strm.avail_in = io_read(pair, &in_buf, - my_min(block_remaining, - IO_BUFFER_SIZE)); +#ifdef HAVE_ENCODERS + const size_t read_size = my_min(block_remaining, + IO_BUFFER_SIZE); +#else + const size_t read_size = IO_BUFFER_SIZE; +#endif + strm.avail_in = io_read(pair, &in_buf, read_size); if (strm.avail_in == SIZE_MAX) break; if (pair->src_eof) { action = LZMA_FINISH; - - } else if (block_remaining != UINT64_MAX) { + } +#ifdef HAVE_ENCODERS + else if (block_remaining != UINT64_MAX) { // Start a new Block after every // opt_block_size bytes of input. block_remaining -= strm.avail_in; @@ -867,6 +1219,7 @@ coder_normal(file_pair *pair) if (action == LZMA_RUN && pair->flush_needed) action = LZMA_SYNC_FLUSH; +#endif } // Let liblzma do the actual work. @@ -878,6 +1231,7 @@ coder_normal(file_pair *pair) break; } +#ifdef HAVE_ENCODERS if (ret == LZMA_STREAM_END && (action == LZMA_SYNC_FLUSH || action == LZMA_FULL_BARRIER)) { if (action == LZMA_SYNC_FLUSH) { @@ -907,8 +1261,9 @@ coder_normal(file_pair *pair) // Start a new Block after LZMA_FULL_FLUSH or continue // the same block after LZMA_SYNC_FLUSH. action = LZMA_RUN; - - } else if (ret != LZMA_OK) { + } else +#endif + if (ret != LZMA_OK) { // Determine if the return value indicates that we // won't continue coding. LZMA_NO_CHECK would be // here too if LZMA_TELL_ANY_CHECK was used. @@ -1103,6 +1458,16 @@ coder_run(const char *filename) extern void coder_free(void) { + // Free starting from the second filter chain since the default + // filter chain may have its options set from a static variable + // in coder_set_compression_settings(). Since this is only run in + // debug mode and will be freed when the process ends anyway, we + // don't worry about freeing it. + for (uint32_t i = 1; i < ARRAY_SIZE(chains); i++) { + if (chains_used_mask & (1U << i)) + lzma_filters_free(chains[i], NULL); + } + lzma_end(&strm); return; } diff --git a/src/xz/coder.h b/src/xz/coder.h index b4f43a2..96755f3 100644 --- a/src/xz/coder.h +++ b/src/xz/coder.h @@ -1,12 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file coder.h /// \brief Compresses or uncompresses a file // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -30,6 +30,16 @@ enum format_type { }; +/// Array of these hold the entries specified with --block-list. +typedef struct { + /// Uncompressed size of the Block + uint64_t size; + + /// Filter chain to use for this Block (chains[chain_num]) + unsigned chain_num; +} block_list_entry; + + /// Operation mode of the command line tool. This is set in args.c and read /// in several files. extern enum operation_mode opt_mode; @@ -50,9 +60,21 @@ extern bool opt_single_stream; /// of input. This has an effect only when compressing to the .xz format. extern uint64_t opt_block_size; -/// This is non-NULL if --block-list was used. This contains the Block sizes -/// as an array that is terminated with 0. -extern uint64_t *opt_block_list; +/// List of block size and filter chain pointer pairs. +extern block_list_entry *opt_block_list; + +/// Size of the largest Block that was specified in --block-list. +/// This is used to limit the block_size option of multithreaded encoder. +/// It's waste of memory to specify a too large block_size and reducing +/// it might even allow using more threads in some cases. +/// +/// NOTE: If the last entry in --block-list is the special value of 0 +/// (which gets converted to UINT64_MAX), it counts here as UINT64_MAX too. +/// This way the multithreaded encoder's Block size won't be reduced. +extern uint64_t block_list_largest; + +/// Bitmask indicating which filter chains we specified in --block-list. +extern uint32_t block_list_chain_mask; /// Set the integrity check type used when compressing extern void coder_set_check(lzma_check check); @@ -77,3 +99,9 @@ extern void coder_run(const char *filename); /// Free the memory allocated for the coder and kill the worker threads. extern void coder_free(void); #endif + +/// Create filter chain from string +extern void coder_add_filters_from_str(const char *filter_str); + +/// Add or overwrite a filter that can be used by the block-list. +extern void coder_add_block_filters(const char *str, size_t slot); diff --git a/src/xz/file_io.c b/src/xz/file_io.c index 2828029..678a9a5 100644 --- a/src/xz/file_io.c +++ b/src/xz/file_io.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file file_io.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -29,15 +28,33 @@ static bool warn_fchown; # include <utime.h> #endif -#ifdef HAVE_CAPSICUM -# ifdef HAVE_SYS_CAPSICUM_H -# include <sys/capsicum.h> +#include "tuklib_open_stdxxx.h" + +#ifdef _MSC_VER +# ifdef _WIN64 + typedef __int64 ssize_t; # else -# include <sys/capability.h> + typedef int ssize_t; # endif -#endif -#include "tuklib_open_stdxxx.h" + typedef int mode_t; +# define S_IRUSR _S_IREAD +# define S_IWUSR _S_IWRITE + +# define setmode _setmode +# define open _open +# define close _close +# define lseek _lseeki64 +# define unlink _unlink + + // The casts are to silence warnings. + // The sizes are known to be small enough. +# define read(fd, buf, size) _read(fd, buf, (unsigned int)(size)) +# define write(fd, buf, size) _write(fd, buf, (unsigned int)(size)) + +# define S_ISDIR(m) (((m) & _S_IFMT) == _S_IFDIR) +# define S_ISREG(m) (((m) & _S_IFMT) == _S_IFREG) +#endif #ifndef O_BINARY # define O_BINARY 0 @@ -66,11 +83,6 @@ typedef enum { /// If true, try to create sparse files when decompressing. static bool try_sparse = true; -#ifdef ENABLE_SANDBOX -/// True if the conditions for sandboxing (described in main()) have been met. -static bool sandbox_allowed = false; -#endif - #ifndef TUKLIB_DOSLIKE /// File status flags of standard input. This is used by io_open_src() /// and io_close_src(). @@ -155,105 +167,6 @@ io_no_sparse(void) } -#ifdef ENABLE_SANDBOX -extern void -io_allow_sandbox(void) -{ - sandbox_allowed = true; - return; -} - - -/// Enables operating-system-specific sandbox if it is possible. -/// src_fd is the file descriptor of the input file. -static void -io_sandbox_enter(int src_fd) -{ - if (!sandbox_allowed) { - // This message is more often annoying than useful so - // it's commented out. It can be useful when developing - // the sandboxing code. - //message(V_DEBUG, _("Sandbox is disabled due " - // "to incompatible command line arguments")); - return; - } - - const char dummy_str[] = "x"; - - // Try to ensure that both libc and xz locale files have been - // loaded when NLS is enabled. - snprintf(NULL, 0, "%s%s", _(dummy_str), strerror(EINVAL)); - - // Try to ensure that iconv data files needed for handling multibyte - // characters have been loaded. This is needed at least with glibc. - tuklib_mbstr_width(dummy_str, NULL); - -#ifdef HAVE_CAPSICUM - // Capsicum needs FreeBSD 10.0 or later. - cap_rights_t rights; - - if (cap_enter()) - goto error; - - if (cap_rights_limit(src_fd, cap_rights_init(&rights, - CAP_EVENT, CAP_FCNTL, CAP_LOOKUP, CAP_READ, CAP_SEEK))) - goto error; - - if (src_fd != STDIN_FILENO && cap_rights_limit( - STDIN_FILENO, cap_rights_clear(&rights))) - goto error; - - if (cap_rights_limit(STDOUT_FILENO, cap_rights_init(&rights, - CAP_EVENT, CAP_FCNTL, CAP_FSTAT, CAP_LOOKUP, - CAP_WRITE, CAP_SEEK))) - goto error; - - if (cap_rights_limit(STDERR_FILENO, cap_rights_init(&rights, - CAP_WRITE))) - goto error; - - if (cap_rights_limit(user_abort_pipe[0], cap_rights_init(&rights, - CAP_EVENT))) - goto error; - - if (cap_rights_limit(user_abort_pipe[1], cap_rights_init(&rights, - CAP_WRITE))) - goto error; - -#elif defined(HAVE_PLEDGE) - // pledge() was introduced in OpenBSD 5.9. - // - // main() unconditionally calls pledge() with fairly relaxed - // promises which work in all situations. Here we make the - // sandbox more strict. - if (pledge("stdio", "")) - goto error; - - (void)src_fd; - -#else -# error ENABLE_SANDBOX is defined but no sandboxing method was found. -#endif - - // This message is annoying in xz -lvv. - //message(V_DEBUG, _("Sandbox was successfully enabled")); - return; - -error: -#ifdef HAVE_CAPSICUM - // If a kernel is configured without capability mode support or - // used in an emulator that does not implement the capability - // system calls, then the Capsicum system calls will fail and set - // errno to ENOSYS. In that case xz will silently run without - // the sandbox. - if (errno == ENOSYS) - return; -#endif - message_fatal(_("Failed to enable the sandbox")); -} -#endif // ENABLE_SANDBOX - - #ifndef TUKLIB_DOSLIKE /// \brief Waits for input or output to become available or for a signal /// @@ -407,7 +320,7 @@ io_copy_attrs(const file_pair *pair) message_warning(_("%s: Cannot set the file group: %s"), pair->dest_name, strerror(errno)); // We can still safely copy some additional permissions: - // `group' must be at least as strict as `other' and + // 'group' must be at least as strict as 'other' and // also vice versa. // // NOTE: After this, the owner of the source file may @@ -809,7 +722,8 @@ io_open_src(const char *src_name) #ifdef ENABLE_SANDBOX if (!error) - io_sandbox_enter(pair.src_fd); + sandbox_enable_strict_if_allowed(pair.src_fd, + user_abort_pipe[0], user_abort_pipe[1]); #endif return error ? NULL : &pair; diff --git a/src/xz/file_io.h b/src/xz/file_io.h index 6992efa..ae7e2f3 100644 --- a/src/xz/file_io.h +++ b/src/xz/file_io.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file file_io.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// // Some systems have suboptimal BUFSIZ. Use a bit bigger value on them. @@ -18,6 +17,16 @@ # define IO_BUFFER_SIZE (BUFSIZ & ~7U) #endif +#ifdef _MSC_VER + // The first one renames both "struct stat" -> "struct _stat64" + // and stat() -> _stat64(). The documentation mentions only + // "struct __stat64", not "struct _stat64", but the latter + // works too. +# define stat _stat64 +# define fstat _fstat64 +# define off_t __int64 +#endif + /// is_sparse() accesses the buffer as uint64_t for maximum speed. /// The u32 and u64 members must only be access through this union @@ -90,12 +99,6 @@ extern void io_write_to_user_abort_pipe(void); extern void io_no_sparse(void); -#ifdef ENABLE_SANDBOX -/// \brief main() calls this if conditions for sandboxing have been met. -extern void io_allow_sandbox(void); -#endif - - /// \brief Open the source file extern file_pair *io_open_src(const char *src_name); diff --git a/src/xz/hardware.c b/src/xz/hardware.c index c694882..952652f 100644 --- a/src/xz/hardware.c +++ b/src/xz/hardware.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -15,7 +14,7 @@ /// Maximum number of worker threads. This can be set with /// the --threads=NUM command line option. -static uint32_t threads_max = 1; +static uint32_t threads_max; /// True when the number of threads is automatically determined based /// on the available hardware threads. @@ -334,5 +333,9 @@ hardware_init(void) memlimit_mt_default = mem_ceiling; #endif + // Enable threaded mode by default. xz 5.4.x and older + // used single-threaded mode by default. + hardware_threads_set(0); + return; } diff --git a/src/xz/hardware.h b/src/xz/hardware.h index a67b26e..25b351e 100644 --- a/src/xz/hardware.h +++ b/src/xz/hardware.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// Initialize some hardware-specific variables, which are needed by other diff --git a/src/xz/list.c b/src/xz/list.c index 86c3a76..ca9cf03 100644 --- a/src/xz/list.c +++ b/src/xz/list.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file list.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -544,11 +543,21 @@ parse_block_header(file_pair *pair, const lzma_index_iter *iter, xfi->memusage_max = bhi->memusage; // Determine the minimum XZ Utils version that supports this Block. + // - RISC-V filter needs 5.6.0. // // - ARM64 filter needs 5.4.0. // // - 5.0.0 doesn't support empty LZMA2 streams and thus empty // Blocks that use LZMA2. This decoder bug was fixed in 5.0.2. + if (xfi->min_version < 50060002U) { + for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { + if (filters[i].id == LZMA_FILTER_RISCV) { + xfi->min_version = 50060002U; + break; + } + } + } + if (xfi->min_version < 50040002U) { for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { if (filters[i].id == LZMA_FILTER_ARM64) { @@ -1266,10 +1275,22 @@ list_totals(void) extern void list_file(const char *filename) { - if (opt_format != FORMAT_XZ && opt_format != FORMAT_AUTO) - message_fatal(_("--list works only on .xz files " + if (opt_format != FORMAT_XZ && opt_format != FORMAT_AUTO) { + // The 'lzmainfo' message is printed only when --format=lzma + // is used (it is implied if using "lzma" as the command + // name). Thus instead of using message_fatal(), print + // the messages separately and then call tuklib_exit() + // like message_fatal() does. + message(V_ERROR, _("--list works only on .xz files " "(--format=xz or --format=auto)")); + if (opt_format == FORMAT_LZMA) + message(V_ERROR, + _("Try 'lzmainfo' with .lzma files.")); + + tuklib_exit(E_ERROR, E_ERROR, false); + } + message_filename(filename); if (filename == stdin_filename) { diff --git a/src/xz/list.h b/src/xz/list.h index a4c6ec7..805880d 100644 --- a/src/xz/list.h +++ b/src/xz/list.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file list.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// \brief List information about the given .xz file diff --git a/src/xz/main.c b/src/xz/main.c index c9c3dec..71b5ef7 100644 --- a/src/xz/main.c +++ b/src/xz/main.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file main.c @@ -5,14 +7,12 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" #include <ctype.h> + /// Exit status to use. This can be changed with set_exit_status(). static enum exit_status_type exit_status = E_SUCCESS; @@ -119,8 +119,8 @@ read_name(const args_info *args) // newlines. message_error(_("%s: Null character found when " "reading filenames; maybe you meant " - "to use `--files0' instead " - "of `--files'?"), args->files_name); + "to use '--files0' instead " + "of '--files'?"), args->files_name); return NULL; } @@ -142,35 +142,38 @@ read_name(const args_info *args) int main(int argc, char **argv) { -#ifdef HAVE_PLEDGE - // OpenBSD's pledge(2) sandbox - // - // Unconditionally enable sandboxing with fairly relaxed promises. - // This is still way better than having no sandbox at all. :-) - // More strict promises will be made later in file_io.c if possible. - if (pledge("stdio rpath wpath cpath fattr", "")) { - // Don't translate the string or use message_fatal() as - // those haven't been initialized yet. - fprintf(stderr, "%s: Failed to enable the sandbox\n", argv[0]); - return E_ERROR; - } -#endif - #if defined(_WIN32) && !defined(__CYGWIN__) InitializeCriticalSection(&exit_status_cs); #endif - // Set up the progname variable. + // Set up the progname variable needed for messages. tuklib_progname_init(argv); // Initialize the file I/O. This makes sure that // stdin, stdout, and stderr are something valid. + // This must be done before we might open any files + // even indirectly like locale and gettext initializations. io_init(); +#ifdef ENABLE_SANDBOX + // Enable such sandboxing that can always be enabled. + // This requires that progname has been set up. + // It's also good that io_init() has been called because it + // might need to do things that the initial sandbox won't allow. + // Otherwise this should be called as early as possible. + // + // NOTE: Calling this before tuklib_gettext_init() means that + // translated error message won't be available if sandbox + // initialization fails. However, sandbox_init() shouldn't + // fail and this order simply feels better. + sandbox_init(); +#endif + // Set up the locale and message translations. tuklib_gettext_init(PACKAGE, LOCALEDIR); - // Initialize handling of error/warning/other messages. + // Initialize progress message handling. It's not always needed + // but it's simpler to do this unconditionally. message_init(); // Set hardware-dependent default values. These can be overridden @@ -220,21 +223,41 @@ main(int argc, char **argv) signals_init(); #ifdef ENABLE_SANDBOX - // Set a flag that sandboxing is allowed if all these are true: - // - --files or --files0 wasn't used. - // - There is exactly one input file or we are reading from stdin. - // - We won't create any files: output goes to stdout or --test - // or --list was used. Note that --test implies opt_stdout = true - // but --list doesn't. + // Read-only sandbox can be enabled if we won't create or delete + // any files: // - // This is obviously not ideal but it was easy to implement and - // it covers the most common use cases. + // - --stdout, --test, or --list was used. Note that --test + // implies opt_stdout = true but --list doesn't. // - // TODO: Make sandboxing work for other situations too. - if (args.files_name == NULL && args.arg_count == 1 - && (opt_stdout || strcmp("-", args.arg_names[0]) == 0 - || opt_mode == MODE_LIST)) - io_allow_sandbox(); + // - Output goes to stdout because --files or --files0 wasn't used + // and no arguments were given on the command line or the + // arguments are all "-" (indicating standard input). + bool to_stdout_only = opt_stdout || opt_mode == MODE_LIST; + if (!to_stdout_only && args.files_name == NULL) { + // If all of the filenames provided are "-" (more than one + // "-" could be specified), then we are only going to be + // writing to standard output. Note that if no filename args + // were provided, args.c puts a single "-" in arg_names[0]. + to_stdout_only = true; + + for (unsigned i = 0; i < args.arg_count; ++i) { + if (strcmp("-", args.arg_names[i]) != 0) { + to_stdout_only = false; + break; + } + } + } + + if (to_stdout_only) { + sandbox_enable_read_only(); + + // Allow strict sandboxing if we are processing exactly one + // file to standard output. This requires that --files or + // --files0 wasn't specified (an unknown number of filenames + // could be provided that way). + if (args.files_name == NULL && args.arg_count == 1) + sandbox_allow_strict(); + } #endif // coder_run() handles compression, decompression, and testing. diff --git a/src/xz/main.h b/src/xz/main.h index 323f2f7..a8a1b45 100644 --- a/src/xz/main.h +++ b/src/xz/main.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file main.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// Possible exit status values. These are the same as used by gzip and bzip2. diff --git a/src/xz/message.c b/src/xz/message.c index abf30ad..deafdb4 100644 --- a/src/xz/message.c +++ b/src/xz/message.c @@ -1,12 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file message.c /// \brief Printing messages // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -42,7 +42,7 @@ static bool current_filename_printed = false; /// True if we should print progress indicator and update it automatically /// if also verbose >= V_VERBOSE. -static bool progress_automatic; +static bool progress_automatic = false; /// True if message_progress_start() has been called but /// message_progress_end() hasn't been called yet. @@ -119,26 +119,7 @@ message_init(void) // exception, even if --verbose was not used, user can send SIGALRM // to make us print progress information once without automatic // updating. - progress_automatic = isatty(STDERR_FILENO); - - // Commented out because COLUMNS is rarely exported to environment. - // Most users have at least 80 columns anyway, let's think something - // fancy here if enough people complain. -/* - if (progress_automatic) { - // stderr is a terminal. Check the COLUMNS environment - // variable to see if the terminal is wide enough. If COLUMNS - // doesn't exist or it has some unparsable value, we assume - // that the terminal is wide enough. - const char *columns_str = getenv("COLUMNS"); - if (columns_str != NULL) { - char *endptr; - const long columns = strtol(columns_str, &endptr, 10); - if (*endptr != '\0' || columns < 80) - progress_automatic = false; - } - } -*/ + progress_automatic = is_tty(STDERR_FILENO); #ifdef SIGALRM // Establish the signal handlers which set a flag to tell us that @@ -932,7 +913,7 @@ message_try_help(void) { // Print this with V_WARNING instead of V_ERROR to prevent it from // showing up when --quiet has been specified. - message(V_WARNING, _("Try `%s --help' for more information."), + message(V_WARNING, _("Try '%s --help' for more information."), progname); return; } @@ -994,7 +975,7 @@ message_help(bool long_help) " ignore possible remaining input data")); puts(_( " --no-sparse do not create sparse files when decompressing\n" -" -S, --suffix=.SUF use the suffix `.SUF' on compressed files\n" +" -S, --suffix=.SUF use the suffix '.SUF' on compressed files\n" " --files[=FILE] read filenames to process from FILE; if FILE is\n" " omitted, filenames are read from the standard input;\n" " filenames must be terminated with the newline character\n" @@ -1005,9 +986,9 @@ message_help(bool long_help) puts(_("\n Basic file format and compression options:\n")); puts(_( " -F, --format=FMT file format to encode or decode; possible values are\n" -" `auto' (default), `xz', `lzma', `lzip', and `raw'\n" -" -C, --check=CHECK integrity check type: `none' (use with caution),\n" -" `crc32', `crc64' (default), or `sha256'")); +" 'auto' (default), 'xz', 'lzma', 'lzip', and 'raw'\n" +" -C, --check=CHECK integrity check type: 'none' (use with caution),\n" +" 'crc32', 'crc64' (default), or 'sha256'")); puts(_( " --ignore-check don't verify the integrity check when decompressing")); } @@ -1021,8 +1002,8 @@ message_help(bool long_help) " does not affect decompressor memory requirements")); puts(_( -" -T, --threads=NUM use at most NUM threads; the default is 1; set to 0\n" -" to use as many threads as there are processor cores")); +" -T, --threads=NUM use at most NUM threads; the default is 0 which uses\n" +" as many threads as there are processor cores")); if (long_help) { puts(_( @@ -1030,9 +1011,11 @@ message_help(bool long_help) " start a new .xz block after every SIZE bytes of input;\n" " use this to set the block size for threaded compression")); puts(_( -" --block-list=SIZES\n" +" --block-list=BLOCKS\n" " start a new .xz block after the given comma-separated\n" -" intervals of uncompressed data")); +" intervals of uncompressed data; optionally, specify a\n" +" filter chain number (0-9) followed by a ':' before the\n" +" uncompressed data size")); puts(_( " --flush-timeout=TIMEOUT\n" " when compressing, if more than TIMEOUT milliseconds has\n" @@ -1057,6 +1040,23 @@ message_help(bool long_help) puts(_( "\n Custom filter chain for compression (alternative for using presets):")); + puts(_( +"\n" +" --filters=FILTERS set the filter chain using the liblzma filter string\n" +" syntax; use --filters-help for more information" + )); + + puts(_( +" --filters1=FILTERS ... --filters9=FILTERS\n" +" set additional filter chains using the liblzma filter\n" +" string syntax to use with --block-list" + )); + + puts(_( +" --filters-help display more information about the liblzma filter string\n" +" syntax and exit." + )); + #if defined(HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) \ || defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2) // TRANSLATORS: The word "literal" in "literal context bits" @@ -1087,6 +1087,7 @@ message_help(bool long_help) " --powerpc[=OPTS] PowerPC BCJ filter (big endian only)\n" " --ia64[=OPTS] IA-64 (Itanium) BCJ filter\n" " --sparc[=OPTS] SPARC BCJ filter\n" +" --riscv[=OPTS] RISC-V BCJ filter\n" " Valid OPTS for all BCJ filters:\n" " start=NUM start offset for conversions (default=0)")); @@ -1144,3 +1145,28 @@ message_help(bool long_help) tuklib_exit(E_SUCCESS, E_ERROR, verbosity != V_SILENT); } + + +extern void +message_filters_help(void) +{ + char *encoder_options; + if (lzma_str_list_filters(&encoder_options, LZMA_VLI_UNKNOWN, + LZMA_STR_ENCODER, NULL) != LZMA_OK) + message_bug(); + + if (!opt_robot) { + puts(_( +"Filter chains are set using the --filters=FILTERS or\n" +"--filters1=FILTERS ... --filters9=FILTERS options. Each filter in the chain\n" +"can be separated by spaces or '--'. Alternatively a preset <0-9>[e] can be\n" +"specified instead of a filter chain.\n" + )); + + puts(_("The supported filters and their options are:")); + } + + puts(encoder_options); + + tuklib_exit(E_SUCCESS, E_ERROR, verbosity != V_SILENT); +} diff --git a/src/xz/message.h b/src/xz/message.h index f608ec7..3f6e183 100644 --- a/src/xz/message.h +++ b/src/xz/message.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file message.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// Verbosity levels @@ -24,7 +23,10 @@ enum message_verbosity { extern const int message_progress_sigs[]; -/// \brief Initializes the message functions +/// \brief Initializes the progress message functions +/// +/// message_fatal() and such can be called even before message_init() +/// has been called. /// /// If an error occurs, this function doesn't return. /// @@ -111,6 +113,12 @@ tuklib_attr_noreturn extern void message_help(bool long_help); +/// Prints a help message specifically for using the --filters and +/// --filtersX command line options. +tuklib_attr_noreturn +extern void message_filters_help(void); + + /// \brief Set the total number of files to be processed /// /// Standard input is counted as a file here. This is used when printing diff --git a/src/xz/mytime.c b/src/xz/mytime.c index 7e8a074..7d9a27d 100644 --- a/src/xz/mytime.c +++ b/src/xz/mytime.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file mytime.c @@ -5,14 +7,14 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" -#if defined(HAVE_CLOCK_GETTIME) && defined(HAVE_CLOCK_MONOTONIC) +#if defined(MYTHREAD_VISTA) || defined(_MSC_VER) + // Nothing +#elif defined(HAVE_CLOCK_GETTIME) \ + && (!defined(__MINGW32__) || defined(MYTHREAD_POSIX)) # include <time.h> #else # include <sys/time.h> @@ -20,7 +22,22 @@ uint64_t opt_flush_timeout = 0; +// start_time holds the time when the (de)compression was started. +// It's from mytime_now() and thus only useful for calculating relative +// time differences (elapsed time). start_time is initialized by calling +// mytime_set_start_time() and modified by mytime_sigtstp_handler(). +// +// When mytime_sigtstp_handler() is used, start_time is made volatile. +// I'm not sure if that is really required since access to it is guarded +// by signals_block()/signals_unblock() since accessing an uint64_t isn't +// atomic on all systems. But since the variable isn't accessed very +// frequently making it volatile doesn't hurt. +#ifdef USE_SIGTSTP_HANDLER +static volatile uint64_t start_time; +#else static uint64_t start_time; +#endif + static uint64_t next_flush; @@ -30,16 +47,41 @@ static uint64_t next_flush; static uint64_t mytime_now(void) { -#if defined(HAVE_CLOCK_GETTIME) && defined(HAVE_CLOCK_MONOTONIC) +#if defined(MYTHREAD_VISTA) || defined(_MSC_VER) + // Since there is no SIGALRM on Windows, this function gets + // called frequently when the progress indicator is in use. + // Progress indicator doesn't need high-resolution time. + // GetTickCount64() has very low overhead but needs at least WinVista. + // + // MinGW-w64 provides the POSIX functions clock_gettime() and + // gettimeofday() in a manner that allow xz to run on older + // than WinVista. If the threading method needs WinVista anyway, + // there's no reason to avoid a WinVista API here either. + return GetTickCount64(); + +#elif defined(HAVE_CLOCK_GETTIME) \ + && (!defined(__MINGW32__) || defined(MYTHREAD_POSIX)) + // MinGW-w64: clock_gettime() is defined in winpthreads but we need + // nothing else from winpthreads (unless, for some odd reason, POSIX + // threading has been selected). By avoiding clock_gettime(), we + // avoid the dependency on libwinpthread-1.dll or the need to link + // against the static version. The downside is that the fallback + // method, gettimeofday(), doesn't provide monotonic time. + struct timespec tv; + +# ifdef HAVE_CLOCK_MONOTONIC // If CLOCK_MONOTONIC was available at compile time but for some // reason isn't at runtime, fallback to CLOCK_REALTIME which // according to POSIX is mandatory for all implementations. static clockid_t clk_id = CLOCK_MONOTONIC; - struct timespec tv; while (clock_gettime(clk_id, &tv)) clk_id = CLOCK_REALTIME; +# else + clock_gettime(CLOCK_REALTIME, &tv); +# endif return (uint64_t)tv.tv_sec * 1000 + (uint64_t)(tv.tv_nsec / 1000000); + #else struct timeval tv; gettimeofday(&tv, NULL); @@ -48,10 +90,49 @@ mytime_now(void) } +#ifdef USE_SIGTSTP_HANDLER +extern void +mytime_sigtstp_handler(int sig lzma_attribute((__unused__))) +{ + // Measure how long the process stays in the stopped state and add + // that amount to start_time. This way the the progress indicator + // won't count the stopped time as elapsed time and the estimated + // remaining time won't be confused by the time spent in the + // stopped state. + // + // FIXME? Is raising SIGSTOP the correct thing to do? POSIX.1-2017 + // says that orphan processes shouldn't stop on SIGTSTP. So perhaps + // the most correct thing to do could be to revert to the default + // handler for SIGTSTP, unblock SIGTSTP, and then raise(SIGTSTP). + // It's quite a bit more complicated than just raising SIGSTOP though. + // + // The difference between raising SIGTSTP vs. SIGSTOP can be seen on + // the shell command line too by running "echo $?" after stopping + // a process but perhaps that doesn't matter. + const uint64_t t = mytime_now(); + raise(SIGSTOP); + start_time += mytime_now() - t; + return; +} +#endif + + extern void mytime_set_start_time(void) { +#ifdef USE_SIGTSTP_HANDLER + // Block the signals when accessing start_time so that we cannot + // end up with a garbage value. start_time is volatile but access + // to it isn't atomic at least on 32-bit systems. + signals_block(); +#endif + start_time = mytime_now(); + +#ifdef USE_SIGTSTP_HANDLER + signals_unblock(); +#endif + return; } @@ -59,7 +140,17 @@ mytime_set_start_time(void) extern uint64_t mytime_get_elapsed(void) { - return mytime_now() - start_time; +#ifdef USE_SIGTSTP_HANDLER + signals_block(); +#endif + + const uint64_t t = mytime_now() - start_time; + +#ifdef USE_SIGTSTP_HANDLER + signals_unblock(); +#endif + + return t; } diff --git a/src/xz/mytime.h b/src/xz/mytime.h index a7be2aa..6dfaeae 100644 --- a/src/xz/mytime.h +++ b/src/xz/mytime.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file mytime.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// @@ -21,6 +20,12 @@ extern uint64_t opt_flush_timeout; +#ifdef USE_SIGTSTP_HANDLER +/// \brief Signal handler for SIGTSTP +extern void mytime_sigtstp_handler(int sig); +#endif + + /// \brief Store the time when (de)compression was started /// /// The start time is also stored as the time of the first flush. diff --git a/src/xz/options.c b/src/xz/options.c index 4d5e899..bc8bc1a 100644 --- a/src/xz/options.c +++ b/src/xz/options.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file options.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -83,7 +82,7 @@ parse_options(const char *str, const option_map *opts, *value++ = '\0'; if (value == NULL || value[0] == '\0') - message_fatal(_("%s: Options must be `name=value' " + message_fatal(_("%s: Options must be 'name=value' " "pairs separated with commas"), str); // Look for the option name from the option map. diff --git a/src/xz/options.h b/src/xz/options.h index 61ec8d5..4a1314d 100644 --- a/src/xz/options.h +++ b/src/xz/options.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file options.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// \brief Parser for Delta options diff --git a/src/xz/private.h b/src/xz/private.h index 6414bdb..b370472 100644 --- a/src/xz/private.h +++ b/src/xz/private.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file private.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "sysdefs.h" @@ -21,7 +20,10 @@ #include <signal.h> #include <locale.h> #include <stdio.h> -#include <unistd.h> + +#ifndef _MSC_VER +# include <unistd.h> +#endif #include "tuklib_gettext.h" #include "tuklib_progname.h" @@ -33,6 +35,10 @@ # include <windows.h> #endif +#ifdef _MSC_VER +# define fileno _fileno +#endif + #ifndef STDIN_FILENO # define STDIN_FILENO (fileno(stdin)) #endif @@ -45,8 +51,15 @@ # define STDERR_FILENO (fileno(stderr)) #endif -#if defined(HAVE_CAPSICUM) || defined(HAVE_PLEDGE) -# define ENABLE_SANDBOX 1 +// Handling SIGTSTP keeps time-keeping for progress indicator correct +// if xz is stopped. It requires use of clock_gettime() as that is +// async-signal safe in POSIX. Require also SIGALRM support since +// on systems where SIGALRM isn't available, progress indicator code +// polls the time and the SIGTSTP handling adds slight overhead to +// that code. Most (all?) systems that have SIGTSTP also have SIGALRM +// so this requirement won't exclude many systems. +#if defined(HAVE_CLOCK_GETTIME) && defined(SIGTSTP) && defined(SIGALRM) +# define USE_SIGTSTP_HANDLER 1 #endif #include "main.h" @@ -57,6 +70,7 @@ #include "hardware.h" #include "file_io.h" #include "options.h" +#include "sandbox.h" #include "signals.h" #include "suffix.h" #include "util.h" diff --git a/src/xz/sandbox.c b/src/xz/sandbox.c new file mode 100644 index 0000000..5bd2273 --- /dev/null +++ b/src/xz/sandbox.c @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file sandbox.c +/// \brief Sandbox support +// +// Author: Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#include "private.h" + + +#ifndef ENABLE_SANDBOX + +// Prevent an empty translation unit when no sandboxing is supported. +typedef int dummy; + +#else + +/// If the conditions for strict sandboxing (described in main()) +/// have been met, sandbox_allow_strict() can be called to set this +/// variable to true. +static bool strict_sandbox_allowed = false; + + +extern void +sandbox_allow_strict(void) +{ + strict_sandbox_allowed = true; + return; +} + + +// Strict sandboxing prevents opening any files. This *tries* to ensure +// that any auxiliary files that might be required are already open. +// +// Returns true if strict sandboxing is allowed, false otherwise. +static bool +prepare_for_strict_sandbox(void) +{ + if (!strict_sandbox_allowed) + return false; + + const char dummy_str[] = "x"; + + // Try to ensure that both libc and xz locale files have been + // loaded when NLS is enabled. + snprintf(NULL, 0, "%s%s", _(dummy_str), strerror(EINVAL)); + + // Try to ensure that iconv data files needed for handling multibyte + // characters have been loaded. This is needed at least with glibc. + tuklib_mbstr_width(dummy_str, NULL); + + return true; +} + +#endif + + +#if defined(HAVE_PLEDGE) + +/////////////// +// pledge(2) // +/////////////// + +#include <unistd.h> + + +extern void +sandbox_init(void) +{ + if (pledge("stdio rpath wpath cpath fattr", "")) { + // gettext hasn't been initialized yet so + // there's no point to call it here. + message_fatal("Failed to enable the sandbox"); + } + + return; +} + + +extern void +sandbox_enable_read_only(void) +{ + // We will be opening files for reading but + // won't create or remove any files. + if (pledge("stdio rpath", "")) + message_fatal(_("Failed to enable the sandbox")); + + return; +} + + +extern void +sandbox_enable_strict_if_allowed(int src_fd lzma_attribute((__unused__)), + int pipe_event_fd lzma_attribute((__unused__)), + int pipe_write_fd lzma_attribute((__unused__))) +{ + if (!prepare_for_strict_sandbox()) + return; + + // All files that need to be opened have already been opened. + if (pledge("stdio", "")) + message_fatal(_("Failed to enable the sandbox")); + + return; +} + + +#elif defined(HAVE_LINUX_LANDLOCK) + +////////////// +// Landlock // +////////////// + +#include <linux/landlock.h> +#include <sys/syscall.h> +#include <sys/prctl.h> + + +// Highest Landlock ABI version supported by this file: +// - For ABI versions 1-3 we don't need anything from <linux/landlock.h> +// that isn't part of version 1. +// - For ABI version 4 we need the larger struct landlock_ruleset_attr +// with the handled_access_net member. That is bundled with the macros +// LANDLOCK_ACCESS_NET_BIND_TCP and LANDLOCK_ACCESS_NET_CONNECT_TCP. +#ifdef LANDLOCK_ACCESS_NET_BIND_TCP +# define LANDLOCK_ABI_MAX 4 +#else +# define LANDLOCK_ABI_MAX 3 +#endif + + +/// Landlock ABI version supported by the kernel +static int landlock_abi; + + +// The required_rights should have those bits set that must not be restricted. +// This function will then bitwise-and ~required_rights with a mask matching +// the Landlock ABI version, leaving only those bits set that are supported +// by the ABI and allowed to be restricted by the function argument. +static void +enable_landlock(uint64_t required_rights) +{ + assert(landlock_abi <= LANDLOCK_ABI_MAX); + + if (landlock_abi <= 0) + return; + + // We want to set all supported flags in handled_access_fs. + // This way the ruleset will initially forbid access to all + // actions that the available Landlock ABI version supports. + // Exceptions can be added using landlock_add_rule(2) to + // allow certain actions on certain files or directories. + // + // The same flag values are used on all archs. ABI v2 and v3 + // both add one new flag. + // + // First in ABI v1: LANDLOCK_ACCESS_FS_EXECUTE = 1ULL << 0 + // Last in ABI v1: LANDLOCK_ACCESS_FS_MAKE_SYM = 1ULL << 12 + // Last in ABI v2: LANDLOCK_ACCESS_FS_REFER = 1ULL << 13 + // Last in ABI v3: LANDLOCK_ACCESS_FS_TRUNCATE = 1ULL << 14 + // + // This makes it simple to set the mask based on the ABI + // version and we don't need to care which flags are #defined + // in the installed <linux/landlock.h> for ABI versions 1-3. + const struct landlock_ruleset_attr attr = { + .handled_access_fs = ~required_rights + & ((1ULL << (12 + my_min(3, landlock_abi))) - 1), +#if LANDLOCK_ABI_MAX >= 4 + .handled_access_net = landlock_abi < 4 ? 0 : + (LANDLOCK_ACCESS_NET_BIND_TCP + | LANDLOCK_ACCESS_NET_CONNECT_TCP), +#endif + }; + + const int ruleset_fd = syscall(SYS_landlock_create_ruleset, + &attr, sizeof(attr), 0U); + if (ruleset_fd < 0) + message_fatal(_("Failed to enable the sandbox")); + + // All files we need should have already been opened. Thus, + // we don't need to add any rules using landlock_add_rule(2) + // before activating the sandbox. + // + // NOTE: It's possible that the hack prepare_for_strict_sandbox() + // isn't be good enough. It tries to get translations and + // libc-specific files loaded but if it's not good enough + // then perhaps a Landlock rule to allow reading from /usr + // and/or the xz installation prefix would be needed. + // + // prctl(PR_SET_NO_NEW_PRIVS, ...) was already called in + // sandbox_init() so we don't do it here again. + if (syscall(SYS_landlock_restrict_self, ruleset_fd, 0U) != 0) + message_fatal(_("Failed to enable the sandbox")); + + return; +} + + +extern void +sandbox_init(void) +{ + // Prevent the process from gaining new privileges. This must be done + // before landlock_restrict_self(2) but since we will never need new + // privileges, this call can be done here already. + // + // This is supported since Linux 3.5. Ignore the return value to + // keep compatibility with old kernels. landlock_restrict_self(2) + // will fail if the no_new_privs attribute isn't set, thus if prctl() + // fails here the error will still be detected when it matters. + (void)prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + + // Get the highest Landlock ABI version supported by the kernel. + landlock_abi = syscall(SYS_landlock_create_ruleset, + (void *)NULL, 0, LANDLOCK_CREATE_RULESET_VERSION); + + // The kernel might support a newer ABI than this file. + if (landlock_abi > LANDLOCK_ABI_MAX) + landlock_abi = LANDLOCK_ABI_MAX; + + // These are all in ABI version 1 already. We don't need truncate + // rights because files are created with open() using O_EXCL and + // without O_TRUNC. + // + // LANDLOCK_ACCESS_FS_READ_DIR is included here to get a clear error + // message if xz is given a directory name. Without this permission + // the message would be "Permission denied" but with this permission + // it's "Is a directory, skipping". It could be worked around with + // stat()/lstat() but just giving this permission is simpler and + // shouldn't make the sandbox much weaker in practice. + const uint64_t required_rights + = LANDLOCK_ACCESS_FS_WRITE_FILE + | LANDLOCK_ACCESS_FS_READ_FILE + | LANDLOCK_ACCESS_FS_READ_DIR + | LANDLOCK_ACCESS_FS_REMOVE_FILE + | LANDLOCK_ACCESS_FS_MAKE_REG; + + enable_landlock(required_rights); + return; +} + + +extern void +sandbox_enable_read_only(void) +{ + // We will be opening files for reading but + // won't create or remove any files. + const uint64_t required_rights + = LANDLOCK_ACCESS_FS_READ_FILE + | LANDLOCK_ACCESS_FS_READ_DIR; + enable_landlock(required_rights); + return; +} + + +extern void +sandbox_enable_strict_if_allowed(int src_fd lzma_attribute((__unused__)), + int pipe_event_fd lzma_attribute((__unused__)), + int pipe_write_fd lzma_attribute((__unused__))) +{ + if (!prepare_for_strict_sandbox()) + return; + + // Allow all restrictions that the kernel supports with the + // highest Landlock ABI version that the kernel or xz supports. + // + // NOTE: LANDLOCK_ACCESS_FS_READ_DIR isn't needed here because + // the only input file has already been opened. + enable_landlock(0); + return; +} + + +#elif defined(HAVE_CAP_RIGHTS_LIMIT) + +////////////// +// Capsicum // +////////////// + +#include <sys/capsicum.h> + + +extern void +sandbox_init(void) +{ + // Nothing to do. + return; +} + + +extern void +sandbox_enable_read_only(void) +{ + // Nothing to do. + return; +} + + +extern void +sandbox_enable_strict_if_allowed( + int src_fd, int pipe_event_fd, int pipe_write_fd) +{ + if (!prepare_for_strict_sandbox()) + return; + + // Capsicum needs FreeBSD 10.2 or later. + cap_rights_t rights; + + if (cap_enter()) + goto error; + + if (cap_rights_limit(src_fd, cap_rights_init(&rights, + CAP_EVENT, CAP_FCNTL, CAP_LOOKUP, CAP_READ, CAP_SEEK))) + goto error; + + // If not reading from stdin, remove all capabilities from it. + if (src_fd != STDIN_FILENO && cap_rights_limit( + STDIN_FILENO, cap_rights_clear(&rights))) + goto error; + + if (cap_rights_limit(STDOUT_FILENO, cap_rights_init(&rights, + CAP_EVENT, CAP_FCNTL, CAP_FSTAT, CAP_LOOKUP, + CAP_WRITE, CAP_SEEK))) + goto error; + + if (cap_rights_limit(STDERR_FILENO, cap_rights_init(&rights, + CAP_WRITE))) + goto error; + + if (cap_rights_limit(pipe_event_fd, cap_rights_init(&rights, + CAP_EVENT))) + goto error; + + if (cap_rights_limit(pipe_write_fd, cap_rights_init(&rights, + CAP_WRITE))) + goto error; + + return; + +error: + // If a kernel is configured without capability mode support or + // used in an emulator that does not implement the capability + // system calls, then the Capsicum system calls will fail and set + // errno to ENOSYS. In that case xz will silently run without + // the sandbox. + if (errno == ENOSYS) + return; + + message_fatal(_("Failed to enable the sandbox")); +} + +#endif diff --git a/src/xz/sandbox.h b/src/xz/sandbox.h new file mode 100644 index 0000000..98b9862 --- /dev/null +++ b/src/xz/sandbox.h @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file sandbox.h +/// \brief Sandbox support +// +// Author: Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#if defined(HAVE_PLEDGE) || defined(HAVE_LINUX_LANDLOCK) \ + || defined(HAVE_CAP_RIGHTS_LIMIT) +# define ENABLE_SANDBOX 1 +#endif + + +/// \brief Enables early sandboxing that can always be enabled +/// +/// This requires that tuklib_progname() and io_init() have been called. +extern void sandbox_init(void); + + +/// \brief Enable sandboxing that only allows opening files for reading +extern void sandbox_enable_read_only(void); + + +/// \brief Tell sandboxing code that strict sandboxing can be used +/// +/// This function only sets a flag which will be read by +/// sandbox_enable_strict_if_allowed(). +extern void sandbox_allow_strict(void); + + +/// \brief Enable sandboxing that allows reading from one file +/// +/// This does nothing if sandbox_allow_strict() hasn't been called. +/// +/// \param src_fd File descriptor open for reading +/// \param pipe_event_fd user_abort_pipe[0] from file_io.c +/// \param pipe_write_fd user_abort_pipe[1] from file_io.c +extern void sandbox_enable_strict_if_allowed( + int src_fd, int pipe_event_fd, int pipe_write_fd); diff --git a/src/xz/signals.c b/src/xz/signals.c index 7aef463..20f03be 100644 --- a/src/xz/signals.c +++ b/src/xz/signals.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file signals.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -50,6 +49,10 @@ signal_handler(int sig) } +#ifdef __APPLE__ +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wsign-conversion" +#endif extern void signals_init(void) { @@ -82,6 +85,11 @@ signals_init(void) sigaddset(&hooked_signals, message_progress_sigs[i]); #endif +#ifdef USE_SIGTSTP_HANDLER + // Add the SIGTSTP handler from mytime.c to hooked_signals. + sigaddset(&hooked_signals, SIGTSTP); +#endif + // Using "my_sa" because "sa" may conflict with a sockaddr variable // from system headers on Solaris. struct sigaction my_sa; @@ -96,10 +104,11 @@ signals_init(void) my_sa.sa_flags = 0; my_sa.sa_handler = &signal_handler; + struct sigaction old; + for (size_t i = 0; i < ARRAY_SIZE(sigs); ++i) { // If the parent process has left some signals ignored, // we don't unignore them. - struct sigaction old; if (sigaction(sigs[i], NULL, &old) == 0 && old.sa_handler == SIG_IGN) continue; @@ -109,10 +118,22 @@ signals_init(void) message_signal_handler(); } +#ifdef USE_SIGTSTP_HANDLER + if (!(sigaction(SIGTSTP, NULL, &old) == 0 + && old.sa_handler == SIG_IGN)) { + my_sa.sa_handler = &mytime_sigtstp_handler; + if (sigaction(SIGTSTP, &my_sa, NULL)) + message_signal_handler(); + } +#endif + signals_are_initialized = true; return; } +#ifdef __APPLE__ +# pragma GCC diagnostic pop +#endif #ifndef __VMS diff --git a/src/xz/signals.h b/src/xz/signals.h index 5b125e0..629335d 100644 --- a/src/xz/signals.h +++ b/src/xz/signals.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file signals.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// If this is true, we will clean up the possibly incomplete output file, diff --git a/src/xz/suffix.c b/src/xz/suffix.c index 6ce9787..1d548e4 100644 --- a/src/xz/suffix.c +++ b/src/xz/suffix.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file suffix.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -21,7 +20,13 @@ # ifdef HAVE_STRINGS_H # include <strings.h> # endif -# define strcmp strcasecmp +# ifdef _MSC_VER +# define suffix_strcmp _stricmp +# else +# define suffix_strcmp strcasecmp +# endif +#else +# define suffix_strcmp strcmp #endif @@ -98,7 +103,7 @@ test_suffix(const char *suffix, const char *src_name, size_t src_len) || is_dir_sep(src_name[src_len - suffix_len - 1])) return 0; - if (strcmp(suffix, src_name + src_len - suffix_len) == 0) + if (suffix_strcmp(suffix, src_name + src_len - suffix_len) == 0) return src_len - suffix_len; return 0; @@ -178,7 +183,7 @@ uncompressed_name(const char *src_name, const size_t src_len) static void msg_suffix(const char *src_name, const char *suffix) { - message_warning(_("%s: File already has `%s' suffix, skipping"), + message_warning(_("%s: File already has '%s' suffix, skipping"), src_name, suffix); return; } diff --git a/src/xz/suffix.h b/src/xz/suffix.h index 135e905..f59e312 100644 --- a/src/xz/suffix.h +++ b/src/xz/suffix.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file suffix.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// \brief Get the name of the destination file diff --git a/src/xz/util.c b/src/xz/util.c index 6ab4c2d..0d339ae 100644 --- a/src/xz/util.c +++ b/src/xz/util.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file util.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -148,8 +147,8 @@ str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max) if (multiplier == 0) { message(V_ERROR, _("%s: Invalid multiplier suffix"), value - 1); - message_fatal(_("Valid suffixes are `KiB' (2^10), " - "`MiB' (2^20), and `GiB' (2^30).")); + message_fatal(_("Valid suffixes are 'KiB' (2^10), " + "'MiB' (2^20), and 'GiB' (2^30).")); } // Don't overflow here either. @@ -165,7 +164,7 @@ str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max) return result; error: - message_fatal(_("Value of the option `%s' must be in the range " + message_fatal(_("Value of the option '%s' must be in the range " "[%" PRIu64 ", %" PRIu64 "]"), name, min, max); } @@ -262,9 +261,30 @@ my_snprintf(char **pos, size_t *left, const char *fmt, ...) extern bool +is_tty(int fd) +{ +#if defined(_WIN32) && !defined(__CYGWIN__) + // There is no need to check if handle == INVALID_HANDLE_VALUE + // because it will return false anyway when used in GetConsoleMode(). + // The resulting HANDLE is owned by the file descriptor. + // The HANDLE must not be closed here. + intptr_t handle = _get_osfhandle(fd); + DWORD mode; + + // GetConsoleMode() is an easy way to tell if the HANDLE is a + // console or not. We do not care about the value of mode since we + // do not plan to use any further Windows console functions. + return GetConsoleMode((HANDLE)handle, &mode); +#else + return isatty(fd); +#endif +} + + +extern bool is_tty_stdin(void) { - const bool ret = isatty(STDIN_FILENO); + const bool ret = is_tty(STDIN_FILENO); if (ret) message_error(_("Compressed data cannot be read from " @@ -277,7 +297,7 @@ is_tty_stdin(void) extern bool is_tty_stdout(void) { - const bool ret = isatty(STDOUT_FILENO); + const bool ret = is_tty(STDOUT_FILENO); if (ret) message_error(_("Compressed data cannot be written to " diff --git a/src/xz/util.h b/src/xz/util.h index 6d7e148..a2fdd05 100644 --- a/src/xz/util.h +++ b/src/xz/util.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file util.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// \brief Safe malloc() that never returns NULL @@ -105,6 +104,20 @@ lzma_attribute((__format__(__printf__, 3, 4))) extern void my_snprintf(char **pos, size_t *left, const char *fmt, ...); +/// \brief Test if file descriptor is a terminal +/// +/// For POSIX systems, this is a simple wrapper around isatty(). However on +/// Windows, isatty() returns true for all character devices, not just +/// terminals. +/// +/// \param fd File descriptor to test +/// +/// \return bool: +/// - true if file descriptor is a terminal +/// - false otherwise +extern bool is_tty(int fd); + + /// \brief Test if stdin is a terminal /// /// If stdin is a terminal, an error message is printed and exit status set diff --git a/src/xz/xz.1 b/src/xz/xz.1 index 73ca6ef..5b880e8 100644 --- a/src/xz/xz.1 +++ b/src/xz/xz.1 @@ -1,12 +1,10 @@ '\" t +.\" SPDX-License-Identifier: 0BSD .\" .\" Authors: Lasse Collin .\" Jia Tan .\" -.\" This file has been put into the public domain. -.\" You can do whatever you want with this file. -.\" -.TH XZ 1 "2023-07-17" "Tukaani" "XZ Utils" +.TH XZ 1 "2024-04-08" "Tukaani" "XZ Utils" . .SH NAME xz, unxz, xzcat, lzma, unlzma, lzcat \- Compress or decompress .xz and .lzma files @@ -801,8 +799,6 @@ in the single-threaded mode. It may vary slightly between .B xz versions. -Memory requirements of some of the future multithreaded modes may -be dramatically higher than that of the single-threaded mode. .IP \(bu 3 DecMem contains the decompressor memory requirements. That is, the compression settings determine @@ -811,6 +807,15 @@ The exact decompressor memory usage is slightly more than the LZMA2 dictionary size, but the values in the table have been rounded up to the next full MiB. .RE +.IP "" +Memory requirements of the multi-threaded mode are +significantly higher than that of the single-threaded mode. +With the default value of +.BR \-\-block\-size , +each thread needs 3*3*DictSize plus CompMem or DecMem. +For example, four threads with preset +.B \-6 +needs 660\(en670\ MiB of memory. .TP .BR \-e ", " \-\-extreme Use a slower variant of the selected compression preset level @@ -902,50 +907,90 @@ Using .I size less than the LZMA2 dictionary size is waste of RAM because then the LZMA2 dictionary buffer will never get fully used. -The sizes of the blocks are stored in the block headers, -which a future version of -.B xz -will use for multi-threaded decompression. +In multi-threaded mode, +the sizes of the blocks are stored in the block headers. +This size information is required for multi-threaded decompression. .IP "" In single-threaded mode no block splitting is done by default. Setting this option doesn't affect memory usage. No size information is stored in block headers, thus files created in single-threaded mode won't be identical to files created in multi-threaded mode. -The lack of size information also means that a future version of +The lack of size information also means that .B xz won't be able decompress the files in multi-threaded mode. .TP -.BI \-\-block\-list= sizes +.BI \-\-block\-list= items When compressing to the .B .xz -format, start a new block after +format, start a new block with an optional custom filter chain after the given intervals of uncompressed data. .IP "" -The uncompressed -.I sizes -of the blocks are specified as a comma-separated list. -Omitting a size (two or more consecutive commas) is a shorthand -to use the size of the previous block. +The +.I items +are a comma-separated list. +Each item consists of an optional filter chain number +between 0 and 9 followed by a colon +.RB ( : ) +and a required size of uncompressed data. +Omitting an item (two or more consecutive commas) is a +shorthand to use the size and filters of the previous item. .IP "" If the input file is bigger than the sum of -.IR sizes , -the last value in -.I sizes -is repeated until the end of the file. +the sizes in +.IR items , +the last item is repeated until the end of the file. A special value of .B 0 -may be used as the last value to indicate that +may be used as the last size to indicate that the rest of the file should be encoded as a single block. .IP "" -If one specifies -.I sizes -that exceed the encoder's block size +An alternative filter chain for each block can be +specified in combination with the +.BI \-\-filters1= filters +\&...\& +.BI \-\-filters9= filters +options. +These options define filter chains with an identifier +between 1\(en9. +Filter chain 0 can be used to refer to the default filter chain, +which is the same as not specifying a filter chain. +The filter chain identifier can be used before the uncompressed +size, followed by a colon +.RB ( : ). +For example, if one specifies +.B \-\-block\-list=1:2MiB,3:2MiB,2:4MiB,,2MiB,0:4MiB +then blocks will be created using: +.RS +.IP \(bu 3 +The filter chain specified by +.B \-\-filters1 +and 2 MiB input +.IP \(bu 3 +The filter chain specified by +.B \-\-filters3 +and 2 MiB input +.IP \(bu 3 +The filter chain specified by +.B \-\-filters2 +and 4 MiB input +.IP \(bu 3 +The filter chain specified by +.B \-\-filters2 +and 4 MiB input +.IP \(bu 3 +The default filter chain and 2 MiB input +.IP \(bu 3 +The default filter chain and 4 MiB input for every block until +end of input. +.RE +.IP "" +If one specifies a size that exceeds the encoder's block size (either the default value in threaded mode or the value specified with \fB\-\-block\-size=\fIsize\fR), the encoder will create additional blocks while keeping the boundaries specified in -.IR sizes . +.IR items . For example, if one specifies .B \-\-block\-size=10MiB .B \-\-block\-list=5MiB,10MiB,8MiB,12MiB,24MiB @@ -1262,6 +1307,15 @@ meet this condition, but files compressed in single-threaded mode don't even if .BI \-\-block\-size= size has been used. +.IP "" +The default value for +.I threads +is +.BR 0 . +In +.B xz +5.4.x and older the default is +.BR 1 . . .SS "Custom compressor filter chains" A custom filter chain allows specifying @@ -1295,22 +1349,37 @@ in the chain. Depending on the filter, this limitation is either inherent to the filter design or exists to prevent security issues. .PP -A custom filter chain is specified by using one or more -filter options in the order they are wanted in the filter chain. -That is, the order of filter options is significant! +A custom filter chain can be specified in two different ways. +The options +.BI \-\-filters= filters +and +.BI \-\-filters1= filters +\&...\& +.BI \-\-filters9= filters +allow specifying an entire filter chain in one option using the +liblzma filter string syntax. +Alternatively, a filter chain can be specified by using one or more +individual filter options in the order they are wanted in the filter chain. +That is, the order of the individual filter options is significant! When decoding raw streams .RB ( \-\-format=raw ), -the filter chain is specified in the same order as +the filter chain must be specified in the same order as it was specified when compressing. -.PP -Filters take filter-specific +Any individual filter or preset options specified before the full +chain option +(\fB\-\-filters=\fIfilters\fR) +will be forgotten. +Individual filters specified after the full chain option will reset the +filter chain. +.PP +Both the full and individual filter options take filter-specific .I options as a comma-separated list. Extra commas in .I options are ignored. -Every option has a default value, so you need to -specify only those you want to change. +Every option has a default value, so +specify those you want to change. .PP To see the whole filter chain and .IR options , @@ -1321,6 +1390,45 @@ use twice). This works also for viewing the filter chain options used by presets. .TP +.BI \-\-filters= filters +Specify the full filter chain or a preset in a single option. +Each filter can be separated by spaces or two dashes +.RB ( \-\- ). +.I filters +may need to be quoted on the shell command line so it is +parsed as a single option. +To denote +.IR options , +use +.B : +or +.BR = . +A preset can be prefixed with a +.B \- +and followed with zero or more flags. +The only supported flag is +.B e +to apply the same options as +.BR \-\-extreme . +.TP +\fB\-\-filters1\fR=\fIfilters\fR ... \fB\-\-filters9\fR=\fIfilters +Specify up to nine additional filter chains that can be used with +.BR \-\-block\-list . +.IP "" +For example, when compressing an archive with executable files +followed by text files, the executable part could use a filter +chain with a BCJ filter and the text part only the LZMA2 filter. +.TP +.B \-\-filters-help +Display a help message describing how to specify presets and +custom filter chains in the +.B \-\-filters +and +.BI \-\-filters1= filters +\&...\& +.BI \-\-filters9= filters +options, and exit successfully. +.TP \fB\-\-lzma1\fR[\fB=\fIoptions\fR] .PD 0 .TP @@ -1704,6 +1812,8 @@ and \fB\-\-ia64\fR[\fB=\fIoptions\fR] .TP \fB\-\-sparc\fR[\fB=\fIoptions\fR] +.TP +\fB\-\-riscv\fR[\fB=\fIoptions\fR] .PD Add a branch/call/jump (BCJ) filter to the filter chain. These filters can be used only as a non-last filter @@ -1762,6 +1872,7 @@ ARM64;4;4096-byte alignment is best PowerPC;4;Big endian only IA-64;16;Itanium SPARC;4; +RISC-V;2; .TE .RE .RE @@ -1770,14 +1881,38 @@ Since the BCJ-filtered data is usually compressed with LZMA2, the compression ratio may be improved slightly if the LZMA2 options are set to match the alignment of the selected BCJ filter. -For example, with the IA-64 filter, it's good to set -.B pb=4 -or even +Examples: +.RS +.IP \(bu 3 +IA-64 filter has 16-byte alignment so .B pb=4,lp=4,lc=0 +is good with LZMA2 (2^4=16). -The x86 filter is an exception; -it's usually good to stick to LZMA2's default -four-byte alignment when compressing x86 executables. +.IP \(bu 3 +RISC-V code has 2-byte or 4-byte alignment +depending on whether the file contains +16-bit compressed instructions (the C extension). +When 16-bit instructions are used, +.B pb=2,lp=1,lc=3 +or +.B pb=1,lp=1,lc=3 +is good. +When 16-bit instructions aren't present, +.B pb=2,lp=2,lc=2 +is the best. +.B readelf \-h +can be used to check if "RVC" +appears on the "Flags" line. +.IP \(bu 3 +ARM64 is always 4-byte aligned so +.B pb=2,lp=2,lc=2 +is the best. +.IP \(bu 3 +The x86 filter is an exception. +It's usually good to stick to LZMA2's defaults +.RB ( pb=2,lp=0,lc=3 ) +when compressing x86 executables. +.RE .IP "" All BCJ filters support the same .IR options : @@ -1954,107 +2089,14 @@ easier to parse by other programs. Currently .B \-\-robot is supported only together with -.BR \-\-version , +.BR \-\-list , +.BR \-\-filters\-help , .BR \-\-info\-memory , and -.BR \-\-list . +.BR \-\-version . It will be supported for compression and decompression in the future. . -.SS Version -.B "xz \-\-robot \-\-version" -prints the version number of -.B xz -and liblzma in the following format: -.PP -.BI XZ_VERSION= XYYYZZZS -.br -.BI LIBLZMA_VERSION= XYYYZZZS -.TP -.I X -Major version. -.TP -.I YYY -Minor version. -Even numbers are stable. -Odd numbers are alpha or beta versions. -.TP -.I ZZZ -Patch level for stable releases or -just a counter for development releases. -.TP -.I S -Stability. -0 is alpha, 1 is beta, and 2 is stable. -.I S -should be always 2 when -.I YYY -is even. -.PP -.I XYYYZZZS -are the same on both lines if -.B xz -and liblzma are from the same XZ Utils release. -.PP -Examples: 4.999.9beta is -.B 49990091 -and -5.0.0 is -.BR 50000002 . -. -.SS "Memory limit information" -.B "xz \-\-robot \-\-info\-memory" -prints a single line with multiple tab-separated columns: -.IP 1. 4 -Total amount of physical memory (RAM) in bytes. -.IP 2. 4 -Memory usage limit for compression in bytes -.RB ( \-\-memlimit\-compress ). -A special value of -.B 0 -indicates the default setting -which for single-threaded mode is the same as no limit. -.IP 3. 4 -Memory usage limit for decompression in bytes -.RB ( \-\-memlimit\-decompress ). -A special value of -.B 0 -indicates the default setting -which for single-threaded mode is the same as no limit. -.IP 4. 4 -Since -.B xz -5.3.4alpha: -Memory usage for multi-threaded decompression in bytes -.RB ( \-\-memlimit\-mt\-decompress ). -This is never zero because a system-specific default value -shown in the column 5 -is used if no limit has been specified explicitly. -This is also never greater than the value in the column 3 -even if a larger value has been specified with -.BR \-\-memlimit\-mt\-decompress . -.IP 5. 4 -Since -.B xz -5.3.4alpha: -A system-specific default memory usage limit -that is used to limit the number of threads -when compressing with an automatic number of threads -.RB ( \-\-threads=0 ) -and no memory usage limit has been specified -.RB ( \-\-memlimit\-compress ). -This is also used as the default value for -.BR \-\-memlimit\-mt\-decompress . -.IP 6. 4 -Since -.B xz -5.3.4alpha: -Number of available processor threads. -.PP -In the future, the output of -.B "xz \-\-robot \-\-info\-memory" -may have more columns, but never more than a single line. -. .SS "List mode" .B "xz \-\-robot \-\-list" uses tab-separated output. @@ -2339,6 +2381,127 @@ Future versions may add new line types and new columns can be added to the existing line types, but the existing columns won't be changed. . +.SS "Filters help" +.B "xz \-\-robot \-\-filters-help" +prints the supported filters in the following format: +.PP +\fIfilter\fB:\fIoption\fB=<\fIvalue\fB>,\fIoption\fB=<\fIvalue\fB>\fR... +.TP +.I filter +Name of the filter +.TP +.I option +Name of a filter specific option +.TP +.I value +Numeric +.I value +ranges appear as +\fB<\fImin\fB\-\fImax\fB>\fR. +String +.I value +choices are shown within +.B "< >" +and separated by a +.B | +character. +.PP +Each filter is printed on its own line. +. +.SS "Memory limit information" +.B "xz \-\-robot \-\-info\-memory" +prints a single line with multiple tab-separated columns: +.IP 1. 4 +Total amount of physical memory (RAM) in bytes. +.IP 2. 4 +Memory usage limit for compression in bytes +.RB ( \-\-memlimit\-compress ). +A special value of +.B 0 +indicates the default setting +which for single-threaded mode is the same as no limit. +.IP 3. 4 +Memory usage limit for decompression in bytes +.RB ( \-\-memlimit\-decompress ). +A special value of +.B 0 +indicates the default setting +which for single-threaded mode is the same as no limit. +.IP 4. 4 +Since +.B xz +5.3.4alpha: +Memory usage for multi-threaded decompression in bytes +.RB ( \-\-memlimit\-mt\-decompress ). +This is never zero because a system-specific default value +shown in the column 5 +is used if no limit has been specified explicitly. +This is also never greater than the value in the column 3 +even if a larger value has been specified with +.BR \-\-memlimit\-mt\-decompress . +.IP 5. 4 +Since +.B xz +5.3.4alpha: +A system-specific default memory usage limit +that is used to limit the number of threads +when compressing with an automatic number of threads +.RB ( \-\-threads=0 ) +and no memory usage limit has been specified +.RB ( \-\-memlimit\-compress ). +This is also used as the default value for +.BR \-\-memlimit\-mt\-decompress . +.IP 6. 4 +Since +.B xz +5.3.4alpha: +Number of available processor threads. +.PP +In the future, the output of +.B "xz \-\-robot \-\-info\-memory" +may have more columns, but never more than a single line. +. +.SS Version +.B "xz \-\-robot \-\-version" +prints the version number of +.B xz +and liblzma in the following format: +.PP +.BI XZ_VERSION= XYYYZZZS +.br +.BI LIBLZMA_VERSION= XYYYZZZS +.TP +.I X +Major version. +.TP +.I YYY +Minor version. +Even numbers are stable. +Odd numbers are alpha or beta versions. +.TP +.I ZZZ +Patch level for stable releases or +just a counter for development releases. +.TP +.I S +Stability. +0 is alpha, 1 is beta, and 2 is stable. +.I S +should be always 2 when +.I YYY +is even. +.PP +.I XYYYZZZS +are the same on both lines if +.B xz +and liblzma are from the same XZ Utils release. +.PP +Examples: 4.999.9beta is +.B 49990091 +and +5.0.0 is +.BR 50000002 . +. .SH "EXIT STATUS" .TP .B 0 @@ -2391,7 +2554,7 @@ is run by a script or tool, for example, GNU .RS .PP .nf -.ft CW +.ft CR XZ_OPT=\-2v tar caf foo.tar.xz foo .ft R .fi @@ -2411,7 +2574,7 @@ scripts one may use something like this: .RS .PP .nf -.ft CW +.ft CR XZ_OPT=${XZ_OPT\-"\-7e"} export XZ_OPT .ft R @@ -2669,7 +2832,7 @@ if compression is successful: .RS .PP .nf -.ft CW +.ft CR xz foo .ft R .fi @@ -2685,7 +2848,7 @@ even if decompression is successful: .RS .PP .nf -.ft CW +.ft CR xz \-dk bar.xz .ft R .fi @@ -2703,7 +2866,7 @@ and 5\ MiB, respectively): .RS .PP .nf -.ft CW +.ft CR tar cf \- baz | xz \-4e > baz.tar.xz .ft R .fi @@ -2714,7 +2877,7 @@ to standard output with a single command: .RS .PP .nf -.ft CW +.ft CR xz \-dcf a.txt b.txt.xz c.txt d.txt.lzma > abcd.txt .ft R .fi @@ -2729,7 +2892,7 @@ can be used to parallelize compression of many files: .RS .PP .nf -.ft CW +.ft CR find . \-type f \e! \-name '*.xz' \-print0 \e | xargs \-0r \-P4 \-n16 xz \-T1 .ft R @@ -2769,7 +2932,7 @@ after compressing multiple files: .RS .PP .nf -.ft CW +.ft CR xz \-\-robot \-\-list *.xz | awk '/^totals/{print $5\-$4}' .ft R .fi @@ -2789,7 +2952,7 @@ option: .RS .PP .nf -.ft CW +.ft CR if ! eval "$(xz \-\-robot \-\-version 2> /dev/null)" || [ "$XZ_VERSION" \-lt 50000002 ]; then echo "Your xz is too old." @@ -2805,7 +2968,7 @@ but if a limit has already been set, don't increase it: .RS .PP .nf -.ft CW +.ft CR NEWLIM=$((123 << 20))\ \ # 123 MiB OLDLIM=$(xz \-\-robot \-\-info\-memory | cut \-f3) if [ $OLDLIM \-eq 0 \-o $OLDLIM \-gt $NEWLIM ]; then @@ -2858,7 +3021,7 @@ can be modified to use a bigger dictionary: .RS .PP .nf -.ft CW +.ft CR xz \-\-lzma2=preset=1,dict=32MiB foo.tar .ft R .fi @@ -2886,7 +3049,7 @@ would use: .RS .PP .nf -.ft CW +.ft CR xz \-vv \-\-lzma2=dict=192MiB big_foo.tar .ft R .fi @@ -2916,7 +3079,7 @@ using about 100\ KiB of memory. .RS .PP .nf -.ft CW +.ft CR xz \-\-check=crc32 \-\-lzma2=preset=6e,dict=64KiB foo .ft R .fi @@ -2944,7 +3107,7 @@ slightly (like 0.1\ %) smaller file than .RS .PP .nf -.ft CW +.ft CR xz \-\-lzma2=preset=6e,pb=0,lc=4 source_code.tar .ft R .fi @@ -2957,7 +3120,7 @@ using the x86 BCJ filter: .RS .PP .nf -.ft CW +.ft CR xz \-\-x86 \-\-lzma2 libfoo.so .ft R .fi @@ -2992,7 +3155,7 @@ to LZMA2 to accommodate the three-byte alignment: .RS .PP .nf -.ft CW +.ft CR xz \-\-delta=dist=3 \-\-lzma2=pb=0 foo.tiff .ft R .fi diff --git a/src/xz/xz_w32res.rc b/src/xz/xz_w32res.rc index bad3020..e80903e 100644 --- a/src/xz/xz_w32res.rc +++ b/src/xz/xz_w32res.rc @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #define MY_TYPE VFT_APP diff --git a/src/xzdec/Makefile.am b/src/xzdec/Makefile.am index 90f1e92..31a6508 100644 --- a/src/xzdec/Makefile.am +++ b/src/xzdec/Makefile.am @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## # Windows resource compiler support. It's fine to use xz_CPPFLAGS # also for lzmadec. diff --git a/src/xzdec/Makefile.in b/src/xzdec/Makefile.in index b05cff1..7d44f9b 100644 --- a/src/xzdec/Makefile.in +++ b/src/xzdec/Makefile.in @@ -103,8 +103,8 @@ bin_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2) @COND_LZMADEC_TRUE@@COND_XZDEC_TRUE@am__append_6 = lzmadec subdir = src/xzdec ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \ + $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \ $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -264,7 +264,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CAPSICUM_LIB = @CAPSICUM_LIB@ CC = @CC@ CCAS = @CCAS@ CCASDEPMODE = @CCASDEPMODE@ @@ -393,6 +392,8 @@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ +localedir_c = @localedir_c@ +localedir_c_make = @localedir_c_make@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ diff --git a/src/xzdec/lzmadec_w32res.rc b/src/xzdec/lzmadec_w32res.rc index b85962d..b4b89b1 100644 --- a/src/xzdec/lzmadec_w32res.rc +++ b/src/xzdec/lzmadec_w32res.rc @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #define MY_TYPE VFT_APP diff --git a/src/xzdec/xzdec.1 b/src/xzdec/xzdec.1 index 78bc9b4..5198efb 100644 --- a/src/xzdec/xzdec.1 +++ b/src/xzdec/xzdec.1 @@ -1,10 +1,8 @@ +.\" SPDX-License-Identifier: 0BSD .\" .\" Author: Lasse Collin .\" -.\" This file has been put into the public domain. -.\" You can do whatever you want with this file. -.\" -.TH XZDEC 1 "2017-04-19" "Tukaani" "XZ Utils" +.TH XZDEC 1 "2024-04-08" "Tukaani" "XZ Utils" .SH NAME xzdec, lzmadec \- Small .xz and .lzma decompressors .SH SYNOPSIS diff --git a/src/xzdec/xzdec.c b/src/xzdec/xzdec.c index 556c548..4d88174 100644 --- a/src/xzdec/xzdec.c +++ b/src/xzdec/xzdec.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file xzdec.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "sysdefs.h" @@ -16,7 +15,30 @@ #include <stdarg.h> #include <errno.h> #include <stdio.h> -#include <unistd.h> + +#ifndef _MSC_VER +# include <unistd.h> +#endif + +#ifdef HAVE_CAP_RIGHTS_LIMIT +# include <sys/capsicum.h> +#endif + +#ifdef HAVE_LINUX_LANDLOCK +# include <linux/landlock.h> +# include <sys/prctl.h> +# include <sys/syscall.h> +# ifdef LANDLOCK_ACCESS_NET_BIND_TCP +# define LANDLOCK_ABI_MAX 4 +# else +# define LANDLOCK_ABI_MAX 3 +# endif +#endif + +#if defined(HAVE_CAP_RIGHTS_LIMIT) || defined(HAVE_PLEDGE) \ + || defined(HAVE_LINUX_LANDLOCK) +# define ENABLE_SANDBOX 1 +#endif #include "getopt.h" #include "tuklib_progname.h" @@ -25,6 +47,10 @@ #ifdef TUKLIB_DOSLIKE # include <fcntl.h> # include <io.h> +# ifdef _MSC_VER +# define fileno _fileno +# define setmode _setmode +# endif #endif @@ -273,9 +299,123 @@ uncompress(lzma_stream *strm, FILE *file, const char *filename) } +#ifdef ENABLE_SANDBOX +static void +sandbox_enter(int src_fd) +{ +#if defined(HAVE_CAP_RIGHTS_LIMIT) + // Capsicum needs FreeBSD 10.2 or later. + cap_rights_t rights; + + if (cap_enter()) + goto error; + + if (cap_rights_limit(src_fd, cap_rights_init(&rights, CAP_READ))) + goto error; + + // If not reading from stdin, remove all capabilities from it. + if (src_fd != STDIN_FILENO && cap_rights_limit( + STDIN_FILENO, cap_rights_clear(&rights))) + goto error; + + if (cap_rights_limit(STDOUT_FILENO, cap_rights_init(&rights, + CAP_WRITE))) + goto error; + + if (cap_rights_limit(STDERR_FILENO, cap_rights_init(&rights, + CAP_WRITE))) + goto error; + +#elif defined(HAVE_PLEDGE) + // pledge() was introduced in OpenBSD 5.9. + if (pledge("stdio", "")) + goto error; + + (void)src_fd; + +#elif defined(HAVE_LINUX_LANDLOCK) + int landlock_abi = syscall(SYS_landlock_create_ruleset, + (void *)NULL, 0, LANDLOCK_CREATE_RULESET_VERSION); + + if (landlock_abi > 0) { + if (landlock_abi > LANDLOCK_ABI_MAX) + landlock_abi = LANDLOCK_ABI_MAX; + + const struct landlock_ruleset_attr attr = { + .handled_access_fs = (1ULL + << (12 + my_min(3, landlock_abi))) - 1, +# if LANDLOCK_ABI_MAX >= 4 + .handled_access_net = landlock_abi < 4 ? 0 : + (LANDLOCK_ACCESS_NET_BIND_TCP + | LANDLOCK_ACCESS_NET_CONNECT_TCP), +# endif + }; + + const int ruleset_fd = syscall(SYS_landlock_create_ruleset, + &attr, sizeof(attr), 0U); + if (ruleset_fd < 0) + goto error; + + // All files we need should have already been opened. Thus, + // we don't need to add any rules using landlock_add_rule(2) + // before activating the sandbox. + if (syscall(SYS_landlock_restrict_self, ruleset_fd, 0U) != 0) + goto error; + } + + (void)src_fd; + +#else +# error ENABLE_SANDBOX is defined but no sandboxing method was found. +#endif + + return; + +error: +#ifdef HAVE_CAP_RIGHTS_LIMIT + // If a kernel is configured without capability mode support or + // used in an emulator that does not implement the capability + // system calls, then the Capsicum system calls will fail and set + // errno to ENOSYS. In that case xzdec will silently run without + // the sandbox. + if (errno == ENOSYS) + return; +#endif + + my_errorf("Failed to enable the sandbox"); + exit(EXIT_FAILURE); +} +#endif + + int main(int argc, char **argv) { +#ifdef HAVE_PLEDGE + // OpenBSD's pledge(2) sandbox. + // Initially enable the sandbox slightly more relaxed so that + // the process can still open files. This allows the sandbox to + // be enabled when parsing command line arguments and decompressing + // all files (the more strict sandbox only restricts the last file + // that is decompressed). + if (pledge("stdio rpath", "")) { + my_errorf("Failed to enable the sandbox"); + exit(EXIT_FAILURE); + } +#endif + +#ifdef HAVE_LINUX_LANDLOCK + // Prevent the process from gaining new privileges. This must be done + // before landlock_restrict_self(2) but since we will never need new + // privileges, this call can be done here already. + // + // This is supported since Linux 3.5. Ignore the return value to + // keep compatibility with old kernels. landlock_restrict_self(2) + // will fail if the no_new_privs attribute isn't set, thus if prctl() + // fails here the error will still be detected when it matters. + (void)prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); +#endif + // Initialize progname which we will be used in error messages. tuklib_progname_init(argv); @@ -295,24 +435,42 @@ main(int argc, char **argv) if (optind == argc) { // No filenames given, decode from stdin. +#ifdef ENABLE_SANDBOX + sandbox_enter(STDIN_FILENO); +#endif uncompress(&strm, stdin, "(stdin)"); } else { // Loop through the filenames given on the command line. do { + FILE *src_file; + const char *src_name; + // "-" indicates stdin. if (strcmp(argv[optind], "-") == 0) { - uncompress(&strm, stdin, "(stdin)"); + src_file = stdin; + src_name = "(stdin)"; } else { - FILE *file = fopen(argv[optind], "rb"); - if (file == NULL) { - my_errorf("%s: %s", argv[optind], + src_name = argv[optind]; + src_file = fopen(src_name, "rb"); + if (src_file == NULL) { + my_errorf("%s: %s", src_name, strerror(errno)); exit(EXIT_FAILURE); } - - uncompress(&strm, file, argv[optind]); - fclose(file); } +#ifdef ENABLE_SANDBOX + // Enable the strict sandbox for the last file. + // Then the process can no longer open additional + // files. The typical xzdec use case is to decompress + // a single file so this way the strictest sandboxing + // is used in most cases. + if (optind == argc - 1) + sandbox_enter(fileno(src_file)); +#endif + uncompress(&strm, src_file, src_name); + + if (src_file != stdin) + (void)fclose(src_file); } while (++optind < argc); } diff --git a/src/xzdec/xzdec_w32res.rc b/src/xzdec/xzdec_w32res.rc index 0e26a22..fdbb102 100644 --- a/src/xzdec/xzdec_w32res.rc +++ b/src/xzdec/xzdec_w32res.rc @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #define MY_TYPE VFT_APP |