diff options
Diffstat (limited to '')
90 files changed, 25382 insertions, 0 deletions
diff --git a/src/zstd/tests/.gitignore b/src/zstd/tests/.gitignore new file mode 100644 index 000000000..9a6939a57 --- /dev/null +++ b/src/zstd/tests/.gitignore @@ -0,0 +1,68 @@ +# local binary (Makefile) +fullbench +fullbench32 +fullbench-lib +fuzzer +fuzzer32 +fuzzer-dll +zbufftest +zbufftest32 +zbufftest-dll +zstreamtest +zstreamtest32 +zstreamtest_asan +zstreamtest_tsan +zstreamtest-dll +datagen +paramgrill +paramgrill32 +roundTripCrash +longmatch +symbols +legacy +decodecorpus +pool +poolTests +invalidDictionaries +checkTag +zcat +zstdcat +tm + +# test artifacts +dictionary +grillResults.txt +_* +tmp* +*.zst +*.gz +!gzip/hufts-segv.gz +result +out +*.zstd +hello* +world + +# Tmp test directory +zstdtest +speedTest +versionsTest +namespaceTest +dirTest* + +# fuzzer +afl + +# Local script +startSpeedTest +speedTest.pid +*.bat + +# Generic Object files +*.o +*.ko + +# Generic Executables +*.exe +*.out +*.app diff --git a/src/zstd/tests/DEPRECATED-test-zstd-speed.py b/src/zstd/tests/DEPRECATED-test-zstd-speed.py new file mode 100755 index 000000000..b3f807459 --- /dev/null +++ b/src/zstd/tests/DEPRECATED-test-zstd-speed.py @@ -0,0 +1,378 @@ +#! /usr/bin/env python3 +# THIS BENCHMARK IS BEING REPLACED BY automated-bencmarking.py + +# ################################################################ +# Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ########################################################################## + +# Limitations: +# - doesn't support filenames with spaces +# - dir1/zstd and dir2/zstd will be merged in a single results file + +import argparse +import os # getloadavg +import string +import subprocess +import time # strftime +import traceback +import hashlib +import platform # system + +script_version = 'v1.1.2 (2017-03-26)' +default_repo_url = 'https://github.com/facebook/zstd.git' +working_dir_name = 'speedTest' +working_path = os.getcwd() + '/' + working_dir_name # /path/to/zstd/tests/speedTest +clone_path = working_path + '/' + 'zstd' # /path/to/zstd/tests/speedTest/zstd +email_header = 'ZSTD_speedTest' +pid = str(os.getpid()) +verbose = False +clang_version = "unknown" +gcc_version = "unknown" +args = None + + +def hashfile(hasher, fname, blocksize=65536): + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(blocksize), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def log(text): + print(time.strftime("%Y/%m/%d %H:%M:%S") + ' - ' + text) + + +def execute(command, print_command=True, print_output=False, print_error=True, param_shell=True): + if print_command: + log("> " + command) + popen = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=param_shell, cwd=execute.cwd) + stdout_lines, stderr_lines = popen.communicate(timeout=args.timeout) + stderr_lines = stderr_lines.decode("utf-8") + stdout_lines = stdout_lines.decode("utf-8") + if print_output: + if stdout_lines: + print(stdout_lines) + if stderr_lines: + print(stderr_lines) + if popen.returncode is not None and popen.returncode != 0: + if stderr_lines and not print_output and print_error: + print(stderr_lines) + raise RuntimeError(stdout_lines + stderr_lines) + return (stdout_lines + stderr_lines).splitlines() +execute.cwd = None + + +def does_command_exist(command): + try: + execute(command, verbose, False, False) + except Exception: + return False + return True + + +def send_email(emails, topic, text, have_mutt, have_mail): + logFileName = working_path + '/' + 'tmpEmailContent' + with open(logFileName, "w") as myfile: + myfile.writelines(text) + myfile.close() + if have_mutt: + execute('mutt -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose) + elif have_mail: + execute('mail -s "' + topic + '" ' + emails + ' < ' + logFileName, verbose) + else: + log("e-mail cannot be sent (mail or mutt not found)") + + +def send_email_with_attachments(branch, commit, last_commit, args, text, results_files, + logFileName, have_mutt, have_mail): + with open(logFileName, "w") as myfile: + myfile.writelines(text) + myfile.close() + email_topic = '[%s:%s] Warning for %s:%s last_commit=%s speed<%s ratio<%s' \ + % (email_header, pid, branch, commit, last_commit, + args.lowerLimit, args.ratioLimit) + if have_mutt: + execute('mutt -s "' + email_topic + '" ' + args.emails + ' -a ' + results_files + + ' < ' + logFileName) + elif have_mail: + execute('mail -s "' + email_topic + '" ' + args.emails + ' < ' + logFileName) + else: + log("e-mail cannot be sent (mail or mutt not found)") + + +def git_get_branches(): + execute('git fetch -p', verbose) + branches = execute('git branch -rl', verbose) + output = [] + for line in branches: + if ("HEAD" not in line) and ("coverity_scan" not in line) and ("gh-pages" not in line): + output.append(line.strip()) + return output + + +def git_get_changes(branch, commit, last_commit): + fmt = '--format="%h: (%an) %s, %ar"' + if last_commit is None: + commits = execute('git log -n 10 %s %s' % (fmt, commit)) + else: + commits = execute('git --no-pager log %s %s..%s' % (fmt, last_commit, commit)) + return str('Changes in %s since %s:\n' % (branch, last_commit)) + '\n'.join(commits) + + +def get_last_results(resultsFileName): + if not os.path.isfile(resultsFileName): + return None, None, None, None + commit = None + csize = [] + cspeed = [] + dspeed = [] + with open(resultsFileName, 'r') as f: + for line in f: + words = line.split() + if len(words) <= 4: # branch + commit + compilerVer + md5 + commit = words[1] + csize = [] + cspeed = [] + dspeed = [] + if (len(words) == 8) or (len(words) == 9): # results: "filename" or "XX files" + csize.append(int(words[1])) + cspeed.append(float(words[3])) + dspeed.append(float(words[5])) + return commit, csize, cspeed, dspeed + + +def benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, + testFilePath, fileName, last_csize, last_cspeed, last_dspeed): + sleepTime = 30 + while os.getloadavg()[0] > args.maxLoadAvg: + log("WARNING: bench loadavg=%.2f is higher than %s, sleeping for %s seconds" + % (os.getloadavg()[0], args.maxLoadAvg, sleepTime)) + time.sleep(sleepTime) + start_load = str(os.getloadavg()) + osType = platform.system() + if osType == 'Linux': + cpuSelector = "taskset --cpu-list 0" + else: + cpuSelector = "" + if args.dictionary: + result = execute('%s programs/%s -rqi5b1e%s -D %s %s' % (cpuSelector, executableName, args.lastCLevel, args.dictionary, testFilePath), print_output=True) + else: + result = execute('%s programs/%s -rqi5b1e%s %s' % (cpuSelector, executableName, args.lastCLevel, testFilePath), print_output=True) + end_load = str(os.getloadavg()) + linesExpected = args.lastCLevel + 1 + if len(result) != linesExpected: + raise RuntimeError("ERROR: number of result lines=%d is different that expected %d\n%s" % (len(result), linesExpected, '\n'.join(result))) + with open(resultsFileName, "a") as myfile: + myfile.write('%s %s %s md5=%s\n' % (branch, commit, compilerVersion, md5sum)) + myfile.write('\n'.join(result) + '\n') + myfile.close() + if (last_cspeed == None): + log("WARNING: No data for comparison for branch=%s file=%s " % (branch, fileName)) + return "" + commit, csize, cspeed, dspeed = get_last_results(resultsFileName) + text = "" + for i in range(0, min(len(cspeed), len(last_cspeed))): + print("%s:%s -%d cSpeed=%6.2f cLast=%6.2f cDiff=%1.4f dSpeed=%6.2f dLast=%6.2f dDiff=%1.4f ratioDiff=%1.4f %s" % (branch, commit, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], float(last_csize[i])/csize[i], fileName)) + if (cspeed[i]/last_cspeed[i] < args.lowerLimit): + text += "WARNING: %s -%d cSpeed=%.2f cLast=%.2f cDiff=%.4f %s\n" % (executableName, i+1, cspeed[i], last_cspeed[i], cspeed[i]/last_cspeed[i], fileName) + if (dspeed[i]/last_dspeed[i] < args.lowerLimit): + text += "WARNING: %s -%d dSpeed=%.2f dLast=%.2f dDiff=%.4f %s\n" % (executableName, i+1, dspeed[i], last_dspeed[i], dspeed[i]/last_dspeed[i], fileName) + if (float(last_csize[i])/csize[i] < args.ratioLimit): + text += "WARNING: %s -%d cSize=%d last_cSize=%d diff=%.4f %s\n" % (executableName, i+1, csize[i], last_csize[i], float(last_csize[i])/csize[i], fileName) + if text: + text = args.message + ("\nmaxLoadAvg=%s load average at start=%s end=%s\n%s last_commit=%s md5=%s\n" % (args.maxLoadAvg, start_load, end_load, compilerVersion, last_commit, md5sum)) + text + return text + + +def update_config_file(branch, commit): + last_commit = None + commitFileName = working_path + "/commit_" + branch.replace("/", "_") + ".txt" + if os.path.isfile(commitFileName): + with open(commitFileName, 'r') as infile: + last_commit = infile.read() + with open(commitFileName, 'w') as outfile: + outfile.write(commit) + return last_commit + + +def double_check(branch, commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName): + last_commit, csize, cspeed, dspeed = get_last_results(resultsFileName) + if not args.dry_run: + text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed) + if text: + log("WARNING: redoing tests for branch %s: commit %s" % (branch, commit)) + text = benchmark_and_compare(branch, commit, last_commit, args, executableName, md5sum, compilerVersion, resultsFileName, filePath, fileName, csize, cspeed, dspeed) + return text + + +def test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail): + local_branch = branch.split('/')[1] + version = local_branch.rpartition('-')[2] + '_' + commit + if not args.dry_run: + execute('make -C programs clean zstd CC=clang MOREFLAGS="-Werror -Wconversion -Wno-sign-conversion -DZSTD_GIT_COMMIT=%s" && ' % version + + 'mv programs/zstd programs/zstd_clang && ' + + 'make -C programs clean zstd zstd32 MOREFLAGS="-DZSTD_GIT_COMMIT=%s"' % version) + md5_zstd = hashfile(hashlib.md5(), clone_path + '/programs/zstd') + md5_zstd32 = hashfile(hashlib.md5(), clone_path + '/programs/zstd32') + md5_zstd_clang = hashfile(hashlib.md5(), clone_path + '/programs/zstd_clang') + print("md5(zstd)=%s\nmd5(zstd32)=%s\nmd5(zstd_clang)=%s" % (md5_zstd, md5_zstd32, md5_zstd_clang)) + print("gcc_version=%s clang_version=%s" % (gcc_version, clang_version)) + + logFileName = working_path + "/log_" + branch.replace("/", "_") + ".txt" + text_to_send = [] + results_files = "" + if args.dictionary: + dictName = args.dictionary.rpartition('/')[2] + else: + dictName = None + + for filePath in testFilePaths: + fileName = filePath.rpartition('/')[2] + if dictName: + resultsFileName = working_path + "/" + dictName.replace(".", "_") + "_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" + else: + resultsFileName = working_path + "/results_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" + text = double_check(branch, commit, args, 'zstd', md5_zstd, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName) + if text: + text_to_send.append(text) + results_files += resultsFileName + " " + resultsFileName = working_path + "/results32_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" + text = double_check(branch, commit, args, 'zstd32', md5_zstd32, 'gcc_version='+gcc_version, resultsFileName, filePath, fileName) + if text: + text_to_send.append(text) + results_files += resultsFileName + " " + resultsFileName = working_path + "/resultsClang_" + branch.replace("/", "_") + "_" + fileName.replace(".", "_") + ".txt" + text = double_check(branch, commit, args, 'zstd_clang', md5_zstd_clang, 'clang_version='+clang_version, resultsFileName, filePath, fileName) + if text: + text_to_send.append(text) + results_files += resultsFileName + " " + if text_to_send: + send_email_with_attachments(branch, commit, last_commit, args, text_to_send, results_files, logFileName, have_mutt, have_mail) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('testFileNames', help='file or directory names list for speed benchmark') + parser.add_argument('emails', help='list of e-mail addresses to send warnings') + parser.add_argument('--dictionary', '-D', help='path to the dictionary') + parser.add_argument('--message', '-m', help='attach an additional message to e-mail', default="") + parser.add_argument('--repoURL', help='changes default repository URL', default=default_repo_url) + parser.add_argument('--lowerLimit', '-l', type=float, help='send email if speed is lower than given limit', default=0.98) + parser.add_argument('--ratioLimit', '-r', type=float, help='send email if ratio is lower than given limit', default=0.999) + parser.add_argument('--maxLoadAvg', type=float, help='maximum load average to start testing', default=0.75) + parser.add_argument('--lastCLevel', type=int, help='last compression level for testing', default=5) + parser.add_argument('--sleepTime', '-s', type=int, help='frequency of repository checking in seconds', default=300) + parser.add_argument('--timeout', '-t', type=int, help='timeout for executing shell commands', default=1800) + parser.add_argument('--dry-run', dest='dry_run', action='store_true', help='not build', default=False) + parser.add_argument('--verbose', '-v', action='store_true', help='more verbose logs', default=False) + args = parser.parse_args() + verbose = args.verbose + + # check if test files are accessible + testFileNames = args.testFileNames.split() + testFilePaths = [] + for fileName in testFileNames: + fileName = os.path.expanduser(fileName) + if os.path.isfile(fileName) or os.path.isdir(fileName): + testFilePaths.append(os.path.abspath(fileName)) + else: + log("ERROR: File/directory not found: " + fileName) + exit(1) + + # check if dictionary is accessible + if args.dictionary: + args.dictionary = os.path.abspath(os.path.expanduser(args.dictionary)) + if not os.path.isfile(args.dictionary): + log("ERROR: Dictionary not found: " + args.dictionary) + exit(1) + + # check availability of e-mail senders + have_mutt = does_command_exist("mutt -h") + have_mail = does_command_exist("mail -V") + if not have_mutt and not have_mail: + log("ERROR: e-mail senders 'mail' or 'mutt' not found") + exit(1) + + clang_version = execute("clang -v 2>&1 | grep ' version ' | sed -e 's:.*version \\([0-9.]*\\).*:\\1:' -e 's:\\.\\([0-9][0-9]\\):\\1:g'", verbose)[0]; + gcc_version = execute("gcc -dumpversion", verbose)[0]; + + if verbose: + print("PARAMETERS:\nrepoURL=%s" % args.repoURL) + print("working_path=%s" % working_path) + print("clone_path=%s" % clone_path) + print("testFilePath(%s)=%s" % (len(testFilePaths), testFilePaths)) + print("message=%s" % args.message) + print("emails=%s" % args.emails) + print("dictionary=%s" % args.dictionary) + print("maxLoadAvg=%s" % args.maxLoadAvg) + print("lowerLimit=%s" % args.lowerLimit) + print("ratioLimit=%s" % args.ratioLimit) + print("lastCLevel=%s" % args.lastCLevel) + print("sleepTime=%s" % args.sleepTime) + print("timeout=%s" % args.timeout) + print("dry_run=%s" % args.dry_run) + print("verbose=%s" % args.verbose) + print("have_mutt=%s have_mail=%s" % (have_mutt, have_mail)) + + # clone ZSTD repo if needed + if not os.path.isdir(working_path): + os.mkdir(working_path) + if not os.path.isdir(clone_path): + execute.cwd = working_path + execute('git clone ' + args.repoURL) + if not os.path.isdir(clone_path): + log("ERROR: ZSTD clone not found: " + clone_path) + exit(1) + execute.cwd = clone_path + + # check if speedTest.pid already exists + pidfile = "./speedTest.pid" + if os.path.isfile(pidfile): + log("ERROR: %s already exists, exiting" % pidfile) + exit(1) + + send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been started' % (email_header, pid, script_version), args.message, have_mutt, have_mail) + with open(pidfile, 'w') as the_file: + the_file.write(pid) + + branch = "" + commit = "" + first_time = True + while True: + try: + if first_time: + first_time = False + else: + time.sleep(args.sleepTime) + loadavg = os.getloadavg()[0] + if (loadavg <= args.maxLoadAvg): + branches = git_get_branches() + for branch in branches: + commit = execute('git show -s --format=%h ' + branch, verbose)[0] + last_commit = update_config_file(branch, commit) + if commit == last_commit: + log("skipping branch %s: head %s already processed" % (branch, commit)) + else: + log("build branch %s: head %s is different from prev %s" % (branch, commit, last_commit)) + execute('git checkout -- . && git checkout ' + branch) + print(git_get_changes(branch, commit, last_commit)) + test_commit(branch, commit, last_commit, args, testFilePaths, have_mutt, have_mail) + else: + log("WARNING: main loadavg=%.2f is higher than %s" % (loadavg, args.maxLoadAvg)) + if verbose: + log("sleep for %s seconds" % args.sleepTime) + except Exception as e: + stack = traceback.format_exc() + email_topic = '[%s:%s] ERROR in %s:%s' % (email_header, pid, branch, commit) + send_email(args.emails, email_topic, stack, have_mutt, have_mail) + print(stack) + except KeyboardInterrupt: + os.unlink(pidfile) + send_email(args.emails, '[%s:%s] test-zstd-speed.py %s has been stopped' % (email_header, pid, script_version), args.message, have_mutt, have_mail) + exit(0) diff --git a/src/zstd/tests/Makefile b/src/zstd/tests/Makefile new file mode 100644 index 000000000..d347a948a --- /dev/null +++ b/src/zstd/tests/Makefile @@ -0,0 +1,475 @@ +# ################################################################ +# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ +# datagen : Synthetic and parametrable data generator, for tests +# fullbench : Precisely measure speed for each zstd inner functions +# fullbench32: Same as fullbench, but forced to compile in 32-bits mode +# fuzzer : Test tool, to check zstd integrity on target platform +# fuzzer32: Same as fuzzer, but forced to compile in 32-bits mode +# paramgrill : parameter tester for zstd +# test-zstd-speed.py : script for testing zstd speed difference between commits +# versionsTest : compatibility test between zstd versions stored on Github (v0.1+) +# zstreamtest : Fuzzer test tool for zstd streaming API +# zstreamtest32: Same as zstreamtest, but forced to compile in 32-bits mode +# ########################################################################## + +ZSTDDIR = ../lib +PRGDIR = ../programs +PYTHON ?= python3 +TESTARTEFACT := versionsTest + +DEBUGLEVEL ?= 1 +DEBUGFLAGS = -g -DDEBUGLEVEL=$(DEBUGLEVEL) +CPPFLAGS += -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ + -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(PRGDIR) +ifeq ($(OS),Windows_NT) # MinGW assumed +CPPFLAGS += -D__USE_MINGW_ANSI_STDIO # compatibility with %zu formatting +endif +CFLAGS ?= -O3 +CFLAGS += -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls -Wmissing-prototypes +CFLAGS += $(DEBUGFLAGS) $(MOREFLAGS) +FLAGS = $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) + + +ZSTDCOMMON_FILES := $(ZSTDDIR)/common/*.c +ZSTDCOMP_FILES := $(ZSTDDIR)/compress/*.c +ZSTDDECOMP_FILES := $(ZSTDDIR)/decompress/*.c +ZSTD_FILES := $(ZSTDDECOMP_FILES) $(ZSTDCOMMON_FILES) $(ZSTDCOMP_FILES) +ZBUFF_FILES := $(ZSTDDIR)/deprecated/*.c +ZDICT_FILES := $(ZSTDDIR)/dictBuilder/*.c + +ZSTD_F1 := $(wildcard $(ZSTD_FILES)) +ZSTD_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdm_,$(ZSTD_F1)) +ZSTD_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdc_,$(ZSTD_OBJ1)) +ZSTD_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdd_,$(ZSTD_OBJ2)) +ZSTD_OBJECTS := $(ZSTD_OBJ3:.c=.o) + +ZSTDMT_OBJ1 := $(subst $(ZSTDDIR)/common/,zstdmt_m_,$(ZSTD_F1)) +ZSTDMT_OBJ2 := $(subst $(ZSTDDIR)/compress/,zstdmt_c_,$(ZSTDMT_OBJ1)) +ZSTDMT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,zstdmt_d_,$(ZSTDMT_OBJ2)) +ZSTDMT_OBJECTS := $(ZSTDMT_OBJ3:.c=.o) + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +MULTITHREAD_CPP = -DZSTD_MULTITHREAD +MULTITHREAD_LD = +else +EXT = +MULTITHREAD_CPP = -DZSTD_MULTITHREAD +MULTITHREAD_LD = -pthread +endif +MULTITHREAD = $(MULTITHREAD_CPP) $(MULTITHREAD_LD) + +VOID = /dev/null +ZSTREAM_TESTTIME ?= -T90s +FUZZERTEST ?= -T200s +ZSTDRTTEST = --test-large-data +DECODECORPUS_TESTTIME ?= -T30 + +.PHONY: default all all32 allnothread dll clean test test32 test-all versionsTest + +default: fullbench + @echo $(ZSTDMT_OBJECTS) + +all: fullbench fuzzer zstreamtest paramgrill datagen decodecorpus roundTripCrash \ + fullbench-lib poolTests + +all32: fullbench32 fuzzer32 zstreamtest32 + +allnothread: MULTITHREAD_CPP= +allnothread: MULTITHREAD_LD= +allnothread: fullbench fuzzer paramgrill datagen decodecorpus + +# note : broken : requires symbols unavailable from dynamic library +dll: fuzzer-dll zstreamtest-dll + +PHONY: zstd zstd32 zstd-nolegacy # must be phony, only external makefile knows how to build them, or if they need an update +zstd zstd32 zstd-nolegacy: + $(MAKE) -C $(PRGDIR) $@ MOREFLAGS+="$(DEBUGFLAGS)" + +gzstd: + $(MAKE) -C $(PRGDIR) $@ HAVE_ZLIB=1 MOREFLAGS+="$(DEBUGFLAGS)" + +.PHONY: libzstd +libzstd : + $(MAKE) -C $(ZSTDDIR) libzstd + +%-dll : libzstd +%-dll : LDFLAGS+= -L$(ZSTDDIR) -lzstd + +.PHONY: zstd-staticLib +zstd-staticLib : + $(MAKE) -C $(ZSTDDIR) libzstd.a + +zstdm_%.o : $(ZSTDDIR)/common/%.c + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + +zstdc_%.o : $(ZSTDDIR)/compress/%.c + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + +zstdd_%.o : $(ZSTDDIR)/decompress/%.c + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + +zstdmt%.o : CPPFLAGS += $(MULTITHREAD_CPP) + +zstdmt_m_%.o : $(ZSTDDIR)/common/%.c + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + +zstdmt_c_%.o : $(ZSTDDIR)/compress/%.c + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + +zstdmt_d_%.o : $(ZSTDDIR)/decompress/%.c + $(CC) -c $(CPPFLAGS) $(CFLAGS) $< -o $@ + +fullbench32: CPPFLAGS += -m32 +fullbench fullbench32 : CPPFLAGS += $(MULTITHREAD_CPP) +fullbench fullbench32 : LDFLAGS += $(MULTITHREAD_LD) +fullbench fullbench32 : DEBUGFLAGS = -DNDEBUG # turn off assert() for speed measurements +fullbench fullbench32 : $(ZSTD_FILES) +fullbench fullbench32 : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +fullbench-lib : CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ +fullbench-lib : zstd-staticLib +fullbench-lib : $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c fullbench.c + $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) $(ZSTDDIR)/libzstd.a + +# note : broken : requires symbols unavailable from dynamic library +fullbench-dll: $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/benchfn.c $(PRGDIR)/timefn.c fullbench.c +# $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) -DZSTD_DLL_IMPORT=1 $(ZSTDDIR)/dll/libzstd.dll + $(CC) $(FLAGS) $(filter %.c,$^) -o $@$(EXT) + +fuzzer : CPPFLAGS += $(MULTITHREAD_CPP) +fuzzer : LDFLAGS += $(MULTITHREAD_LD) +fuzzer32: CFLAGS += -m32 +fuzzer : $(ZSTDMT_OBJECTS) +fuzzer32: $(ZSTD_FILES) +fuzzer fuzzer32 : $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +# note : broken : requires symbols unavailable from dynamic library +fuzzer-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c fuzzer.c + $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) + +zbufftest zbufftest32 zbufftest-dll : CPPFLAGS += -I$(ZSTDDIR)/deprecated +zbufftest zbufftest32 zbufftest-dll : CFLAGS += -Wno-deprecated-declarations # required to silence deprecation warnings +zbufftest32 : CFLAGS += -m32 +zbufftest zbufftest32 : $(ZSTD_OBJECTS) $(ZBUFF_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +zbufftest-dll : $(ZSTDDIR)/common/xxhash.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/datagen.c zbufftest.c + $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) + +ZSTREAM_LOCAL_FILES := $(PRGDIR)/datagen.c $(PRGDIR)/util.c $(PRGDIR)/timefn.c seqgen.c zstreamtest.c +ZSTREAM_PROPER_FILES := $(ZDICT_FILES) $(ZSTREAM_LOCAL_FILES) +ZSTREAMFILES := $(ZSTD_FILES) $(ZSTREAM_PROPER_FILES) +zstreamtest32 : CFLAGS += -m32 +zstreamtest zstreamtest32 : CPPFLAGS += $(MULTITHREAD_CPP) +zstreamtest zstreamtest32 : LDFLAGS += $(MULTITHREAD_LD) +zstreamtest : $(ZSTDMT_OBJECTS) $(ZSTREAM_PROPER_FILES) +zstreamtest32 : $(ZSTREAMFILES) +zstreamtest zstreamtest32 : + $(CC) $(FLAGS) $^ -o $@$(EXT) + +zstreamtest_asan : CFLAGS += -fsanitize=address +zstreamtest_asan : $(ZSTREAMFILES) + $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + +zstreamtest_tsan : CFLAGS += -fsanitize=thread +zstreamtest_tsan : $(ZSTREAMFILES) + $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + +# note : broken : requires symbols unavailable from dynamic library +zstreamtest-dll : $(ZSTDDIR)/common/xxhash.c # xxh symbols not exposed from dll +zstreamtest-dll : $(ZSTREAM_LOCAL_FILES) + $(CC) $(CPPFLAGS) $(CFLAGS) $(filter %.c,$^) $(LDFLAGS) -o $@$(EXT) + +paramgrill : DEBUGFLAGS = # turn off assert() by default for speed measurements +paramgrill : $(ZSTD_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c $(PRGDIR)/benchfn.c $(PRGDIR)/benchzstd.c $(PRGDIR)/datagen.c paramgrill.c + $(CC) $(FLAGS) $^ -lm -o $@$(EXT) + +datagen : $(PRGDIR)/datagen.c datagencli.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +roundTripCrash : $(ZSTD_OBJECTS) roundTripCrash.c + $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + +longmatch : $(ZSTD_OBJECTS) longmatch.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +bigdict: $(ZSTDMT_OBJECTS) $(PRGDIR)/datagen.c bigdict.c + $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + +invalidDictionaries : $(ZSTD_OBJECTS) invalidDictionaries.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +legacy : CPPFLAGS += -I$(ZSTDDIR)/legacy -DZSTD_LEGACY_SUPPORT=4 +legacy : $(ZSTD_FILES) $(wildcard $(ZSTDDIR)/legacy/*.c) legacy.c + $(CC) $(FLAGS) $^ -o $@$(EXT) + +decodecorpus : $(filter-out zstdc_zstd_compress.o, $(ZSTD_OBJECTS)) $(ZDICT_FILES) $(PRGDIR)/util.c $(PRGDIR)/timefn.c decodecorpus.c + $(CC) $(FLAGS) $^ -o $@$(EXT) -lm + +poolTests : $(PRGDIR)/util.c $(PRGDIR)/timefn.c poolTests.c $(ZSTDDIR)/common/pool.c $(ZSTDDIR)/common/threading.c $(ZSTDDIR)/common/zstd_common.c $(ZSTDDIR)/common/error_private.c + $(CC) $(FLAGS) $(MULTITHREAD) $^ -o $@$(EXT) + +.PHONY: versionsTest +versionsTest: clean + $(PYTHON) test-zstd-versions.py + +automated_benchmarking: clean + $(PYTHON) automated_benchmarking.py + +checkTag: checkTag.c $(ZSTDDIR)/zstd.h + $(CC) $(FLAGS) $< -o $@$(EXT) + +clean: + $(MAKE) -C $(ZSTDDIR) clean + $(MAKE) -C $(PRGDIR) clean + @$(RM) -fR $(TESTARTEFACT) + @$(RM) -rf tmp* # some test directories are named tmp* + @$(RM) core *.o *.tmp result* *.gcda dictionary *.zst \ + $(PRGDIR)/zstd$(EXT) $(PRGDIR)/zstd32$(EXT) \ + fullbench$(EXT) fullbench32$(EXT) \ + fullbench-lib$(EXT) fullbench-dll$(EXT) \ + fuzzer$(EXT) fuzzer32$(EXT) zbufftest$(EXT) zbufftest32$(EXT) \ + fuzzer-dll$(EXT) zstreamtest-dll$(EXT) zbufftest-dll$(EXT) \ + zstreamtest$(EXT) zstreamtest32$(EXT) \ + datagen$(EXT) paramgrill$(EXT) roundTripCrash$(EXT) longmatch$(EXT) \ + symbols$(EXT) invalidDictionaries$(EXT) legacy$(EXT) poolTests$(EXT) \ + decodecorpus$(EXT) checkTag$(EXT) bigdict$(EXT) + @echo Cleaning completed + + +#---------------------------------------------------------------------------------- +# valgrind tests are validated only for some posix platforms +#---------------------------------------------------------------------------------- +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS)) +HOST_OS = POSIX + +valgrindTest: VALGRIND = valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 +valgrindTest: zstd datagen fuzzer fullbench + @echo "\n ---- valgrind tests : memory analyzer ----" + $(VALGRIND) ./datagen -g50M > $(VOID) + $(VALGRIND) $(PRGDIR)/zstd ; if [ $$? -eq 0 ] ; then echo "zstd without argument should have failed"; false; fi + ./datagen -g80 | $(VALGRIND) $(PRGDIR)/zstd - -c > $(VOID) + ./datagen -g16KB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID) + ./datagen -g2930KB | $(VALGRIND) $(PRGDIR)/zstd -5 -vf - -o tmp + $(VALGRIND) $(PRGDIR)/zstd -vdf tmp -c > $(VOID) + ./datagen -g64MB | $(VALGRIND) $(PRGDIR)/zstd -vf - -c > $(VOID) + @rm tmp + $(VALGRIND) ./fuzzer -T1mn -t1 + $(VALGRIND) ./fullbench -i1 + +endif + + +ifneq (,$(filter MINGW% MSYS%,$(shell uname))) +HOST_OS = MSYS +endif + + +#----------------------------------------------------------------------------- +# make tests validated only for below targets +#----------------------------------------------------------------------------- +ifneq (,$(filter $(HOST_OS),MSYS POSIX)) + +DIFF:=diff +ifneq (,$(filter $(shell uname),SunOS)) +DIFF:=gdiff +endif + +.PHONY: list +list: + @$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | xargs + +.PHONY: shortest +shortest: ZSTDRTTEST= +shortest: test-zstd + +.PHONY: fuzztest +fuzztest: test-fuzzer test-zstream test-decodecorpus + +.PHONY: test +test: test-zstd test-fullbench test-fuzzer test-zstream test-invalidDictionaries test-legacy test-decodecorpus +ifeq ($(QEMU_SYS),) +test: test-pool +endif + +test32: test-zstd32 test-fullbench32 test-fuzzer32 test-zstream32 + +test-all: test test32 valgrindTest test-decodecorpus-cli + + +.PHONY: test-zstd test-zstd32 test-zstd-nolegacy test-zstdgrep +test-zstd: ZSTD = $(PRGDIR)/zstd +test-zstd: zstd + +test-zstd32: ZSTD = $(PRGDIR)/zstd32 +test-zstd32: zstd32 + +test-zstd-nolegacy: ZSTD = $(PRGDIR)/zstd-nolegacy +test-zstd-nolegacy: zstd-nolegacy + +test-zstd test-zstd32 test-zstd-nolegacy: datagen + file $(ZSTD) + EXE_PREFIX="$(QEMU_SYS)" ZSTD_BIN="$(ZSTD)" DATAGEN_BIN=./datagen ./playTests.sh $(ZSTDRTTEST) + + +test-gzstd: gzstd + $(PRGDIR)/zstd -f README.md test-zstd-speed.py + gzip -f README.md test-zstd-speed.py + cat README.md.zst test-zstd-speed.py.gz >zstd_gz.zst + cat README.md.gz test-zstd-speed.py.zst >gz_zstd.gz + $(PRGDIR)/zstd -df README.md.gz -o README2.md + $(PRGDIR)/zstd -df README.md.gz test-zstd-speed.py.gz + $(PRGDIR)/zstd -df zstd_gz.zst gz_zstd.gz + $(DIFF) -q zstd_gz gz_zstd + echo Hello World ZSTD | $(PRGDIR)/zstd -c - >hello.zst + echo Hello World GZIP | gzip -c - >hello.gz + echo Hello World TEXT >hello.txt + cat hello.zst hello.gz hello.txt >hello_zst_gz_txt.gz + $(PRGDIR)/zstd -dcf hello.* + $(PRGDIR)/zstd -dcf - <hello_zst_gz_txt.gz + $(RM) *.gz *.zst README2.md gz_zstd zstd_gz hello.txt + +test-zstdgrep: gzstd + -[ -f /tmp/zstdcat ] || ln -s $(PWD)/$(PRGDIR)/zstd /tmp/zstdcat + echo a | $(PRGDIR)/zstd | env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep a + echo a | $(PRGDIR)/zstd | env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep b && return 1 || return 0 + -echo 'hello world' > test.txt && $(PRGDIR)/zstd test.txt + env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep hello test.txt.zst + env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep weird test.txt.zst && return 1 || return 0 + -echo 'hello' > pattern.txt + env ZCAT=/tmp/zstdcat $(PRGDIR)/zstdgrep -f pattern.txt test.txt.zst + $(RM) test.txt test.txt.zst pattern.txt + +test-fullbench: fullbench datagen + $(QEMU_SYS) ./fullbench -i1 + $(QEMU_SYS) ./fullbench -i1 -P0 + +test-fullbench32: fullbench32 datagen + $(QEMU_SYS) ./fullbench32 -i1 + $(QEMU_SYS) ./fullbench32 -i1 -P0 + +test-fuzzer: fuzzer + $(QEMU_SYS) ./fuzzer -v $(FUZZERTEST) $(FUZZER_FLAGS) + +test-fuzzer-stackmode: MOREFLAGS += -DZSTD_HEAPMODE=0 +test-fuzzer-stackmode: test-fuzzer + +test-fuzzer32: fuzzer32 + $(QEMU_SYS) ./fuzzer32 -v $(FUZZERTEST) $(FUZZER_FLAGS) + +test-zbuff: zbufftest + $(QEMU_SYS) ./zbufftest $(ZSTREAM_TESTTIME) + +test-zbuff32: zbufftest32 + $(QEMU_SYS) ./zbufftest32 $(ZSTREAM_TESTTIME) + +test-zstream: zstreamtest + $(QEMU_SYS) ./zstreamtest -v $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) + $(QEMU_SYS) ./zstreamtest --mt -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) + $(QEMU_SYS) ./zstreamtest --newapi -t1 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) + +test-zstream32: zstreamtest32 + $(QEMU_SYS) ./zstreamtest32 $(ZSTREAM_TESTTIME) $(FUZZER_FLAGS) + +test-longmatch: longmatch + $(QEMU_SYS) ./longmatch + +test-bigdict: bigdict + $(QEMU_SYS) ./bigdict + +test-invalidDictionaries: invalidDictionaries + $(QEMU_SYS) ./invalidDictionaries + +test-legacy: legacy + $(QEMU_SYS) ./legacy + +test-decodecorpus: decodecorpus + $(QEMU_SYS) ./decodecorpus -t $(DECODECORPUS_TESTTIME) + +test-decodecorpus-cli: decodecorpus + @echo "\n ---- decodecorpus basic cli tests ----" + @mkdir testdir + ./decodecorpus -n5 -otestdir -ptestdir + @cd testdir && \ + $(ZSTD) -d z000000.zst -o tmp0 && \ + $(ZSTD) -d z000001.zst -o tmp1 && \ + $(ZSTD) -d z000002.zst -o tmp2 && \ + $(ZSTD) -d z000003.zst -o tmp3 && \ + $(ZSTD) -d z000004.zst -o tmp4 && \ + diff z000000 tmp0 && \ + diff z000001 tmp1 && \ + diff z000002 tmp2 && \ + diff z000003 tmp3 && \ + diff z000004 tmp4 && \ + rm ./* && \ + cd .. + @echo "\n ---- decodecorpus dictionary cli tests ----" + ./decodecorpus -n5 -otestdir -ptestdir --use-dict=1MB + @cd testdir && \ + $(ZSTD) -d z000000.zst -D dictionary -o tmp0 && \ + $(ZSTD) -d z000001.zst -D dictionary -o tmp1 && \ + $(ZSTD) -d z000002.zst -D dictionary -o tmp2 && \ + $(ZSTD) -d z000003.zst -D dictionary -o tmp3 && \ + $(ZSTD) -d z000004.zst -D dictionary -o tmp4 && \ + diff z000000 tmp0 && \ + diff z000001 tmp1 && \ + diff z000002 tmp2 && \ + diff z000003 tmp3 && \ + diff z000004 tmp4 && \ + cd .. + @rm -rf testdir + +test-pool: poolTests + $(QEMU_SYS) ./poolTests + +test-lz4: ZSTD = LD_LIBRARY_PATH=/usr/local/lib $(PRGDIR)/zstd +test-lz4: ZSTD_LZ4 = LD_LIBRARY_PATH=/usr/local/lib ./lz4 +test-lz4: ZSTD_UNLZ4 = LD_LIBRARY_PATH=/usr/local/lib ./unlz4 +test-lz4: zstd decodecorpus datagen + [ -f lz4 ] || ln -s $(PRGDIR)/zstd lz4 + [ -f unlz4 ] || ln -s $(PRGDIR)/zstd unlz4 + + ./decodecorpus -ptmp + # lz4 -> zstd + lz4 < tmp | \ + $(ZSTD) -d | \ + cmp - tmp + lz4 < tmp | \ + $(ZSTD_UNLZ4) | \ + cmp - tmp + # zstd -> lz4 + $(ZSTD) --format=lz4 < tmp | \ + lz4 -d | \ + cmp - tmp + $(ZSTD_LZ4) < tmp | \ + lz4 -d | \ + cmp - tmp + # zstd -> zstd + $(ZSTD) --format=lz4 < tmp | \ + $(ZSTD) -d | \ + cmp - tmp + # zstd -> zstd + $(ZSTD) < tmp | \ + $(ZSTD) -d | \ + cmp - tmp + + ./datagen -g384KB | $(ZSTD) --format=lz4 | $(ZSTD) -d > /dev/null + + rm tmp lz4 unlz4 + +endif diff --git a/src/zstd/tests/README.md b/src/zstd/tests/README.md new file mode 100644 index 000000000..23e00767c --- /dev/null +++ b/src/zstd/tests/README.md @@ -0,0 +1,185 @@ +Programs and scripts for automated testing of Zstandard +======================================================= + +This directory contains the following programs and scripts: +- `datagen` : Synthetic and parametrable data generator, for tests +- `fullbench` : Precisely measure speed for each zstd inner functions +- `fuzzer` : Test tool, to check zstd integrity on target platform +- `paramgrill` : parameter tester for zstd +- `test-zstd-speed.py` : script for testing zstd speed difference between commits +- `test-zstd-versions.py` : compatibility test between zstd versions stored on Github (v0.1+) +- `zbufftest` : Test tool to check ZBUFF (a buffered streaming API) integrity +- `zstreamtest` : Fuzzer test tool for zstd streaming API +- `legacy` : Test tool to test decoding of legacy zstd frames +- `decodecorpus` : Tool to generate valid Zstandard frames, for verifying decoder implementations + + +#### `test-zstd-versions.py` - script for testing zstd interoperability between versions + +This script creates `versionsTest` directory to which zstd repository is cloned. +Then all tagged (released) versions of zstd are compiled. +In the following step interoperability between zstd versions is checked. + +#### `automated-benchmarking.py` - script for benchmarking zstd prs to dev + +This script benchmarks facebook:dev and changes from pull requests made to zstd and compares +them against facebook:dev to detect regressions. This script currently runs on a dedicated +desktop machine for every pull request that is made to the zstd repo but can also +be run on any machine via the command line interface. + +There are three modes of usage for this script: fastmode will just run a minimal single +build comparison (between facebook:dev and facebook:master), onetime will pull all the current +pull requests from the zstd repo and compare facebook:dev to all of them once, continuous +will continuously get pull requests from the zstd repo and run benchmarks against facebook:dev. + +``` +Example usage: python automated_benchmarking.py +``` + +``` +usage: automated_benchmarking.py [-h] [--directory DIRECTORY] + [--levels LEVELS] [--iterations ITERATIONS] + [--emails EMAILS] [--frequency FREQUENCY] + [--mode MODE] [--dict DICT] + +optional arguments: + -h, --help show this help message and exit + --directory DIRECTORY + directory with files to benchmark + --levels LEVELS levels to test eg ('1,2,3') + --iterations ITERATIONS + number of benchmark iterations to run + --emails EMAILS email addresses of people who will be alerted upon + regression. Only for continuous mode + --frequency FREQUENCY + specifies the number of seconds to wait before each + successive check for new PRs in continuous mode + --mode MODE 'fastmode', 'onetime', 'current', or 'continuous' (see + README.md for details) + --dict DICT filename of dictionary to use (when set, this + dictioanry will be used to compress the files provided + inside --directory) +``` + +#### `test-zstd-speed.py` - script for testing zstd speed difference between commits + +DEPRECATED + +This script creates `speedTest` directory to which zstd repository is cloned. +Then it compiles all branches of zstd and performs a speed benchmark for a given list of files (the `testFileNames` parameter). +After `sleepTime` (an optional parameter, default 300 seconds) seconds the script checks repository for new commits. +If a new commit is found it is compiled and a speed benchmark for this commit is performed. +The results of the speed benchmark are compared to the previous results. +If compression or decompression speed for one of zstd levels is lower than `lowerLimit` (an optional parameter, default 0.98) the speed benchmark is restarted. +If second results are also lower than `lowerLimit` the warning e-mail is send to recipients from the list (the `emails` parameter). + +Additional remarks: +- To be sure that speed results are accurate the script should be run on a "stable" target system with no other jobs running in parallel +- Using the script with virtual machines can lead to large variations of speed results +- The speed benchmark is not performed until computers' load average is lower than `maxLoadAvg` (an optional parameter, default 0.75) +- The script sends e-mails using `mutt`; if `mutt` is not available it sends e-mails without attachments using `mail`; if both are not available it only prints a warning + + +The example usage with two test files, one e-mail address, and with an additional message: +``` +./test-zstd-speed.py "silesia.tar calgary.tar" "email@gmail.com" --message "tested on my laptop" --sleepTime 60 +``` + +To run the script in background please use: +``` +nohup ./test-zstd-speed.py testFileNames emails & +``` + +The full list of parameters: +``` +positional arguments: + testFileNames file names list for speed benchmark + emails list of e-mail addresses to send warnings + +optional arguments: + -h, --help show this help message and exit + --message MESSAGE attach an additional message to e-mail + --lowerLimit LOWERLIMIT + send email if speed is lower than given limit + --maxLoadAvg MAXLOADAVG + maximum load average to start testing + --lastCLevel LASTCLEVEL + last compression level for testing + --sleepTime SLEEPTIME + frequency of repository checking in seconds +``` + +#### `decodecorpus` - tool to generate Zstandard frames for decoder testing +Command line tool to generate test .zst files. + +This tool will generate .zst files with checksums, +as well as optionally output the corresponding correct uncompressed data for +extra verification. + +Example: +``` +./decodecorpus -ptestfiles -otestfiles -n10000 -s5 +``` +will generate 10,000 sample .zst files using a seed of 5 in the `testfiles` directory, +with the zstd checksum field set, +as well as the 10,000 original files for more detailed comparison of decompression results. + +``` +./decodecorpus -t -T1mn +``` +will choose a random seed, and for 1 minute, +generate random test frames and ensure that the +zstd library correctly decompresses them in both simple and streaming modes. + +#### `paramgrill` - tool for generating compression table parameters and optimizing parameters on file given constraints + +Full list of arguments +``` + -T# : set level 1 speed objective + -B# : cut input into blocks of size # (default : single block) + -S : benchmarks a single run (example command: -Sl3w10h12) + w# - windowLog + h# - hashLog + c# - chainLog + s# - searchLog + l# - minMatch + t# - targetLength + S# - strategy + L# - level + --zstd= : Single run, parameter selection syntax same as zstdcli with more parameters + (Added forceAttachDictionary / fadt) + When invoked with --optimize, this represents the sample to exceed. + --optimize= : find parameters to maximize compression ratio given parameters + Can use all --zstd= commands to constrain the type of solution found in addition to the following constraints + cSpeed= : Minimum compression speed + dSpeed= : Minimum decompression speed + cMem= : Maximum compression memory + lvl= : Searches for solutions which are strictly better than that compression lvl in ratio and cSpeed, + stc= : When invoked with lvl=, represents percentage slack in ratio/cSpeed allowed for a solution to be considered (Default 100%) + : In normal operation, represents percentage slack in choosing viable starting strategy selection in choosing the default parameters + (Lower value will begin with stronger strategies) (Default 90%) + speedRatio= (accepts decimals) + : determines value of gains in speed vs gains in ratio + when determining overall winner (default 5 (1% ratio = 5% speed)). + tries= : Maximum number of random restarts on a single strategy before switching (Default 5) + Higher values will make optimizer run longer, more chances to find better solution. + memLog : Limits the log of the size of each memotable (1 per strategy). Will use hash tables when state space is larger than max size. + Setting memLog = 0 turns off memoization + --display= : specify which parameters are included in the output + can use all --zstd parameter names and 'cParams' as a shorthand for all parameters used in ZSTD_compressionParameters + (Default: display all params available) + -P# : generated sample compressibility (when no file is provided) + -t# : Caps runtime of operation in seconds (default : 99999 seconds (about 27 hours )) + -v : Prints Benchmarking output + -D : Next argument dictionary file + -s : Benchmark all files separately + -q : Quiet, repeat for more quiet + -q Prints parameters + results whenever a new best is found + -qq Only prints parameters whenever a new best is found, prints final parameters + results + -qqq Only print final parameters + results + -qqqq Only prints final parameter set in the form --zstd= + -v : Verbose, cancels quiet, repeat for more volume + -v Prints all candidate parameters and results + +``` + Any inputs afterwards are treated as files to benchmark. diff --git a/src/zstd/tests/automated_benchmarking.py b/src/zstd/tests/automated_benchmarking.py new file mode 100644 index 000000000..d0cfb1fbe --- /dev/null +++ b/src/zstd/tests/automated_benchmarking.py @@ -0,0 +1,326 @@ +# ################################################################ +# Copyright (c) 2020-2020, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ########################################################################## + +import argparse +import glob +import json +import os +import time +import pickle as pk +import subprocess +import urllib.request + + +GITHUB_API_PR_URL = "https://api.github.com/repos/facebook/zstd/pulls?state=open" +GITHUB_URL_TEMPLATE = "https://github.com/{}/zstd" +MASTER_BUILD = {"user": "facebook", "branch": "dev", "hash": None} + +# check to see if there are any new PRs every minute +DEFAULT_MAX_API_CALL_FREQUENCY_SEC = 60 +PREVIOUS_PRS_FILENAME = "prev_prs.pk" + +# Not sure what the threshold for triggering alarms should be +# 1% regression sounds like a little too sensitive but the desktop +# that I'm running it on is pretty stable so I think this is fine +CSPEED_REGRESSION_TOLERANCE = 0.01 +DSPEED_REGRESSION_TOLERANCE = 0.01 + + +def get_new_open_pr_builds(prev_state=True): + prev_prs = None + if os.path.exists(PREVIOUS_PRS_FILENAME): + with open(PREVIOUS_PRS_FILENAME, "rb") as f: + prev_prs = pk.load(f) + data = json.loads(urllib.request.urlopen(GITHUB_API_PR_URL).read().decode("utf-8")) + prs = { + d["url"]: { + "user": d["user"]["login"], + "branch": d["head"]["ref"], + "hash": d["head"]["sha"].strip(), + } + for d in data + } + with open(PREVIOUS_PRS_FILENAME, "wb") as f: + pk.dump(prs, f) + if not prev_state or prev_prs == None: + return list(prs.values()) + return [pr for url, pr in prs.items() if url not in prev_prs or prev_prs[url] != pr] + + +def get_latest_hashes(): + tmp = subprocess.run(["git", "log", "-1"], stdout=subprocess.PIPE).stdout.decode( + "utf-8" + ) + sha1 = tmp.split("\n")[0].split(" ")[1] + tmp = subprocess.run( + ["git", "show", "{}^1".format(sha1)], stdout=subprocess.PIPE + ).stdout.decode("utf-8") + sha2 = tmp.split("\n")[0].split(" ")[1] + tmp = subprocess.run( + ["git", "show", "{}^2".format(sha1)], stdout=subprocess.PIPE + ).stdout.decode("utf-8") + sha3 = "" if len(tmp) == 0 else tmp.split("\n")[0].split(" ")[1] + return [sha1.strip(), sha2.strip(), sha3.strip()] + + +def get_builds_for_latest_hash(): + hashes = get_latest_hashes() + for b in get_new_open_pr_builds(False): + if b["hash"] in hashes: + return [b] + return [] + + +def clone_and_build(build): + if build["user"] != None: + github_url = GITHUB_URL_TEMPLATE.format(build["user"]) + os.system( + """ + rm -rf zstd-{user}-{sha} && + git clone {github_url} zstd-{user}-{sha} && + cd zstd-{user}-{sha} && + {checkout_command} + make && + cd ../ + """.format( + user=build["user"], + github_url=github_url, + sha=build["hash"], + checkout_command="git checkout {} &&".format(build["hash"]) + if build["hash"] != None + else "", + ) + ) + return "zstd-{user}-{sha}/zstd".format(user=build["user"], sha=build["hash"]) + else: + os.system("cd ../ && make && cd tests") + return "../zstd" + + +def parse_benchmark_output(output): + idx = [i for i, d in enumerate(output) if d == "MB/s"] + return [float(output[idx[0] - 1]), float(output[idx[1] - 1])] + + +def benchmark_single(executable, level, filename): + return parse_benchmark_output(( + subprocess.run( + [executable, "-qb{}".format(level), filename], stderr=subprocess.PIPE + ) + .stderr.decode("utf-8") + .split(" ") + )) + + +def benchmark_n(executable, level, filename, n): + speeds_arr = [benchmark_single(executable, level, filename) for _ in range(n)] + cspeed, dspeed = max(b[0] for b in speeds_arr), max(b[1] for b in speeds_arr) + print( + "Bench (executable={} level={} filename={}, iterations={}):\n\t[cspeed: {} MB/s, dspeed: {} MB/s]".format( + os.path.basename(executable), + level, + os.path.basename(filename), + n, + cspeed, + dspeed, + ) + ) + return (cspeed, dspeed) + + +def benchmark(build, filenames, levels, iterations): + executable = clone_and_build(build) + return [ + [benchmark_n(executable, l, f, iterations) for f in filenames] for l in levels + ] + + +def benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, level, iterations): + cspeeds, dspeeds = [], [] + for _ in range(iterations): + output = subprocess.run([executable, "-qb{}".format(level), "-D", dictionary_filename, "-r", filenames_directory], stderr=subprocess.PIPE).stderr.decode("utf-8").split(" ") + cspeed, dspeed = parse_benchmark_output(output) + cspeeds.append(cspeed) + dspeeds.append(dspeed) + max_cspeed, max_dspeed = max(cspeeds), max(dspeeds) + print( + "Bench (executable={} level={} filenames_directory={}, dictionary_filename={}, iterations={}):\n\t[cspeed: {} MB/s, dspeed: {} MB/s]".format( + os.path.basename(executable), + level, + os.path.basename(filenames_directory), + os.path.basename(dictionary_filename), + iterations, + max_cspeed, + max_dspeed, + ) + ) + return (max_cspeed, max_dspeed) + + +def benchmark_dictionary(build, filenames_directory, dictionary_filename, levels, iterations): + executable = clone_and_build(build) + return [benchmark_dictionary_single(executable, filenames_directory, dictionary_filename, l, iterations) for l in levels] + + +def parse_regressions_and_labels(old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build): + cspeed_reg = (old_cspeed - new_cspeed) / old_cspeed + dspeed_reg = (old_dspeed - new_dspeed) / old_dspeed + baseline_label = "{}:{} ({})".format( + baseline_build["user"], baseline_build["branch"], baseline_build["hash"] + ) + test_label = "{}:{} ({})".format( + test_build["user"], test_build["branch"], test_build["hash"] + ) + return cspeed_reg, dspeed_reg, baseline_label, test_label + + +def get_regressions(baseline_build, test_build, iterations, filenames, levels): + old = benchmark(baseline_build, filenames, levels, iterations) + new = benchmark(test_build, filenames, levels, iterations) + regressions = [] + for j, level in enumerate(levels): + for k, filename in enumerate(filenames): + old_cspeed, old_dspeed = old[j][k] + new_cspeed, new_dspeed = new[j][k] + cspeed_reg, dspeed_reg, baseline_label, test_label = parse_regressions_and_labels( + old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build + ) + if cspeed_reg > CSPEED_REGRESSION_TOLERANCE: + regressions.append( + "[COMPRESSION REGRESSION] (level={} filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format( + level, + filename, + baseline_label, + test_label, + old_cspeed, + new_cspeed, + cspeed_reg * 100.0, + ) + ) + if dspeed_reg > DSPEED_REGRESSION_TOLERANCE: + regressions.append( + "[DECOMPRESSION REGRESSION] (level={} filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format( + level, + filename, + baseline_label, + test_label, + old_dspeed, + new_dspeed, + dspeed_reg * 100.0, + ) + ) + return regressions + +def get_regressions_dictionary(baseline_build, test_build, filenames_directory, dictionary_filename, levels, iterations): + old = benchmark_dictionary(baseline_build, filenames_directory, dictionary_filename, levels, iterations) + new = benchmark_dictionary(test_build, filenames_directory, dictionary_filename, levels, iterations) + regressions = [] + for j, level in enumerate(levels): + old_cspeed, old_dspeed = old[j] + new_cspeed, new_dspeed = new[j] + cspeed_reg, dspeed_reg, baesline_label, test_label = parse_regressions_and_labels( + old_cspeed, new_cspeed, old_dspeed, new_dspeed, baseline_build, test_build + ) + if cspeed_reg > CSPEED_REGRESSION_TOLERANCE: + regressions.append( + "[COMPRESSION REGRESSION] (level={} filenames_directory={} dictionary_filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format( + level, + filenames_directory, + dictionary_filename, + baseline_label, + test_label, + old_cspeed, + new_cspeed, + cspeed_reg * 100.0, + ) + ) + if dspeed_reg > DSPEED_REGRESSION_TOLERANCE: + regressions.append( + "[DECOMPRESSION REGRESSION] (level={} filenames_directory={} dictionary_filename={})\n\t{} -> {}\n\t{} -> {} ({:0.2f}%)".format( + level, + filenames_directory, + dictionary_filename, + baseline_label, + test_label, + old_dspeed, + new_dspeed, + dspeed_reg * 100.0, + ) + ) + return regressions + + +def main(filenames, levels, iterations, builds=None, emails=None, continuous=False, frequency=DEFAULT_MAX_API_CALL_FREQUENCY_SEC, dictionary_filename=None): + if builds == None: + builds = get_new_open_pr_builds() + while True: + for test_build in builds: + if dictionary_filename == None: + regressions = get_regressions( + MASTER_BUILD, test_build, iterations, filenames, levels + ) + else: + regressions = get_regressions_dictionary( + MASTER_BUILD, test_build, filenames, dictionary_filename, levels, iterations + ) + body = "\n".join(regressions) + if len(regressions) > 0: + if emails != None: + os.system( + """ + echo "{}" | mutt -s "[zstd regression] caused by new pr" {} + """.format( + body, emails + ) + ) + print("Emails sent to {}".format(emails)) + print(body) + if not continuous: + break + time.sleep(frequency) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--directory", help="directory with files to benchmark", default="golden-compression") + parser.add_argument("--levels", help="levels to test eg ('1,2,3')", default="1") + parser.add_argument("--iterations", help="number of benchmark iterations to run", default="1") + parser.add_argument("--emails", help="email addresses of people who will be alerted upon regression. Only for continuous mode", default=None) + parser.add_argument("--frequency", help="specifies the number of seconds to wait before each successive check for new PRs in continuous mode", default=DEFAULT_MAX_API_CALL_FREQUENCY_SEC) + parser.add_argument("--mode", help="'fastmode', 'onetime', 'current', or 'continuous' (see README.md for details)", default="current") + parser.add_argument("--dict", help="filename of dictionary to use (when set, this dictioanry will be used to compress the files provided inside --directory)", default=None) + + args = parser.parse_args() + filenames = args.directory + levels = [int(l) for l in args.levels.split(",")] + mode = args.mode + iterations = int(args.iterations) + emails = args.emails + frequency = int(args.frequency) + dictionary_filename = args.dict + + if dictionary_filename == None: + filenames = glob.glob("{}/**".format(filenames)) + + if (len(filenames) == 0): + print("0 files found") + quit() + + if mode == "onetime": + main(filenames, levels, iterations, frequency=frequenc, dictionary_filename=dictionary_filename) + elif mode == "current": + builds = [{"user": None, "branch": "None", "hash": None}] + main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename) + elif mode == "fastmode": + builds = [{"user": "facebook", "branch": "master", "hash": None}] + main(filenames, levels, iterations, builds, frequency=frequency, dictionary_filename=dictionary_filename) + else: + main(filenames, levels, iterations, None, emails, True, frequency=frequency, dictionary_filename=dictionary_filename) diff --git a/src/zstd/tests/bigdict.c b/src/zstd/tests/bigdict.c new file mode 100644 index 000000000..aeda56cb5 --- /dev/null +++ b/src/zstd/tests/bigdict.c @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2017-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include <assert.h> +#include <stdio.h> +#include <stddef.h> +#include <stdlib.h> +#include <stdint.h> +#include "datagen.h" +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +static int +compress(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + void const* src, size_t srcSize, + void* roundtrip, ZSTD_EndDirective end) +{ + ZSTD_inBuffer in = {src, srcSize, 0}; + ZSTD_outBuffer out = {dst, dstCapacity, 0}; + int ended = 0; + + while (!ended && (in.pos < in.size || out.pos > 0)) { + size_t rc; + out.pos = 0; + rc = ZSTD_compressStream2(cctx, &out, &in, end); + if (ZSTD_isError(rc)) + return 1; + if (end == ZSTD_e_end && rc == 0) + ended = 1; + { + ZSTD_inBuffer rtIn = {dst, out.pos, 0}; + ZSTD_outBuffer rtOut = {roundtrip, srcSize, 0}; + rc = 1; + while (rtIn.pos < rtIn.size || rtOut.pos > 0) { + rtOut.pos = 0; + rc = ZSTD_decompressStream(dctx, &rtOut, &rtIn); + if (ZSTD_isError(rc)) { + fprintf(stderr, "Decompression error: %s\n", ZSTD_getErrorName(rc)); + return 1; + } + if (rc == 0) + break; + } + if (ended && rc != 0) { + fprintf(stderr, "Frame not finished!\n"); + return 1; + } + } + } + + return 0; +} + +int main(int argc, const char** argv) +{ + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + const size_t dataSize = (size_t)1 << 30; + const size_t outSize = ZSTD_compressBound(dataSize); + const size_t bufferSize = (size_t)1 << 31; + char* buffer = (char*)malloc(bufferSize); + void* out = malloc(outSize); + void* roundtrip = malloc(dataSize); + (void)argc; + (void)argv; + + if (!buffer || !out || !roundtrip || !cctx || !dctx) { + fprintf(stderr, "Allocation failure\n"); + return 1; + } + + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 31))) + return 1; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1))) + return 1; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_overlapLog, 9))) + return 1; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1))) + return 1; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, ZSTD_btopt))) + return 1; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, 7))) + return 1; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, 7))) + return 1; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, 1))) + return 1; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 10))) + return 1; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, 10))) + return 1; + + if (ZSTD_isError(ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 31))) + return 1; + + RDG_genBuffer(buffer, bufferSize, 1.0, 0.0, 0xbeefcafe); + + /* Compress 30 GB */ + { + int i; + for (i = 0; i < 10; ++i) { + fprintf(stderr, "Compressing 1 GB\n"); + if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_continue)) + return 1; + } + } + fprintf(stderr, "Compressing 1 GB\n"); + if (compress(cctx, dctx, out, outSize, buffer, dataSize, roundtrip, ZSTD_e_end)) + return 1; + + fprintf(stderr, "Success!\n"); + + free(roundtrip); + free(out); + free(buffer); + ZSTD_freeDCtx(dctx); + ZSTD_freeCCtx(cctx); + return 0; +} diff --git a/src/zstd/tests/checkTag.c b/src/zstd/tests/checkTag.c new file mode 100644 index 000000000..90af24ab1 --- /dev/null +++ b/src/zstd/tests/checkTag.c @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2018-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* checkTag : validation tool for libzstd + * command : + * $ ./checkTag tag + * checkTag validates tags of following format : v[0-9].[0-9].[0-9]{any} + * The tag is then compared to zstd version number. + * They are compatible if first 3 digits are identical. + * Anything beyond that is free, and doesn't impact validation. + * Example : tag v1.8.1.2 is compatible with version 1.8.1 + * When tag and version are not compatible, program exits with error code 1. + * When they are compatible, it exists with a code 0. + * checkTag is intended to be used in automated testing environment. + */ + +#include <stdio.h> /* printf */ +#include <string.h> /* strlen, strncmp */ +#include "zstd.h" /* ZSTD_VERSION_STRING */ + + +/* validate() : + * @return 1 if tag is compatible, 0 if not. + */ +static int validate(const char* const tag) +{ + size_t const tagLength = strlen(tag); + size_t const verLength = strlen(ZSTD_VERSION_STRING); + + if (tagLength < 2) return 0; + if (tag[0] != 'v') return 0; + if (tagLength <= verLength) return 0; + + if (strncmp(ZSTD_VERSION_STRING, tag+1, verLength)) return 0; + + return 1; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + const char* const tag = argv[1]; + if (argc!=2) { + printf("incorrect usage : %s tag \n", exeName); + return 2; + } + + printf("Version : %s \n", ZSTD_VERSION_STRING); + printf("Tag : %s \n", tag); + + if (validate(tag)) { + printf("OK : tag is compatible with zstd version \n"); + return 0; + } + + printf("!! error : tag and versions are not compatible !! \n"); + return 1; +} diff --git a/src/zstd/tests/datagencli.c b/src/zstd/tests/datagencli.c new file mode 100644 index 000000000..713ca9963 --- /dev/null +++ b/src/zstd/tests/datagencli.c @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*-************************************ +* Dependencies +**************************************/ +#include "util.h" /* Compiler options */ +#include <stdio.h> /* fprintf, stderr */ +#include "datagen.h" /* RDG_generate */ + + +/*-************************************ +* Constants +**************************************/ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define SIZE_DEFAULT ((64 KB) + 1) +#define SEED_DEFAULT 0 +#define COMPRESSIBILITY_DEFAULT 50 + + +/*-************************************ +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static unsigned displayLevel = 2; + + +/*-******************************************************* +* Command line +*********************************************************/ +static int usage(const char* programName) +{ + DISPLAY( "Compressible data generator\n"); + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -g# : generate # data (default:%i)\n", SIZE_DEFAULT); + DISPLAY( " -s# : Select seed (default:%i)\n", SEED_DEFAULT); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", + COMPRESSIBILITY_DEFAULT); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, const char** argv) +{ + unsigned probaU32 = COMPRESSIBILITY_DEFAULT; + double litProba = 0.0; + U64 size = SIZE_DEFAULT; + U32 seed = SEED_DEFAULT; + const char* const programName = argv[0]; + + int argNb; + for(argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + + if(!argument) continue; /* Protection if argument empty */ + + /* Handle commands. Aggregated commands are allowed */ + if (*argument=='-') { + argument++; + while (*argument!=0) { + switch(*argument) + { + case 'h': + return usage(programName); + case 'g': + argument++; + size=0; + while ((*argument>='0') && (*argument<='9')) + size *= 10, size += *argument++ - '0'; + if (*argument=='K') { size <<= 10; argument++; } + if (*argument=='M') { size <<= 20; argument++; } + if (*argument=='G') { size <<= 30; argument++; } + if (*argument=='B') { argument++; } + break; + case 's': + argument++; + seed=0; + while ((*argument>='0') && (*argument<='9')) + seed *= 10, seed += *argument++ - '0'; + break; + case 'P': + argument++; + probaU32 = 0; + while ((*argument>='0') && (*argument<='9')) + probaU32 *= 10, probaU32 += *argument++ - '0'; + if (probaU32>100) probaU32 = 100; + break; + case 'L': /* hidden argument : Literal distribution probability */ + argument++; + litProba=0.; + while ((*argument>='0') && (*argument<='9')) + litProba *= 10, litProba += *argument++ - '0'; + if (litProba>100.) litProba=100.; + litProba /= 100.; + break; + case 'v': + displayLevel = 4; + argument++; + break; + default: + return usage(programName); + } + } } } /* for(argNb=1; argNb<argc; argNb++) */ + + DISPLAYLEVEL(4, "Compressible data Generator \n"); + if (probaU32!=COMPRESSIBILITY_DEFAULT) + DISPLAYLEVEL(3, "Compressibility : %i%%\n", probaU32); + DISPLAYLEVEL(3, "Seed = %u \n", (unsigned)seed); + + RDG_genStdout(size, (double)probaU32/100, litProba, seed); + DISPLAYLEVEL(1, "\n"); + + return 0; +} diff --git a/src/zstd/tests/decodecorpus.c b/src/zstd/tests/decodecorpus.c new file mode 100644 index 000000000..a46fc24d8 --- /dev/null +++ b/src/zstd/tests/decodecorpus.c @@ -0,0 +1,1932 @@ +/* + * Copyright (c) 2017-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include <limits.h> +#include <math.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "util.h" +#include "timefn.h" /* UTIL_clockSpanMicro, SEC_TO_MICRO, UTIL_TIME_INITIALIZER */ +#include "zstd.h" +#include "zstd_internal.h" +#include "mem.h" +#define ZDICT_STATIC_LINKING_ONLY +#include "zdict.h" + +/* Direct access to internal compression functions is required */ +#include "zstd_compress.c" + +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" /* XXH64 */ + +#ifndef MIN + #define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +#ifndef MAX_PATH + #ifdef PATH_MAX + #define MAX_PATH PATH_MAX + #else + #define MAX_PATH 256 + #endif +#endif + +/*-************************************ +* DISPLAY Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; + +#define DISPLAYUPDATE(...) \ + do { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || \ + (g_displayLevel >= 4)) { \ + g_displayClock = UTIL_getTime(); \ + DISPLAY(__VA_ARGS__); \ + if (g_displayLevel >= 4) fflush(stderr); \ + } \ + } while (0) + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define CHECKERR(code) \ + do { \ + if (ZSTD_isError(code)) { \ + DISPLAY("Error occurred while generating data: %s\n", \ + ZSTD_getErrorName(code)); \ + exit(1); \ + } \ + } while (0) + +/*-******************************************************* +* Random function +*********************************************************/ +static U32 RAND(U32* src) +{ +#define RAND_rotl32(x,r) ((x << r) | (x >> (32 - r))) + static const U32 prime1 = 2654435761U; + static const U32 prime2 = 2246822519U; + U32 rand32 = *src; + rand32 *= prime1; + rand32 += prime2; + rand32 = RAND_rotl32(rand32, 13); + *src = rand32; + return RAND_rotl32(rand32, 27); +#undef RAND_rotl32 +} + +#define DISTSIZE (8192) + +/* Write `size` bytes into `ptr`, all of which are less than or equal to `maxSymb` */ +static void RAND_bufferMaxSymb(U32* seed, void* ptr, size_t size, int maxSymb) +{ + size_t i; + BYTE* op = ptr; + + for (i = 0; i < size; i++) { + op[i] = (BYTE) (RAND(seed) % (maxSymb + 1)); + } +} + +/* Write `size` random bytes into `ptr` */ +static void RAND_buffer(U32* seed, void* ptr, size_t size) +{ + size_t i; + BYTE* op = ptr; + + for (i = 0; i + 4 <= size; i += 4) { + MEM_writeLE32(op + i, RAND(seed)); + } + for (; i < size; i++) { + op[i] = RAND(seed) & 0xff; + } +} + +/* Write `size` bytes into `ptr` following the distribution `dist` */ +static void RAND_bufferDist(U32* seed, BYTE* dist, void* ptr, size_t size) +{ + size_t i; + BYTE* op = ptr; + + for (i = 0; i < size; i++) { + op[i] = dist[RAND(seed) % DISTSIZE]; + } +} + +/* Generate a random distribution where the frequency of each symbol follows a + * geometric distribution defined by `weight` + * `dist` should have size at least `DISTSIZE` */ +static void RAND_genDist(U32* seed, BYTE* dist, double weight) +{ + size_t i = 0; + size_t statesLeft = DISTSIZE; + BYTE symb = (BYTE) (RAND(seed) % 256); + BYTE step = (BYTE) ((RAND(seed) % 256) | 1); /* force it to be odd so it's relatively prime to 256 */ + + while (i < DISTSIZE) { + size_t states = ((size_t)(weight * statesLeft)) + 1; + size_t j; + for (j = 0; j < states && i < DISTSIZE; j++, i++) { + dist[i] = symb; + } + + symb += step; + statesLeft -= states; + } +} + +/* Generates a random number in the range [min, max) */ +static inline U32 RAND_range(U32* seed, U32 min, U32 max) +{ + return (RAND(seed) % (max-min)) + min; +} + +#define ROUND(x) ((U32)(x + 0.5)) + +/* Generates a random number in an exponential distribution with mean `mean` */ +static double RAND_exp(U32* seed, double mean) +{ + double const u = RAND(seed) / (double) UINT_MAX; + return log(1-u) * (-mean); +} + +/*-******************************************************* +* Constants and Structs +*********************************************************/ +const char *BLOCK_TYPES[] = {"raw", "rle", "compressed"}; + +#define MAX_DECOMPRESSED_SIZE_LOG 20 +#define MAX_DECOMPRESSED_SIZE (1ULL << MAX_DECOMPRESSED_SIZE_LOG) + +#define MAX_WINDOW_LOG 22 /* Recommended support is 8MB, so limit to 4MB + mantissa */ + +#define MIN_SEQ_LEN (3) +#define MAX_NB_SEQ ((ZSTD_BLOCKSIZE_MAX + MIN_SEQ_LEN - 1) / MIN_SEQ_LEN) + +BYTE CONTENT_BUFFER[MAX_DECOMPRESSED_SIZE]; +BYTE FRAME_BUFFER[MAX_DECOMPRESSED_SIZE * 2]; +BYTE LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; + +seqDef SEQUENCE_BUFFER[MAX_NB_SEQ]; +BYTE SEQUENCE_LITERAL_BUFFER[ZSTD_BLOCKSIZE_MAX]; /* storeSeq expects a place to copy literals to */ +BYTE SEQUENCE_LLCODE[ZSTD_BLOCKSIZE_MAX]; +BYTE SEQUENCE_MLCODE[ZSTD_BLOCKSIZE_MAX]; +BYTE SEQUENCE_OFCODE[ZSTD_BLOCKSIZE_MAX]; + +unsigned WKSP[HUF_WORKSPACE_SIZE_U32]; + +typedef struct { + size_t contentSize; /* 0 means unknown (unless contentSize == windowSize == 0) */ + unsigned windowSize; /* contentSize >= windowSize means single segment */ +} frameHeader_t; + +/* For repeat modes */ +typedef struct { + U32 rep[ZSTD_REP_NUM]; + + int hufInit; + /* the distribution used in the previous block for repeat mode */ + BYTE hufDist[DISTSIZE]; + U32 hufTable [256]; /* HUF_CElt is an incomplete type */ + + int fseInit; + FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; + + /* Symbols that were present in the previous distribution, for use with + * set_repeat */ + BYTE litlengthSymbolSet[36]; + BYTE offsetSymbolSet[29]; + BYTE matchlengthSymbolSet[53]; +} cblockStats_t; + +typedef struct { + void* data; + void* dataStart; + void* dataEnd; + + void* src; + void* srcStart; + void* srcEnd; + + frameHeader_t header; + + cblockStats_t stats; + cblockStats_t oldStats; /* so they can be rolled back if uncompressible */ +} frame_t; + +typedef struct { + int useDict; + U32 dictID; + size_t dictContentSize; + BYTE* dictContent; +} dictInfo; + +typedef enum { + gt_frame = 0, /* generate frames */ + gt_block, /* generate compressed blocks without block/frame headers */ +} genType_e; + +/*-******************************************************* +* Global variables (set from command line) +*********************************************************/ +U32 g_maxDecompressedSizeLog = MAX_DECOMPRESSED_SIZE_LOG; /* <= 20 */ +U32 g_maxBlockSize = ZSTD_BLOCKSIZE_MAX; /* <= 128 KB */ + +/*-******************************************************* +* Generator Functions +*********************************************************/ + +struct { + int contentSize; /* force the content size to be present */ +} opts; /* advanced options on generation */ + +/* Generate and write a random frame header */ +static void writeFrameHeader(U32* seed, frame_t* frame, dictInfo info) +{ + BYTE* const op = frame->data; + size_t pos = 0; + frameHeader_t fh; + + BYTE windowByte = 0; + + int singleSegment = 0; + int contentSizeFlag = 0; + int fcsCode = 0; + + memset(&fh, 0, sizeof(fh)); + + /* generate window size */ + { + /* Follow window algorithm from specification */ + int const exponent = RAND(seed) % (MAX_WINDOW_LOG - 10); + int const mantissa = RAND(seed) % 8; + windowByte = (BYTE) ((exponent << 3) | mantissa); + fh.windowSize = (1U << (exponent + 10)); + fh.windowSize += fh.windowSize / 8 * mantissa; + } + + { + /* Generate random content size */ + size_t highBit; + if (RAND(seed) & 7 && g_maxDecompressedSizeLog > 7) { + /* do content of at least 128 bytes */ + highBit = 1ULL << RAND_range(seed, 7, g_maxDecompressedSizeLog); + } else if (RAND(seed) & 3) { + /* do small content */ + highBit = 1ULL << RAND_range(seed, 0, MIN(7, 1U << g_maxDecompressedSizeLog)); + } else { + /* 0 size frame */ + highBit = 0; + } + fh.contentSize = highBit ? highBit + (RAND(seed) % highBit) : 0; + + /* provide size sometimes */ + contentSizeFlag = opts.contentSize | (RAND(seed) & 1); + + if (contentSizeFlag && (fh.contentSize == 0 || !(RAND(seed) & 7))) { + /* do single segment sometimes */ + fh.windowSize = (U32) fh.contentSize; + singleSegment = 1; + } + } + + if (contentSizeFlag) { + /* Determine how large fcs field has to be */ + int minFcsCode = (fh.contentSize >= 256) + + (fh.contentSize >= 65536 + 256) + + (fh.contentSize > 0xFFFFFFFFU); + if (!singleSegment && !minFcsCode) { + minFcsCode = 1; + } + fcsCode = minFcsCode + (RAND(seed) % (4 - minFcsCode)); + if (fcsCode == 1 && fh.contentSize < 256) fcsCode++; + } + + /* write out the header */ + MEM_writeLE32(op + pos, ZSTD_MAGICNUMBER); + pos += 4; + + { + /* + * fcsCode: 2-bit flag specifying how many bytes used to represent Frame_Content_Size (bits 7-6) + * singleSegment: 1-bit flag describing if data must be regenerated within a single continuous memory segment. (bit 5) + * contentChecksumFlag: 1-bit flag that is set if frame includes checksum at the end -- set to 1 below (bit 2) + * dictBits: 2-bit flag describing how many bytes Dictionary_ID uses -- set to 3 (bits 1-0) + * For more information: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#frame_header + */ + int const dictBits = info.useDict ? 3 : 0; + BYTE const frameHeaderDescriptor = + (BYTE) ((fcsCode << 6) | (singleSegment << 5) | (1 << 2) | dictBits); + op[pos++] = frameHeaderDescriptor; + } + + if (!singleSegment) { + op[pos++] = windowByte; + } + if (info.useDict) { + MEM_writeLE32(op + pos, (U32) info.dictID); + pos += 4; + } + if (contentSizeFlag) { + switch (fcsCode) { + default: /* Impossible */ + case 0: op[pos++] = (BYTE) fh.contentSize; break; + case 1: MEM_writeLE16(op + pos, (U16) (fh.contentSize - 256)); pos += 2; break; + case 2: MEM_writeLE32(op + pos, (U32) fh.contentSize); pos += 4; break; + case 3: MEM_writeLE64(op + pos, (U64) fh.contentSize); pos += 8; break; + } + } + + DISPLAYLEVEL(3, " frame content size:\t%u\n", (unsigned)fh.contentSize); + DISPLAYLEVEL(3, " frame window size:\t%u\n", fh.windowSize); + DISPLAYLEVEL(3, " content size flag:\t%d\n", contentSizeFlag); + DISPLAYLEVEL(3, " single segment flag:\t%d\n", singleSegment); + + frame->data = op + pos; + frame->header = fh; +} + +/* Write a literal block in either raw or RLE form, return the literals size */ +static size_t writeLiteralsBlockSimple(U32* seed, frame_t* frame, size_t contentSize) +{ + BYTE* op = (BYTE*)frame->data; + int const type = RAND(seed) % 2; + int const sizeFormatDesc = RAND(seed) % 8; + size_t litSize; + size_t maxLitSize = MIN(contentSize, g_maxBlockSize); + + if (sizeFormatDesc == 0) { + /* Size_FormatDesc = ?0 */ + maxLitSize = MIN(maxLitSize, 31); + } else if (sizeFormatDesc <= 4) { + /* Size_FormatDesc = 01 */ + maxLitSize = MIN(maxLitSize, 4095); + } else { + /* Size_Format = 11 */ + maxLitSize = MIN(maxLitSize, 1048575); + } + + litSize = RAND(seed) % (maxLitSize + 1); + if (frame->src == frame->srcStart && litSize == 0) { + litSize = 1; /* no empty literals if there's nothing preceding this block */ + } + if (litSize + 3 > contentSize) { + litSize = contentSize; /* no matches shorter than 3 are allowed */ + } + /* use smallest size format that fits */ + if (litSize < 32) { + op[0] = (type | (0 << 2) | (litSize << 3)) & 0xff; + op += 1; + } else if (litSize < 4096) { + op[0] = (type | (1 << 2) | (litSize << 4)) & 0xff; + op[1] = (litSize >> 4) & 0xff; + op += 2; + } else { + op[0] = (type | (3 << 2) | (litSize << 4)) & 0xff; + op[1] = (litSize >> 4) & 0xff; + op[2] = (litSize >> 12) & 0xff; + op += 3; + } + + if (type == 0) { + /* Raw literals */ + DISPLAYLEVEL(4, " raw literals\n"); + + RAND_buffer(seed, LITERAL_BUFFER, litSize); + memcpy(op, LITERAL_BUFFER, litSize); + op += litSize; + } else { + /* RLE literals */ + BYTE const symb = (BYTE) (RAND(seed) % 256); + + DISPLAYLEVEL(4, " rle literals: 0x%02x\n", (unsigned)symb); + + memset(LITERAL_BUFFER, symb, litSize); + op[0] = symb; + op++; + } + + frame->data = op; + + return litSize; +} + +/* Generate a Huffman header for the given source */ +static size_t writeHufHeader(U32* seed, HUF_CElt* hufTable, void* dst, size_t dstSize, + const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + + unsigned huffLog = 11; + unsigned maxSymbolValue = 255; + + unsigned count[HUF_SYMBOLVALUE_MAX+1]; + + /* Scan input and build symbol stats */ + { size_t const largest = HIST_count_wksp (count, &maxSymbolValue, (const BYTE*)src, srcSize, WKSP, sizeof(WKSP)); + assert(!HIST_isError(largest)); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 0; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ + } + + /* Build Huffman Tree */ + /* Max Huffman log is 11, min is highbit(maxSymbolValue)+1 */ + huffLog = RAND_range(seed, ZSTD_highbit32(maxSymbolValue)+1, huffLog+1); + DISPLAYLEVEL(6, " huffman log: %u\n", huffLog); + { size_t const maxBits = HUF_buildCTable_wksp (hufTable, count, maxSymbolValue, huffLog, WKSP, sizeof(WKSP)); + CHECKERR(maxBits); + huffLog = (U32)maxBits; + } + + /* Write table description header */ + { size_t const hSize = HUF_writeCTable (op, dstSize, hufTable, maxSymbolValue, huffLog); + if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ + op += hSize; + } + + return op - ostart; +} + +/* Write a Huffman coded literals block and return the literals size */ +static size_t writeLiteralsBlockCompressed(U32* seed, frame_t* frame, size_t contentSize) +{ + BYTE* origop = (BYTE*)frame->data; + BYTE* opend = (BYTE*)frame->dataEnd; + BYTE* op; + BYTE* const ostart = origop; + int const sizeFormat = RAND(seed) % 4; + size_t litSize; + size_t hufHeaderSize = 0; + size_t compressedSize = 0; + size_t maxLitSize = MIN(contentSize-3, g_maxBlockSize); + + symbolEncodingType_e hType; + + if (contentSize < 64) { + /* make sure we get reasonably-sized literals for compression */ + return ERROR(GENERIC); + } + + DISPLAYLEVEL(4, " compressed literals\n"); + + switch (sizeFormat) { + case 0: /* fall through, size is the same as case 1 */ + case 1: + maxLitSize = MIN(maxLitSize, 1023); + origop += 3; + break; + case 2: + maxLitSize = MIN(maxLitSize, 16383); + origop += 4; + break; + case 3: + maxLitSize = MIN(maxLitSize, 262143); + origop += 5; + break; + default:; /* impossible */ + } + + do { + op = origop; + do { + litSize = RAND(seed) % (maxLitSize + 1); + } while (litSize < 32); /* avoid small literal sizes */ + if (litSize + 3 > contentSize) { + litSize = contentSize; /* no matches shorter than 3 are allowed */ + } + + /* most of the time generate a new distribution */ + if ((RAND(seed) & 3) || !frame->stats.hufInit) { + do { + if (RAND(seed) & 3) { + /* add 10 to ensure some compressibility */ + double const weight = ((RAND(seed) % 90) + 10) / 100.0; + + DISPLAYLEVEL(5, " distribution weight: %d%%\n", + (int)(weight * 100)); + + RAND_genDist(seed, frame->stats.hufDist, weight); + } else { + /* sometimes do restricted range literals to force + * non-huffman headers */ + DISPLAYLEVEL(5, " small range literals\n"); + RAND_bufferMaxSymb(seed, frame->stats.hufDist, DISTSIZE, + 15); + } + RAND_bufferDist(seed, frame->stats.hufDist, LITERAL_BUFFER, + litSize); + + /* generate the header from the distribution instead of the + * actual data to avoid bugs with symbols that were in the + * distribution but never showed up in the output */ + hufHeaderSize = writeHufHeader( + seed, (HUF_CElt*)frame->stats.hufTable, op, opend - op, + frame->stats.hufDist, DISTSIZE); + CHECKERR(hufHeaderSize); + /* repeat until a valid header is written */ + } while (hufHeaderSize == 0); + op += hufHeaderSize; + hType = set_compressed; + + frame->stats.hufInit = 1; + } else { + /* repeat the distribution/table from last time */ + DISPLAYLEVEL(5, " huffman repeat stats\n"); + RAND_bufferDist(seed, frame->stats.hufDist, LITERAL_BUFFER, + litSize); + hufHeaderSize = 0; + hType = set_repeat; + } + + do { + compressedSize = + sizeFormat == 0 + ? HUF_compress1X_usingCTable( + op, opend - op, LITERAL_BUFFER, litSize, + (HUF_CElt*)frame->stats.hufTable) + : HUF_compress4X_usingCTable( + op, opend - op, LITERAL_BUFFER, litSize, + (HUF_CElt*)frame->stats.hufTable); + CHECKERR(compressedSize); + /* this only occurs when it could not compress or similar */ + } while (compressedSize <= 0); + + op += compressedSize; + + compressedSize += hufHeaderSize; + DISPLAYLEVEL(5, " regenerated size: %u\n", (unsigned)litSize); + DISPLAYLEVEL(5, " compressed size: %u\n", (unsigned)compressedSize); + if (compressedSize >= litSize) { + DISPLAYLEVEL(5, " trying again\n"); + /* if we have to try again, reset the stats so we don't accidentally + * try to repeat a distribution we just made */ + frame->stats = frame->oldStats; + } else { + break; + } + } while (1); + + /* write header */ + switch (sizeFormat) { + case 0: /* fall through, size is the same as case 1 */ + case 1: { + U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) | + ((U32)compressedSize << 14); + MEM_writeLE24(ostart, header); + break; + } + case 2: { + U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) | + ((U32)compressedSize << 18); + MEM_writeLE32(ostart, header); + break; + } + case 3: { + U32 const header = hType | (sizeFormat << 2) | ((U32)litSize << 4) | + ((U32)compressedSize << 22); + MEM_writeLE32(ostart, header); + ostart[4] = (BYTE)(compressedSize >> 10); + break; + } + default:; /* impossible */ + } + + frame->data = op; + return litSize; +} + +static size_t writeLiteralsBlock(U32* seed, frame_t* frame, size_t contentSize) +{ + /* only do compressed for larger segments to avoid compressibility issues */ + if (RAND(seed) & 7 && contentSize >= 64) { + return writeLiteralsBlockCompressed(seed, frame, contentSize); + } else { + return writeLiteralsBlockSimple(seed, frame, contentSize); + } +} + +static inline void initSeqStore(seqStore_t *seqStore) { + seqStore->maxNbSeq = MAX_NB_SEQ; + seqStore->maxNbLit = ZSTD_BLOCKSIZE_MAX; + seqStore->sequencesStart = SEQUENCE_BUFFER; + seqStore->litStart = SEQUENCE_LITERAL_BUFFER; + seqStore->llCode = SEQUENCE_LLCODE; + seqStore->mlCode = SEQUENCE_MLCODE; + seqStore->ofCode = SEQUENCE_OFCODE; + + ZSTD_resetSeqStore(seqStore); +} + +/* Randomly generate sequence commands */ +static U32 generateSequences(U32* seed, frame_t* frame, seqStore_t* seqStore, + size_t contentSize, size_t literalsSize, dictInfo info) +{ + /* The total length of all the matches */ + size_t const remainingMatch = contentSize - literalsSize; + size_t excessMatch = 0; + U32 numSequences = 0; + + U32 i; + + + const BYTE* literals = LITERAL_BUFFER; + BYTE* srcPtr = frame->src; + + if (literalsSize != contentSize) { + /* each match must be at least MIN_SEQ_LEN, so this is the maximum + * number of sequences we can have */ + U32 const maxSequences = (U32)remainingMatch / MIN_SEQ_LEN; + numSequences = (RAND(seed) % maxSequences) + 1; + + /* the extra match lengths we have to allocate to each sequence */ + excessMatch = remainingMatch - numSequences * MIN_SEQ_LEN; + } + + DISPLAYLEVEL(5, " total match lengths: %u\n", (unsigned)remainingMatch); + for (i = 0; i < numSequences; i++) { + /* Generate match and literal lengths by exponential distribution to + * ensure nice numbers */ + U32 matchLen = + MIN_SEQ_LEN + + ROUND(RAND_exp(seed, excessMatch / (double)(numSequences - i))); + U32 literalLen = + (RAND(seed) & 7) + ? ROUND(RAND_exp(seed, + literalsSize / + (double)(numSequences - i))) + : 0; + /* actual offset, code to send, and point to copy up to when shifting + * codes in the repeat offsets history */ + U32 offset, offsetCode, repIndex; + + /* bounds checks */ + matchLen = (U32) MIN(matchLen, excessMatch + MIN_SEQ_LEN); + literalLen = MIN(literalLen, (U32) literalsSize); + if (i == 0 && srcPtr == frame->srcStart && literalLen == 0) literalLen = 1; + if (i + 1 == numSequences) matchLen = MIN_SEQ_LEN + (U32) excessMatch; + + memcpy(srcPtr, literals, literalLen); + srcPtr += literalLen; + do { + if (RAND(seed) & 7) { + /* do a normal offset */ + U32 const dataDecompressed = (U32)((BYTE*)srcPtr-(BYTE*)frame->srcStart); + offset = (RAND(seed) % + MIN(frame->header.windowSize, + (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) + + 1; + if (info.useDict && (RAND(seed) & 1) && i + 1 != numSequences && dataDecompressed < frame->header.windowSize) { + /* need to occasionally generate offsets that go past the start */ + /* including i+1 != numSequences because the last sequences has to adhere to predetermined contentSize */ + U32 lenPastStart = (RAND(seed) % info.dictContentSize) + 1; + offset = (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart)+lenPastStart; + if (offset > frame->header.windowSize) { + if (lenPastStart < MIN_SEQ_LEN) { + /* when offset > windowSize, matchLen bound by end of dictionary (lenPastStart) */ + /* this also means that lenPastStart must be greater than MIN_SEQ_LEN */ + /* make sure lenPastStart does not go past dictionary start though */ + lenPastStart = MIN(lenPastStart+MIN_SEQ_LEN, (U32)info.dictContentSize); + offset = (U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) + lenPastStart; + } + { + U32 const matchLenBound = MIN(frame->header.windowSize, lenPastStart); + matchLen = MIN(matchLen, matchLenBound); + } + } + } + offsetCode = offset + ZSTD_REP_MOVE; + repIndex = 2; + } else { + /* do a repeat offset */ + offsetCode = RAND(seed) % 3; + if (literalLen > 0) { + offset = frame->stats.rep[offsetCode]; + repIndex = offsetCode; + } else { + /* special case */ + offset = offsetCode == 2 ? frame->stats.rep[0] - 1 + : frame->stats.rep[offsetCode + 1]; + repIndex = MIN(2, offsetCode + 1); + } + } + } while (((!info.useDict) && (offset > (size_t)((BYTE*)srcPtr - (BYTE*)frame->srcStart))) || offset == 0); + + { + size_t j; + BYTE* const dictEnd = info.dictContent + info.dictContentSize; + for (j = 0; j < matchLen; j++) { + if ((U32)((BYTE*)srcPtr - (BYTE*)frame->srcStart) < offset) { + /* copy from dictionary instead of literals */ + size_t const dictOffset = offset - (srcPtr - (BYTE*)frame->srcStart); + *srcPtr = *(dictEnd - dictOffset); + } + else { + *srcPtr = *(srcPtr-offset); + } + srcPtr++; + } + } + + { int r; + for (r = repIndex; r > 0; r--) { + frame->stats.rep[r] = frame->stats.rep[r - 1]; + } + frame->stats.rep[0] = offset; + } + + DISPLAYLEVEL(6, " LL: %5u OF: %5u ML: %5u", + (unsigned)literalLen, (unsigned)offset, (unsigned)matchLen); + DISPLAYLEVEL(7, " srcPos: %8u seqNb: %3u", + (unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart), (unsigned)i); + DISPLAYLEVEL(6, "\n"); + if (offsetCode < 3) { + DISPLAYLEVEL(7, " repeat offset: %d\n", (int)repIndex); + } + /* use libzstd sequence handling */ + ZSTD_storeSeq(seqStore, literalLen, literals, literals + literalLen, + offsetCode, matchLen - MINMATCH); + + literalsSize -= literalLen; + excessMatch -= (matchLen - MIN_SEQ_LEN); + literals += literalLen; + } + + memcpy(srcPtr, literals, literalsSize); + srcPtr += literalsSize; + DISPLAYLEVEL(6, " excess literals: %5u", (unsigned)literalsSize); + DISPLAYLEVEL(7, " srcPos: %8u", (unsigned)((BYTE*)srcPtr - (BYTE*)frame->srcStart)); + DISPLAYLEVEL(6, "\n"); + + return numSequences; +} + +static void initSymbolSet(const BYTE* symbols, size_t len, BYTE* set, BYTE maxSymbolValue) +{ + size_t i; + + memset(set, 0, (size_t)maxSymbolValue+1); + + for (i = 0; i < len; i++) { + set[symbols[i]] = 1; + } +} + +static int isSymbolSubset(const BYTE* symbols, size_t len, const BYTE* set, BYTE maxSymbolValue) +{ + size_t i; + + for (i = 0; i < len; i++) { + if (symbols[i] > maxSymbolValue || !set[symbols[i]]) { + return 0; + } + } + return 1; +} + +static size_t writeSequences(U32* seed, frame_t* frame, seqStore_t* seqStorePtr, + size_t nbSeq) +{ + /* This code is mostly copied from ZSTD_compressSequences in zstd_compress.c */ + unsigned count[MaxSeq+1]; + S16 norm[MaxSeq+1]; + FSE_CTable* CTable_LitLength = frame->stats.litlengthCTable; + FSE_CTable* CTable_OffsetBits = frame->stats.offcodeCTable; + FSE_CTable* CTable_MatchLength = frame->stats.matchlengthCTable; + U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ + const seqDef* const sequences = seqStorePtr->sequencesStart; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const oend = (BYTE*)frame->dataEnd; + BYTE* op = (BYTE*)frame->data; + BYTE* seqHead; + BYTE scratchBuffer[1<<MAX(MLFSELog,LLFSELog)]; + + /* literals compressing block removed so that can be done separately */ + + /* Sequences Header */ + if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall); + if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + + if (nbSeq==0) { + frame->data = op; + return 0; + } + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + + /* convert length/distances into codes */ + ZSTD_seqToCodes(seqStorePtr); + + /* CTable for Literal Lengths */ + { unsigned max = MaxLL; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, llCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */ + assert(!HIST_isError(mostFrequent)); + if (frame->stats.fseInit && !(RAND(seed) & 3) && + isSymbolSubset(llCodeTable, nbSeq, + frame->stats.litlengthSymbolSet, 35)) { + /* maybe do repeat mode if we're allowed to */ + LLtype = set_repeat; + } else if (mostFrequent == nbSeq) { + /* do RLE if we have the chance */ + *op++ = llCodeTable[0]; + FSE_buildCTable_rle(CTable_LitLength, (BYTE)max); + LLtype = set_rle; + } else if (!(RAND(seed) & 3)) { + /* maybe use the default distribution */ + FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); + LLtype = set_basic; + } else { + /* fall back on a full table */ + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); + if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; } + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return ERROR(GENERIC); + op += NCountSize; } + FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); + LLtype = set_compressed; + } } + + /* CTable for Offsets */ + /* see Literal Lengths for descriptions of mode choices */ + { unsigned max = MaxOff; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, ofCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */ + assert(!HIST_isError(mostFrequent)); + if (frame->stats.fseInit && !(RAND(seed) & 3) && + isSymbolSubset(ofCodeTable, nbSeq, + frame->stats.offsetSymbolSet, 28)) { + Offtype = set_repeat; + } else if (mostFrequent == nbSeq) { + *op++ = ofCodeTable[0]; + FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max); + Offtype = set_rle; + } else if (!(RAND(seed) & 3)) { + FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, DefaultMaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); + Offtype = set_basic; + } else { + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); + if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; } + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return ERROR(GENERIC); + op += NCountSize; } + FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); + Offtype = set_compressed; + } } + + /* CTable for MatchLengths */ + /* see Literal Lengths for descriptions of mode choices */ + { unsigned max = MaxML; + size_t const mostFrequent = HIST_countFast_wksp(count, &max, mlCodeTable, nbSeq, WKSP, sizeof(WKSP)); /* cannot fail */ + assert(!HIST_isError(mostFrequent)); + if (frame->stats.fseInit && !(RAND(seed) & 3) && + isSymbolSubset(mlCodeTable, nbSeq, + frame->stats.matchlengthSymbolSet, 52)) { + MLtype = set_repeat; + } else if (mostFrequent == nbSeq) { + *op++ = *mlCodeTable; + FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max); + MLtype = set_rle; + } else if (!(RAND(seed) & 3)) { + /* sometimes do default distribution */ + FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); + MLtype = set_basic; + } else { + /* fall back on table */ + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); + if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; } + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return ERROR(GENERIC); + op += NCountSize; } + FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); + MLtype = set_compressed; + } } + frame->stats.fseInit = 1; + initSymbolSet(llCodeTable, nbSeq, frame->stats.litlengthSymbolSet, 35); + initSymbolSet(ofCodeTable, nbSeq, frame->stats.offsetSymbolSet, 28); + initSymbolSet(mlCodeTable, nbSeq, frame->stats.matchlengthSymbolSet, 52); + + DISPLAYLEVEL(5, " LL type: %d OF type: %d ML type: %d\n", (unsigned)LLtype, (unsigned)Offtype, (unsigned)MLtype); + + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + + /* Encoding Sequences */ + { BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + RETURN_ERROR_IF( + ERR_isError(BIT_initCStream(&blockStream, op, oend-op)), + dstSize_tooSmall, "not enough space remaining"); + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n<nbSeq ; n--) { /* intentional underflow */ + BYTE const llCode = llCodeTable[n]; + BYTE const ofCode = ofCodeTable[n]; + BYTE const mlCode = mlCodeTable[n]; + U32 const llBits = LL_bits[llCode]; + U32 const ofBits = ofCode; /* 32b*/ /* 64b*/ + U32 const mlBits = ML_bits[mlCode]; + /* (7)*/ /* (7)*/ + FSE_encodeSymbol(&blockStream, &stateOffsetBits, ofCode); /* 15 */ /* 15 */ + FSE_encodeSymbol(&blockStream, &stateMatchLength, mlCode); /* 24 */ /* 24 */ + if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/ + FSE_encodeSymbol(&blockStream, &stateLitLength, llCode); /* 16 */ /* 33 */ + if (MEM_32bits() || (ofBits+mlBits+llBits >= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + BIT_flushBits(&blockStream); /* (7)*/ + } } + + FSE_flushCState(&blockStream, &stateMatchLength); + FSE_flushCState(&blockStream, &stateOffsetBits); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ + op += streamSize; + } } + + frame->data = op; + + return 0; +} + +static size_t writeSequencesBlock(U32* seed, frame_t* frame, size_t contentSize, + size_t literalsSize, dictInfo info) +{ + seqStore_t seqStore; + size_t numSequences; + + + initSeqStore(&seqStore); + + /* randomly generate sequences */ + numSequences = generateSequences(seed, frame, &seqStore, contentSize, literalsSize, info); + /* write them out to the frame data */ + CHECKERR(writeSequences(seed, frame, &seqStore, numSequences)); + + return numSequences; +} + +static size_t writeCompressedBlock(U32* seed, frame_t* frame, size_t contentSize, dictInfo info) +{ + BYTE* const blockStart = (BYTE*)frame->data; + size_t literalsSize; + size_t nbSeq; + + DISPLAYLEVEL(4, " compressed block:\n"); + + literalsSize = writeLiteralsBlock(seed, frame, contentSize); + + DISPLAYLEVEL(4, " literals size: %u\n", (unsigned)literalsSize); + + nbSeq = writeSequencesBlock(seed, frame, contentSize, literalsSize, info); + + DISPLAYLEVEL(4, " number of sequences: %u\n", (unsigned)nbSeq); + + return (BYTE*)frame->data - blockStart; +} + +static void writeBlock(U32* seed, frame_t* frame, size_t contentSize, + int lastBlock, dictInfo info) +{ + int const blockTypeDesc = RAND(seed) % 8; + size_t blockSize; + int blockType; + + BYTE *const header = (BYTE*)frame->data; + BYTE *op = header + 3; + + DISPLAYLEVEL(4, " block:\n"); + DISPLAYLEVEL(4, " block content size: %u\n", (unsigned)contentSize); + DISPLAYLEVEL(4, " last block: %s\n", lastBlock ? "yes" : "no"); + + if (blockTypeDesc == 0) { + /* Raw data frame */ + + RAND_buffer(seed, frame->src, contentSize); + memcpy(op, frame->src, contentSize); + + op += contentSize; + blockType = 0; + blockSize = contentSize; + } else if (blockTypeDesc == 1 && frame->header.contentSize > 0) { + /* RLE (Don't create RLE block if frame content is 0 since block size of 1 may exceed max block size)*/ + BYTE const symbol = RAND(seed) & 0xff; + + op[0] = symbol; + memset(frame->src, symbol, contentSize); + + op++; + blockType = 1; + blockSize = contentSize; + } else { + /* compressed, most common */ + size_t compressedSize; + blockType = 2; + + frame->oldStats = frame->stats; + + frame->data = op; + compressedSize = writeCompressedBlock(seed, frame, contentSize, info); + if (compressedSize >= contentSize) { /* compressed block must be strictly smaller than uncompressed one */ + blockType = 0; + memcpy(op, frame->src, contentSize); + + op += contentSize; + blockSize = contentSize; /* fall back on raw block if data doesn't + compress */ + + frame->stats = frame->oldStats; /* don't update the stats */ + } else { + op += compressedSize; + blockSize = compressedSize; + } + } + frame->src = (BYTE*)frame->src + contentSize; + + DISPLAYLEVEL(4, " block type: %s\n", BLOCK_TYPES[blockType]); + DISPLAYLEVEL(4, " block size field: %u\n", (unsigned)blockSize); + + header[0] = (BYTE) ((lastBlock | (blockType << 1) | (blockSize << 3)) & 0xff); + MEM_writeLE16(header + 1, (U16) (blockSize >> 5)); + + frame->data = op; +} + +static void writeBlocks(U32* seed, frame_t* frame, dictInfo info) +{ + size_t contentLeft = frame->header.contentSize; + size_t const maxBlockSize = MIN(g_maxBlockSize, frame->header.windowSize); + while (1) { + /* 1 in 4 chance of ending frame */ + int const lastBlock = contentLeft > maxBlockSize ? 0 : !(RAND(seed) & 3); + size_t blockContentSize; + if (lastBlock) { + blockContentSize = contentLeft; + } else { + if (contentLeft > 0 && (RAND(seed) & 7)) { + /* some variable size block */ + blockContentSize = RAND(seed) % (MIN(maxBlockSize, contentLeft)+1); + } else if (contentLeft > maxBlockSize && (RAND(seed) & 1)) { + /* some full size block */ + blockContentSize = maxBlockSize; + } else { + /* some empty block */ + blockContentSize = 0; + } + } + + writeBlock(seed, frame, blockContentSize, lastBlock, info); + + contentLeft -= blockContentSize; + if (lastBlock) break; + } +} + +static void writeChecksum(frame_t* frame) +{ + /* write checksum so implementations can verify their output */ + U64 digest = XXH64(frame->srcStart, (BYTE*)frame->src-(BYTE*)frame->srcStart, 0); + DISPLAYLEVEL(3, " checksum: %08x\n", (unsigned)digest); + MEM_writeLE32(frame->data, (U32)digest); + frame->data = (BYTE*)frame->data + 4; +} + +static void outputBuffer(const void* buf, size_t size, const char* const path) +{ + /* write data out to file */ + const BYTE* ip = (const BYTE*)buf; + FILE* out; + if (path) { + out = fopen(path, "wb"); + } else { + out = stdout; + } + if (!out) { + fprintf(stderr, "Failed to open file at %s: ", path); + perror(NULL); + exit(1); + } + + { size_t fsize = size; + size_t written = 0; + while (written < fsize) { + written += fwrite(ip + written, 1, fsize - written, out); + if (ferror(out)) { + fprintf(stderr, "Failed to write to file at %s: ", path); + perror(NULL); + exit(1); + } + } + } + + if (path) { + fclose(out); + } +} + +static void initFrame(frame_t* fr) +{ + memset(fr, 0, sizeof(*fr)); + fr->data = fr->dataStart = FRAME_BUFFER; + fr->dataEnd = FRAME_BUFFER + sizeof(FRAME_BUFFER); + fr->src = fr->srcStart = CONTENT_BUFFER; + fr->srcEnd = CONTENT_BUFFER + sizeof(CONTENT_BUFFER); + + /* init repeat codes */ + fr->stats.rep[0] = 1; + fr->stats.rep[1] = 4; + fr->stats.rep[2] = 8; +} + +/** + * Generated a single zstd compressed block with no block/frame header. + * Returns the final seed. + */ +static U32 generateCompressedBlock(U32 seed, frame_t* frame, dictInfo info) +{ + size_t blockContentSize; + int blockWritten = 0; + BYTE* op; + DISPLAYLEVEL(4, "block seed: %u\n", (unsigned)seed); + initFrame(frame); + op = (BYTE*)frame->data; + + while (!blockWritten) { + size_t cSize; + /* generate window size */ + { int const exponent = RAND(&seed) % (MAX_WINDOW_LOG - 10); + int const mantissa = RAND(&seed) % 8; + frame->header.windowSize = (1U << (exponent + 10)); + frame->header.windowSize += (frame->header.windowSize / 8) * mantissa; + } + + /* generate content size */ + { size_t const maxBlockSize = MIN(g_maxBlockSize, frame->header.windowSize); + if (RAND(&seed) & 15) { + /* some full size blocks */ + blockContentSize = maxBlockSize; + } else if (RAND(&seed) & 7 && g_maxBlockSize >= (1U << 7)) { + /* some small blocks <= 128 bytes*/ + blockContentSize = RAND(&seed) % (1U << 7); + } else { + /* some variable size blocks */ + blockContentSize = RAND(&seed) % maxBlockSize; + } + } + + /* try generating a compressed block */ + frame->oldStats = frame->stats; + frame->data = op; + cSize = writeCompressedBlock(&seed, frame, blockContentSize, info); + if (cSize >= blockContentSize) { /* compressed size must be strictly smaller than decompressed size : https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#blocks */ + /* data doesn't compress -- try again */ + frame->stats = frame->oldStats; /* don't update the stats */ + DISPLAYLEVEL(5, " can't compress block : try again \n"); + } else { + blockWritten = 1; + DISPLAYLEVEL(4, " block size: %u \n", (unsigned)cSize); + frame->src = (BYTE*)frame->src + blockContentSize; + } + } + return seed; +} + +/* Return the final seed */ +static U32 generateFrame(U32 seed, frame_t* fr, dictInfo info) +{ + /* generate a complete frame */ + DISPLAYLEVEL(3, "frame seed: %u\n", (unsigned)seed); + initFrame(fr); + + writeFrameHeader(&seed, fr, info); + writeBlocks(&seed, fr, info); + writeChecksum(fr); + + return seed; +} + +/*_******************************************************* +* Dictionary Helper Functions +*********************************************************/ +/* returns 0 if successful, otherwise returns 1 upon error */ +static int genRandomDict(U32 dictID, U32 seed, size_t dictSize, BYTE* fullDict) +{ + /* allocate space for samples */ + int ret = 0; + unsigned const numSamples = 4; + size_t sampleSizes[4]; + BYTE* const samples = malloc(5000*sizeof(BYTE)); + if (samples == NULL) { + DISPLAY("Error: could not allocate space for samples\n"); + return 1; + } + + /* generate samples */ + { unsigned literalValue = 1; + unsigned samplesPos = 0; + size_t currSize = 1; + while (literalValue <= 4) { + sampleSizes[literalValue - 1] = currSize; + { size_t k; + for (k = 0; k < currSize; k++) { + *(samples + (samplesPos++)) = (BYTE)literalValue; + } } + literalValue++; + currSize *= 16; + } } + + { size_t dictWriteSize = 0; + ZDICT_params_t zdictParams; + size_t const headerSize = MAX(dictSize/4, 256); + size_t const dictContentSize = dictSize - headerSize; + BYTE* const dictContent = fullDict + headerSize; + if (dictContentSize < ZDICT_CONTENTSIZE_MIN || dictSize < ZDICT_DICTSIZE_MIN) { + DISPLAY("Error: dictionary size is too small\n"); + ret = 1; + goto exitGenRandomDict; + } + + /* init dictionary params */ + memset(&zdictParams, 0, sizeof(zdictParams)); + zdictParams.dictID = dictID; + zdictParams.notificationLevel = 1; + + /* fill in dictionary content */ + RAND_buffer(&seed, (void*)dictContent, dictContentSize); + + /* finalize dictionary with random samples */ + dictWriteSize = ZDICT_finalizeDictionary(fullDict, dictSize, + dictContent, dictContentSize, + samples, sampleSizes, numSamples, + zdictParams); + + if (ZDICT_isError(dictWriteSize)) { + DISPLAY("Could not finalize dictionary: %s\n", ZDICT_getErrorName(dictWriteSize)); + ret = 1; + } + } + +exitGenRandomDict: + free(samples); + return ret; +} + +static dictInfo initDictInfo(int useDict, size_t dictContentSize, BYTE* dictContent, U32 dictID){ + /* allocate space statically */ + dictInfo dictOp; + memset(&dictOp, 0, sizeof(dictOp)); + dictOp.useDict = useDict; + dictOp.dictContentSize = dictContentSize; + dictOp.dictContent = dictContent; + dictOp.dictID = dictID; + return dictOp; +} + +/*-******************************************************* +* Test Mode +*********************************************************/ + +BYTE DECOMPRESSED_BUFFER[MAX_DECOMPRESSED_SIZE]; + +static size_t testDecodeSimple(frame_t* fr) +{ + /* test decoding the generated data with the simple API */ + size_t const ret = ZSTD_decompress(DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, + fr->dataStart, (BYTE*)fr->data - (BYTE*)fr->dataStart); + + if (ZSTD_isError(ret)) return ret; + + if (memcmp(DECOMPRESSED_BUFFER, fr->srcStart, + (BYTE*)fr->src - (BYTE*)fr->srcStart) != 0) { + return ERROR(corruption_detected); + } + + return ret; +} + +static size_t testDecodeStreaming(frame_t* fr) +{ + /* test decoding the generated data with the streaming API */ + ZSTD_DStream* zd = ZSTD_createDStream(); + ZSTD_inBuffer in; + ZSTD_outBuffer out; + size_t ret; + + if (!zd) return ERROR(memory_allocation); + + in.src = fr->dataStart; + in.pos = 0; + in.size = (BYTE*)fr->data - (BYTE*)fr->dataStart; + + out.dst = DECOMPRESSED_BUFFER; + out.pos = 0; + out.size = ZSTD_DStreamOutSize(); + + ZSTD_initDStream(zd); + while (1) { + ret = ZSTD_decompressStream(zd, &out, &in); + if (ZSTD_isError(ret)) goto cleanup; /* error */ + if (ret == 0) break; /* frame is done */ + + /* force decoding to be done in chunks */ + out.size += MIN(ZSTD_DStreamOutSize(), MAX_DECOMPRESSED_SIZE - out.size); + } + + ret = out.pos; + + if (memcmp(out.dst, fr->srcStart, out.pos) != 0) { + return ERROR(corruption_detected); + } + +cleanup: + ZSTD_freeDStream(zd); + return ret; +} + +static size_t testDecodeWithDict(U32 seed, genType_e genType) +{ + /* create variables */ + size_t const dictSize = RAND(&seed) % (10 << 20) + ZDICT_DICTSIZE_MIN + ZDICT_CONTENTSIZE_MIN; + U32 const dictID = RAND(&seed); + size_t errorDetected = 0; + BYTE* const fullDict = malloc(dictSize); + if (fullDict == NULL) { + return ERROR(GENERIC); + } + + /* generate random dictionary */ + if (genRandomDict(dictID, seed, dictSize, fullDict)) { /* return 0 on success */ + errorDetected = ERROR(GENERIC); + goto dictTestCleanup; + } + + + { frame_t fr; + dictInfo info; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t ret; + + /* get dict info */ + { size_t const headerSize = MAX(dictSize/4, 256); + size_t const dictContentSize = dictSize-headerSize; + BYTE* const dictContent = fullDict+headerSize; + info = initDictInfo(1, dictContentSize, dictContent, dictID); + } + + /* manually decompress and check difference */ + if (genType == gt_frame) { + /* Test frame */ + generateFrame(seed, &fr, info); + ret = ZSTD_decompress_usingDict(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, + fullDict, dictSize); + } else { + /* Test block */ + generateCompressedBlock(seed, &fr, info); + ret = ZSTD_decompressBegin_usingDict(dctx, fullDict, dictSize); + if (ZSTD_isError(ret)) { + errorDetected = ret; + ZSTD_freeDCtx(dctx); + goto dictTestCleanup; + } + ret = ZSTD_decompressBlock(dctx, DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, + fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart); + } + ZSTD_freeDCtx(dctx); + + if (ZSTD_isError(ret)) { + errorDetected = ret; + goto dictTestCleanup; + } + + if (memcmp(DECOMPRESSED_BUFFER, fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart) != 0) { + errorDetected = ERROR(corruption_detected); + goto dictTestCleanup; + } + } + +dictTestCleanup: + free(fullDict); + return errorDetected; +} + +static size_t testDecodeRawBlock(frame_t* fr) +{ + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t ret = ZSTD_decompressBegin(dctx); + if (ZSTD_isError(ret)) return ret; + + ret = ZSTD_decompressBlock( + dctx, + DECOMPRESSED_BUFFER, MAX_DECOMPRESSED_SIZE, + fr->dataStart, (BYTE*)fr->data - (BYTE*)fr->dataStart); + ZSTD_freeDCtx(dctx); + if (ZSTD_isError(ret)) return ret; + + if (memcmp(DECOMPRESSED_BUFFER, fr->srcStart, + (BYTE*)fr->src - (BYTE*)fr->srcStart) != 0) { + return ERROR(corruption_detected); + } + + return ret; +} + +static int runBlockTest(U32* seed) +{ + frame_t fr; + U32 const seedCopy = *seed; + { dictInfo const info = initDictInfo(0, 0, NULL, 0); + *seed = generateCompressedBlock(*seed, &fr, info); + } + + { size_t const r = testDecodeRawBlock(&fr); + if (ZSTD_isError(r)) { + DISPLAY("Error in block mode on test seed %u: %s\n", + (unsigned)seedCopy, ZSTD_getErrorName(r)); + return 1; + } + } + + { size_t const r = testDecodeWithDict(*seed, gt_block); + if (ZSTD_isError(r)) { + DISPLAY("Error in block mode with dictionary on test seed %u: %s\n", + (unsigned)seedCopy, ZSTD_getErrorName(r)); + return 1; + } + } + return 0; +} + +static int runFrameTest(U32* seed) +{ + frame_t fr; + U32 const seedCopy = *seed; + { dictInfo const info = initDictInfo(0, 0, NULL, 0); + *seed = generateFrame(*seed, &fr, info); + } + + { size_t const r = testDecodeSimple(&fr); + if (ZSTD_isError(r)) { + DISPLAY("Error in simple mode on test seed %u: %s\n", + (unsigned)seedCopy, ZSTD_getErrorName(r)); + return 1; + } + } + { size_t const r = testDecodeStreaming(&fr); + if (ZSTD_isError(r)) { + DISPLAY("Error in streaming mode on test seed %u: %s\n", + (unsigned)seedCopy, ZSTD_getErrorName(r)); + return 1; + } + } + { size_t const r = testDecodeWithDict(*seed, gt_frame); /* avoid big dictionaries */ + if (ZSTD_isError(r)) { + DISPLAY("Error in dictionary mode on test seed %u: %s\n", + (unsigned)seedCopy, ZSTD_getErrorName(r)); + return 1; + } + } + return 0; +} + +static int runTestMode(U32 seed, unsigned numFiles, unsigned const testDurationS, + genType_e genType) +{ + unsigned fnum; + + UTIL_time_t const startClock = UTIL_getTime(); + U64 const maxClockSpan = testDurationS * SEC_TO_MICRO; + + if (numFiles == 0 && !testDurationS) numFiles = 1; + + DISPLAY("seed: %u\n", (unsigned)seed); + + for (fnum = 0; fnum < numFiles || UTIL_clockSpanMicro(startClock) < maxClockSpan; fnum++) { + if (fnum < numFiles) + DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); + else + DISPLAYUPDATE("\r%u ", fnum); + + { int const ret = (genType == gt_frame) ? + runFrameTest(&seed) : + runBlockTest(&seed); + if (ret) return ret; + } + } + + DISPLAY("\r%u tests completed: ", fnum); + DISPLAY("OK\n"); + + return 0; +} + +/*-******************************************************* +* File I/O +*********************************************************/ + +static int generateFile(U32 seed, const char* const path, + const char* const origPath, genType_e genType) +{ + frame_t fr; + + DISPLAY("seed: %u\n", (unsigned)seed); + + { dictInfo const info = initDictInfo(0, 0, NULL, 0); + if (genType == gt_frame) { + generateFrame(seed, &fr, info); + } else { + generateCompressedBlock(seed, &fr, info); + } + } + outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); + if (origPath) { + outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, origPath); + } + return 0; +} + +static int generateCorpus(U32 seed, unsigned numFiles, const char* const path, + const char* const origPath, genType_e genType) +{ + char outPath[MAX_PATH]; + unsigned fnum; + + DISPLAY("seed: %u\n", (unsigned)seed); + + for (fnum = 0; fnum < numFiles; fnum++) { + frame_t fr; + + DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); + + { dictInfo const info = initDictInfo(0, 0, NULL, 0); + if (genType == gt_frame) { + seed = generateFrame(seed, &fr, info); + } else { + seed = generateCompressedBlock(seed, &fr, info); + } + } + + if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + return 1; + } + outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); + + if (origPath) { + if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + return 1; + } + outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); + } + } + + DISPLAY("\r%u/%u \n", fnum, numFiles); + + return 0; +} + +static int generateCorpusWithDict(U32 seed, unsigned numFiles, const char* const path, + const char* const origPath, const size_t dictSize, + genType_e genType) +{ + char outPath[MAX_PATH]; + BYTE* fullDict; + U32 const dictID = RAND(&seed); + int errorDetected = 0; + + if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + return 1; + } + + /* allocate space for the dictionary */ + fullDict = malloc(dictSize); + if (fullDict == NULL) { + DISPLAY("Error: could not allocate space for full dictionary.\n"); + return 1; + } + + /* randomly generate the dictionary */ + { int const ret = genRandomDict(dictID, seed, dictSize, fullDict); + if (ret != 0) { + errorDetected = ret; + goto dictCleanup; + } + } + + /* write out dictionary */ + if (numFiles != 0) { + if (snprintf(outPath, MAX_PATH, "%s/dictionary", path) + 1 > MAX_PATH) { + DISPLAY("Error: dictionary path too long\n"); + errorDetected = 1; + goto dictCleanup; + } + outputBuffer(fullDict, dictSize, outPath); + } + else { + outputBuffer(fullDict, dictSize, "dictionary"); + } + + /* generate random compressed/decompressed files */ + { unsigned fnum; + for (fnum = 0; fnum < MAX(numFiles, 1); fnum++) { + frame_t fr; + DISPLAYUPDATE("\r%u/%u ", fnum, numFiles); + { + size_t const headerSize = MAX(dictSize/4, 256); + size_t const dictContentSize = dictSize-headerSize; + BYTE* const dictContent = fullDict+headerSize; + dictInfo const info = initDictInfo(1, dictContentSize, dictContent, dictID); + if (genType == gt_frame) { + seed = generateFrame(seed, &fr, info); + } else { + seed = generateCompressedBlock(seed, &fr, info); + } + } + + if (numFiles != 0) { + if (snprintf(outPath, MAX_PATH, "%s/z%06u.zst", path, fnum) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + errorDetected = 1; + goto dictCleanup; + } + outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, outPath); + + if (origPath) { + if (snprintf(outPath, MAX_PATH, "%s/z%06u", origPath, fnum) + 1 > MAX_PATH) { + DISPLAY("Error: path too long\n"); + errorDetected = 1; + goto dictCleanup; + } + outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, outPath); + } + } + else { + outputBuffer(fr.dataStart, (BYTE*)fr.data - (BYTE*)fr.dataStart, path); + if (origPath) { + outputBuffer(fr.srcStart, (BYTE*)fr.src - (BYTE*)fr.srcStart, origPath); + } + } + } + } + +dictCleanup: + free(fullDict); + return errorDetected; +} + + +/*_******************************************************* +* Command line +*********************************************************/ +static U32 makeSeed(void) +{ + U32 t = (U32) time(NULL); + return XXH32(&t, sizeof(t), 0) % 65536; +} + +static unsigned readInt(const char** argument) +{ + unsigned val = 0; + while ((**argument>='0') && (**argument<='9')) { + val *= 10; + val += **argument - '0'; + (*argument)++; + } + return val; +} + +static void usage(const char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -p<path> : select output path (default:stdout)\n"); + DISPLAY( " in multiple files mode this should be a directory\n"); + DISPLAY( " -o<path> : select path to output original file (default:no output)\n"); + DISPLAY( " in multiple files mode this should be a directory\n"); + DISPLAY( " -s# : select seed (default:random based on time)\n"); + DISPLAY( " -n# : number of files to generate (default:1)\n"); + DISPLAY( " -t : activate test mode (test files against libzstd instead of outputting them)\n"); + DISPLAY( " -T# : length of time to run tests for\n"); + DISPLAY( " -v : increase verbosity level (default:0, max:7)\n"); + DISPLAY( " -h/H : display help/long help and exit\n"); +} + +static void advancedUsage(const char* programName) +{ + usage(programName); + DISPLAY( "\n"); + DISPLAY( "Advanced arguments :\n"); + DISPLAY( " --content-size : always include the content size in the frame header\n"); + DISPLAY( " --use-dict=# : include a dictionary used to decompress the corpus\n"); + DISPLAY( " --gen-blocks : generate raw compressed blocks without block/frame headers\n"); + DISPLAY( " --max-block-size-log=# : max block size log, must be in range [2, 17]\n"); + DISPLAY( " --max-content-size-log=# : max content size log, must be <= 20\n"); + DISPLAY( " (this is ignored with gen-blocks)\n"); +} + +/*! readU32FromChar() : + @return : unsigned integer value read from input in `char` format + allows and interprets K, KB, KiB, M, MB and MiB suffix. + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : function result can overflow if digit string > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + */ +static unsigned longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + +int main(int argc, char** argv) +{ + U32 seed = 0; + int seedset = 0; + unsigned numFiles = 0; + unsigned testDuration = 0; + int testMode = 0; + const char* path = NULL; + const char* origPath = NULL; + int useDict = 0; + unsigned dictSize = (10 << 10); /* 10 kB default */ + genType_e genType = gt_frame; + + int argNb; + + /* Check command line */ + for (argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + if(!argument) continue; /* Protection if argument empty */ + + /* Handle commands. Aggregated commands are allowed */ + if (argument[0]=='-') { + argument++; + while (*argument!=0) { + switch(*argument) + { + case 'h': + usage(argv[0]); + return 0; + case 'H': + advancedUsage(argv[0]); + return 0; + case 'v': + argument++; + g_displayLevel++; + break; + case 's': + argument++; + seedset=1; + seed = readInt(&argument); + break; + case 'n': + argument++; + numFiles = readInt(&argument); + break; + case 'T': + argument++; + testDuration = readInt(&argument); + if (*argument == 'm') { + testDuration *= 60; + argument++; + if (*argument == 'n') argument++; + } + break; + case 'o': + argument++; + origPath = argument; + argument += strlen(argument); + break; + case 'p': + argument++; + path = argument; + argument += strlen(argument); + break; + case 't': + argument++; + testMode = 1; + break; + case '-': + argument++; + if (strcmp(argument, "content-size") == 0) { + opts.contentSize = 1; + } else if (longCommandWArg(&argument, "use-dict=")) { + dictSize = readU32FromChar(&argument); + useDict = 1; + } else if (strcmp(argument, "gen-blocks") == 0) { + genType = gt_block; + } else if (longCommandWArg(&argument, "max-block-size-log=")) { + U32 value = readU32FromChar(&argument); + if (value >= 2 && value <= ZSTD_BLOCKSIZE_MAX) { + g_maxBlockSize = 1U << value; + } + } else if (longCommandWArg(&argument, "max-content-size-log=")) { + U32 value = readU32FromChar(&argument); + g_maxDecompressedSizeLog = + MIN(MAX_DECOMPRESSED_SIZE_LOG, value); + } else { + advancedUsage(argv[0]); + return 1; + } + argument += strlen(argument); + break; + default: + usage(argv[0]); + return 1; + } } } } /* for (argNb=1; argNb<argc; argNb++) */ + + if (!seedset) { + seed = makeSeed(); + } + + if (testMode) { + return runTestMode(seed, numFiles, testDuration, genType); + } else { + if (testDuration) { + DISPLAY("Error: -T requires test mode (-t)\n\n"); + usage(argv[0]); + return 1; + } + } + + if (!path) { + DISPLAY("Error: path is required in file generation mode\n"); + usage(argv[0]); + return 1; + } + + if (numFiles == 0 && useDict == 0) { + return generateFile(seed, path, origPath, genType); + } else if (useDict == 0){ + return generateCorpus(seed, numFiles, path, origPath, genType); + } else { + /* should generate files with a dictionary */ + return generateCorpusWithDict(seed, numFiles, path, origPath, dictSize, genType); + } + +} diff --git a/src/zstd/tests/dict-files/zero-weight-dict b/src/zstd/tests/dict-files/zero-weight-dict Binary files differnew file mode 100644 index 000000000..c40412052 --- /dev/null +++ b/src/zstd/tests/dict-files/zero-weight-dict diff --git a/src/zstd/tests/fullbench.c b/src/zstd/tests/fullbench.c new file mode 100644 index 000000000..f0179a9d1 --- /dev/null +++ b/src/zstd/tests/fullbench.c @@ -0,0 +1,843 @@ +/* + * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*_************************************ +* Includes +**************************************/ +#include "util.h" /* Compiler options, UTIL_GetFileSize */ +#include <stdlib.h> /* malloc */ +#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <assert.h> + +#include "timefn.h" /* UTIL_clockSpanNano, UTIL_getTime */ +#include "mem.h" /* U32 */ +#ifndef ZSTD_DLL_IMPORT + #include "zstd_internal.h" /* ZSTD_decodeSeqHeaders, ZSTD_blockHeaderSize, ZSTD_getcBlockSize, blockType_e, KB, MB */ +#else + #define KB *(1 <<10) + #define MB *(1 <<20) + #define GB *(1U<<30) + typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; +#endif +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressBegin, ZSTD_compressContinue, etc. */ +#include "zstd.h" /* ZSTD_versionString */ +#include "util.h" /* time functions */ +#include "datagen.h" +#include "benchfn.h" /* CustomBench */ +#include "benchzstd.h" /* MB_UNIT */ + + +/*_************************************ +* Constants +**************************************/ +#define PROGRAM_DESCRIPTION "Zstandard speed analyzer" +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s (%s) ***\n", PROGRAM_DESCRIPTION, ZSTD_versionString(), (int)(sizeof(void*)*8), AUTHOR, __DATE__ + +#define NBLOOPS 6 +#define TIMELOOP_S 2 + +#define MAX_MEM (1984 MB) + +#define DEFAULT_CLEVEL 1 + +#define COMPRESSIBILITY_DEFAULT 0.50 +static const size_t kSampleSizeDefault = 10000000; + +#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ + + +/*_************************************ +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + +#define CONTROL(c) { if (!(c)) { abort(); } } /* like assert(), but cannot be disabled */ + +/*_************************************ +* Benchmark Parameters +**************************************/ +static unsigned g_nbIterations = NBLOOPS; + + +/*_******************************************************* +* Private functions +*********************************************************/ +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t const step = 64 MB; + void* testmem = NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; + + requiredMem += step; + do { + testmem = malloc ((size_t)requiredMem); + requiredMem -= step; + } while (!testmem); + + free (testmem); + return (size_t) requiredMem; +} + + +/*_******************************************************* +* Benchmark wrappers +*********************************************************/ + +static ZSTD_CCtx* g_zcc = NULL; + +static size_t +local_ZSTD_compress(const void* src, size_t srcSize, + void* dst, size_t dstSize, + void* payload) +{ + ZSTD_parameters p; + ZSTD_frameParameters f = { 1 /* contentSizeHeader*/, 0, 0 }; + p.fParams = f; + p.cParams = *(ZSTD_compressionParameters*)payload; + return ZSTD_compress_advanced (g_zcc, dst, dstSize, src, srcSize, NULL ,0, p); + //return ZSTD_compress(dst, dstSize, src, srcSize, cLevel); +} + +static size_t g_cSize = 0; +static size_t local_ZSTD_decompress(const void* src, size_t srcSize, + void* dst, size_t dstSize, + void* buff2) +{ + (void)src; (void)srcSize; + return ZSTD_decompress(dst, dstSize, buff2, g_cSize); +} + +static ZSTD_DCtx* g_zdc = NULL; + +#ifndef ZSTD_DLL_IMPORT +extern size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* ctx, const void* src, size_t srcSize); +static size_t local_ZSTD_decodeLiteralsBlock(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) +{ + (void)src; (void)srcSize; (void)dst; (void)dstSize; + return ZSTD_decodeLiteralsBlock(g_zdc, buff2, g_cSize); +} + +static size_t local_ZSTD_decodeSeqHeaders(const void* src, size_t srcSize, void* dst, size_t dstSize, void* buff2) +{ + int nbSeq; + (void)src; (void)srcSize; (void)dst; (void)dstSize; + return ZSTD_decodeSeqHeaders(g_zdc, &nbSeq, buff2, g_cSize); +} +#endif + +static ZSTD_CStream* g_cstream= NULL; +static size_t +local_ZSTD_compressStream(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + ZSTD_outBuffer buffOut; + ZSTD_inBuffer buffIn; + ZSTD_parameters p; + ZSTD_frameParameters f = {1 /* contentSizeHeader*/, 0, 0}; + p.fParams = f; + p.cParams = *(ZSTD_compressionParameters*)payload; + ZSTD_initCStream_advanced(g_cstream, NULL, 0, p, ZSTD_CONTENTSIZE_UNKNOWN); + buffOut.dst = dst; + buffOut.size = dstCapacity; + buffOut.pos = 0; + buffIn.src = src; + buffIn.size = srcSize; + buffIn.pos = 0; + ZSTD_compressStream(g_cstream, &buffOut, &buffIn); + ZSTD_endStream(g_cstream, &buffOut); + return buffOut.pos; +} + +static size_t +local_ZSTD_compressStream_freshCCtx(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t r; + assert(cctx != NULL); + + r = local_ZSTD_compressStream(src, srcSize, dst, dstCapacity, payload); + + ZSTD_freeCCtx(cctx); + + return r; +} + +static size_t +local_ZSTD_compress_generic_end(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + (void)payload; + return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize); +} + +static size_t +local_ZSTD_compress_generic_continue(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + ZSTD_outBuffer buffOut; + ZSTD_inBuffer buffIn; + (void)payload; + buffOut.dst = dst; + buffOut.size = dstCapacity; + buffOut.pos = 0; + buffIn.src = src; + buffIn.size = srcSize; + buffIn.pos = 0; + ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue); + ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end); + return buffOut.pos; +} + +static size_t +local_ZSTD_compress_generic_T2_end(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + (void)payload; + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2); + return ZSTD_compress2(g_cstream, dst, dstCapacity, src, srcSize); +} + +static size_t +local_ZSTD_compress_generic_T2_continue(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + ZSTD_outBuffer buffOut; + ZSTD_inBuffer buffIn; + (void)payload; + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_nbWorkers, 2); + buffOut.dst = dst; + buffOut.size = dstCapacity; + buffOut.pos = 0; + buffIn.src = src; + buffIn.size = srcSize; + buffIn.pos = 0; + ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_continue); + while(ZSTD_compressStream2(g_cstream, &buffOut, &buffIn, ZSTD_e_end)) {} + return buffOut.pos; +} + +static ZSTD_DStream* g_dstream= NULL; +static size_t +local_ZSTD_decompressStream(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* buff2) +{ + ZSTD_outBuffer buffOut; + ZSTD_inBuffer buffIn; + (void)src; (void)srcSize; + ZSTD_initDStream(g_dstream); + buffOut.dst = dst; + buffOut.size = dstCapacity; + buffOut.pos = 0; + buffIn.src = buff2; + buffIn.size = g_cSize; + buffIn.pos = 0; + ZSTD_decompressStream(g_dstream, &buffOut, &buffIn); + return buffOut.pos; +} + +#ifndef ZSTD_DLL_IMPORT +static size_t local_ZSTD_compressContinue(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + ZSTD_parameters p; + ZSTD_frameParameters f = { 1 /* contentSizeHeader*/, 0, 0 }; + p.fParams = f; + p.cParams = *(ZSTD_compressionParameters*)payload; + ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize); + return ZSTD_compressEnd(g_zcc, dst, dstCapacity, src, srcSize); +} + +#define FIRST_BLOCK_SIZE 8 +static size_t +local_ZSTD_compressContinue_extDict(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* payload) +{ + BYTE firstBlockBuf[FIRST_BLOCK_SIZE]; + + ZSTD_parameters p; + ZSTD_frameParameters const f = { 1, 0, 0 }; + p.fParams = f; + p.cParams = *(ZSTD_compressionParameters*)payload; + ZSTD_compressBegin_advanced(g_zcc, NULL, 0, p, srcSize); + memcpy(firstBlockBuf, src, FIRST_BLOCK_SIZE); + + { size_t const compressResult = ZSTD_compressContinue(g_zcc, + dst, dstCapacity, + firstBlockBuf, FIRST_BLOCK_SIZE); + if (ZSTD_isError(compressResult)) { + DISPLAY("local_ZSTD_compressContinue_extDict error : %s\n", + ZSTD_getErrorName(compressResult)); + return compressResult; + } + dst = (BYTE*)dst + compressResult; + dstCapacity -= compressResult; + } + return ZSTD_compressEnd(g_zcc, dst, dstCapacity, + (const BYTE*)src + FIRST_BLOCK_SIZE, + srcSize - FIRST_BLOCK_SIZE); +} + +static size_t local_ZSTD_decompressContinue(const void* src, size_t srcSize, + void* dst, size_t dstCapacity, + void* buff2) +{ + size_t regeneratedSize = 0; + const BYTE* ip = (const BYTE*)buff2; + const BYTE* const iend = ip + g_cSize; + BYTE* op = (BYTE*)dst; + size_t remainingCapacity = dstCapacity; + + (void)src; (void)srcSize; /* unused */ + ZSTD_decompressBegin(g_zdc); + while (ip < iend) { + size_t const iSize = ZSTD_nextSrcSizeToDecompress(g_zdc); + size_t const decodedSize = ZSTD_decompressContinue(g_zdc, op, remainingCapacity, ip, iSize); + ip += iSize; + regeneratedSize += decodedSize; + op += decodedSize; + remainingCapacity -= decodedSize; + } + + return regeneratedSize; +} +#endif + + +/*_******************************************************* +* Bench functions +*********************************************************/ +static int benchMem(unsigned benchNb, + const void* src, size_t srcSize, + int cLevel, ZSTD_compressionParameters cparams) +{ + size_t dstBuffSize = ZSTD_compressBound(srcSize); + BYTE* dstBuff; + void* dstBuff2; + void* payload; + const char* benchName; + BMK_benchFn_t benchFunction; + int errorcode = 0; + + /* Selection */ + switch(benchNb) + { + case 1: + benchFunction = local_ZSTD_compress; benchName = "compress"; + break; + case 2: + benchFunction = local_ZSTD_decompress; benchName = "decompress"; + break; +#ifndef ZSTD_DLL_IMPORT + case 11: + benchFunction = local_ZSTD_compressContinue; benchName = "compressContinue"; + break; + case 12: + benchFunction = local_ZSTD_compressContinue_extDict; benchName = "compressContinue_extDict"; + break; + case 13: + benchFunction = local_ZSTD_decompressContinue; benchName = "decompressContinue"; + break; + case 31: + benchFunction = local_ZSTD_decodeLiteralsBlock; benchName = "decodeLiteralsBlock"; + break; + case 32: + benchFunction = local_ZSTD_decodeSeqHeaders; benchName = "decodeSeqHeaders"; + break; +#endif + case 41: + benchFunction = local_ZSTD_compressStream; benchName = "compressStream"; + break; + case 42: + benchFunction = local_ZSTD_decompressStream; benchName = "decompressStream"; + break; + case 43: + benchFunction = local_ZSTD_compressStream_freshCCtx; benchName = "compressStream_freshCCtx"; + break; + case 51: + benchFunction = local_ZSTD_compress_generic_continue; benchName = "compress_generic, continue"; + break; + case 52: + benchFunction = local_ZSTD_compress_generic_end; benchName = "compress_generic, end"; + break; + case 61: + benchFunction = local_ZSTD_compress_generic_T2_continue; benchName = "compress_generic, -T2, continue"; + break; + case 62: + benchFunction = local_ZSTD_compress_generic_T2_end; benchName = "compress_generic, -T2, end"; + break; + default : + return 0; + } + + /* Allocation */ + dstBuff = (BYTE*)malloc(dstBuffSize); + dstBuff2 = malloc(dstBuffSize); + if ((!dstBuff) || (!dstBuff2)) { + DISPLAY("\nError: not enough memory!\n"); + free(dstBuff); free(dstBuff2); + return 12; + } + payload = dstBuff2; + if (g_zcc==NULL) g_zcc = ZSTD_createCCtx(); + if (g_zdc==NULL) g_zdc = ZSTD_createDCtx(); + if (g_cstream==NULL) g_cstream = ZSTD_createCStream(); + if (g_dstream==NULL) g_dstream = ZSTD_createDStream(); + + /* DISPLAY("params: cLevel %d, wlog %d hlog %d clog %d slog %d mml %d tlen %d strat %d \n", + cLevel, cparams->windowLog, cparams->hashLog, cparams->chainLog, cparams->searchLog, + cparams->minMatch, cparams->targetLength, cparams->strategy); */ + + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_windowLog, (int)cparams.windowLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_hashLog, (int)cparams.hashLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_chainLog, (int)cparams.chainLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_searchLog, (int)cparams.searchLog); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_minMatch, (int)cparams.minMatch); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_targetLength, (int)cparams.targetLength); + ZSTD_CCtx_setParameter(g_zcc, ZSTD_c_strategy, cparams.strategy); + + + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_compressionLevel, cLevel); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_windowLog, (int)cparams.windowLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_hashLog, (int)cparams.hashLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_chainLog, (int)cparams.chainLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_searchLog, (int)cparams.searchLog); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_minMatch, (int)cparams.minMatch); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_targetLength, (int)cparams.targetLength); + ZSTD_CCtx_setParameter(g_cstream, ZSTD_c_strategy, cparams.strategy); + + /* Preparation */ + switch(benchNb) + { + case 1: + payload = &cparams; + break; + case 2: + g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel); + break; +#ifndef ZSTD_DLL_IMPORT + case 11: + payload = &cparams; + break; + case 12: + payload = &cparams; + break; + case 13 : + g_cSize = ZSTD_compress(dstBuff2, dstBuffSize, src, srcSize, cLevel); + break; + case 31: /* ZSTD_decodeLiteralsBlock : starts literals block in dstBuff2 */ + { size_t frameHeaderSize; + g_cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); + frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1)); + CONTROL(!ZSTD_isError(frameHeaderSize)); + /* check block is compressible, hence contains a literals section */ + { blockProperties_t bp; + ZSTD_getcBlockSize(dstBuff+frameHeaderSize, dstBuffSize, &bp); /* Get 1st block type */ + if (bp.blockType != bt_compressed) { + DISPLAY("ZSTD_decodeLiteralsBlock : impossible to test on this sample (not compressible)\n"); + goto _cleanOut; + } } + { size_t const skippedSize = frameHeaderSize + ZSTD_blockHeaderSize; + memcpy(dstBuff2, dstBuff+skippedSize, g_cSize-skippedSize); + } + srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ + ZSTD_decompressBegin(g_zdc); + break; + } + case 32: /* ZSTD_decodeSeqHeaders */ + { blockProperties_t bp; + const BYTE* ip = dstBuff; + const BYTE* iend; + { size_t const cSize = ZSTD_compress(dstBuff, dstBuffSize, src, srcSize, cLevel); + CONTROL(cSize > ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1)); + } + /* Skip frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize(dstBuff, ZSTD_FRAMEHEADERSIZE_PREFIX(ZSTD_f_zstd1)); + CONTROL(!ZSTD_isError(frameHeaderSize)); + ip += frameHeaderSize; + } + /* Find end of block */ + { size_t const cBlockSize = ZSTD_getcBlockSize(ip, dstBuffSize, &bp); /* Get 1st block type */ + if (bp.blockType != bt_compressed) { + DISPLAY("ZSTD_decodeSeqHeaders : impossible to test on this sample (not compressible)\n"); + goto _cleanOut; + } + iend = ip + ZSTD_blockHeaderSize + cBlockSize; /* End of first block */ + } + ip += ZSTD_blockHeaderSize; /* skip block header */ + ZSTD_decompressBegin(g_zdc); + CONTROL(iend > ip); + ip += ZSTD_decodeLiteralsBlock(g_zdc, ip, (size_t)(iend-ip)); /* skip literal segment */ + g_cSize = (size_t)(iend-ip); + memcpy(dstBuff2, ip, g_cSize); /* copy rest of block (it starts by SeqHeader) */ + srcSize = srcSize > 128 KB ? 128 KB : srcSize; /* speed relative to block */ + break; + } +#else + case 31: + goto _cleanOut; +#endif + case 41 : + payload = &cparams; + break; + case 42 : + g_cSize = ZSTD_compress(payload, dstBuffSize, src, srcSize, cLevel); + break; + case 43 : + payload = &cparams; + break; + + /* test functions */ + /* convention: test functions have ID > 100 */ + + default : ; + } + + /* warming up dstBuff */ + { size_t i; for (i=0; i<dstBuffSize; i++) dstBuff[i]=(BYTE)i; } + + /* benchmark loop */ + { BMK_timedFnState_t* const tfs = BMK_createTimedFnState(g_nbIterations * 1000, 1000); + void* const avoidStrictAliasingPtr = &dstBuff; + BMK_benchParams_t bp; + BMK_runTime_t bestResult; + bestResult.sumOfReturn = 0; + bestResult.nanoSecPerRun = (double)TIMELOOP_NANOSEC * 2000000000; /* hopefully large enough : must be larger than any potential measurement */ + CONTROL(tfs != NULL); + + bp.benchFn = benchFunction; + bp.benchPayload = payload; + bp.initFn = NULL; + bp.initPayload = NULL; + bp.errorFn = ZSTD_isError; + bp.blockCount = 1; + bp.srcBuffers = &src; + bp.srcSizes = &srcSize; + bp.dstBuffers = (void* const*) avoidStrictAliasingPtr; /* circumvent strict aliasing warning on gcc-8, + * because gcc considers that `void* const *` and `void**` are 2 different types */ + bp.dstCapacities = &dstBuffSize; + bp.blockResults = NULL; + + for (;;) { + BMK_runOutcome_t const bOutcome = BMK_benchTimedFn(tfs, bp); + + if (!BMK_isSuccessful_runOutcome(bOutcome)) { + DISPLAY("ERROR benchmarking function ! ! \n"); + errorcode = 1; + goto _cleanOut; + } + + { BMK_runTime_t const newResult = BMK_extract_runTime(bOutcome); + if (newResult.nanoSecPerRun < bestResult.nanoSecPerRun ) + bestResult.nanoSecPerRun = newResult.nanoSecPerRun; + DISPLAY("\r%2u#%-29.29s:%8.1f MB/s (%8u) ", + benchNb, benchName, + (double)srcSize * TIMELOOP_NANOSEC / bestResult.nanoSecPerRun / MB_UNIT, + (unsigned)newResult.sumOfReturn ); + } + + if ( BMK_isCompleted_TimedFn(tfs) ) break; + } + BMK_freeTimedFnState(tfs); + } + DISPLAY("\n"); + +_cleanOut: + free(dstBuff); + free(dstBuff2); + ZSTD_freeCCtx(g_zcc); g_zcc=NULL; + ZSTD_freeDCtx(g_zdc); g_zdc=NULL; + ZSTD_freeCStream(g_cstream); g_cstream=NULL; + ZSTD_freeDStream(g_dstream); g_dstream=NULL; + return errorcode; +} + + +static int benchSample(U32 benchNb, + size_t benchedSize, double compressibility, + int cLevel, ZSTD_compressionParameters cparams) +{ + /* Allocation */ + void* const origBuff = malloc(benchedSize); + if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); return 12; } + + /* Fill buffer */ + RDG_genBuffer(origBuff, benchedSize, compressibility, 0.0, 0); + + /* bench */ + DISPLAY("\r%70s\r", ""); + DISPLAY(" Sample %u bytes : \n", (unsigned)benchedSize); + if (benchNb) { + benchMem(benchNb, origBuff, benchedSize, cLevel, cparams); + } else { /* 0 == run all tests */ + for (benchNb=0; benchNb<100; benchNb++) { + benchMem(benchNb, origBuff, benchedSize, cLevel, cparams); + } } + + free(origBuff); + return 0; +} + + +static int benchFiles(U32 benchNb, + const char** fileNamesTable, const int nbFiles, + int cLevel, ZSTD_compressionParameters cparams) +{ + /* Loop for each file */ + int fileIdx; + for (fileIdx=0; fileIdx<nbFiles; fileIdx++) { + const char* const inFileName = fileNamesTable[fileIdx]; + FILE* const inFile = fopen( inFileName, "rb" ); + size_t benchedSize; + + /* Check file existence */ + if (inFile==NULL) { DISPLAY( "Pb opening %s\n", inFileName); return 11; } + + /* Memory allocation & restrictions */ + { U64 const inFileSize = UTIL_getFileSize(inFileName); + if (inFileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY( "Cannot measure size of %s\n", inFileName); + fclose(inFile); + return 11; + } + benchedSize = BMK_findMaxMem(inFileSize*3) / 3; + if ((U64)benchedSize > inFileSize) + benchedSize = (size_t)inFileSize; + if ((U64)benchedSize < inFileSize) { + DISPLAY("Not enough memory for '%s' full size; testing %u MB only... \n", + inFileName, (unsigned)(benchedSize>>20)); + } } + + /* Alloc */ + { void* const origBuff = malloc(benchedSize); + if (!origBuff) { DISPLAY("\nError: not enough memory!\n"); fclose(inFile); return 12; } + + /* Fill input buffer */ + DISPLAY("Loading %s... \r", inFileName); + { size_t const readSize = fread(origBuff, 1, benchedSize, inFile); + fclose(inFile); + if (readSize != benchedSize) { + DISPLAY("\nError: problem reading file '%s' !! \n", inFileName); + free(origBuff); + return 13; + } } + + /* bench */ + DISPLAY("\r%70s\r", ""); /* blank line */ + DISPLAY(" %s : \n", inFileName); + if (benchNb) { + benchMem(benchNb, origBuff, benchedSize, cLevel, cparams); + } else { + for (benchNb=0; benchNb<100; benchNb++) { + benchMem(benchNb, origBuff, benchedSize, cLevel, cparams); + } } + + free(origBuff); + } } + + return 0; +} + + + +/*_******************************************************* +* Argument Parsing +*********************************************************/ + +#define ERROR_OUT(msg) { DISPLAY("%s \n", msg); exit(1); } + +static unsigned readU32FromChar(const char** stringPtr) +{ + const char errorMsg[] = "error: numeric value too large"; + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + unsigned const max = (((unsigned)(-1)) / 10) - 1; + if (result > max) ERROR_OUT(errorMsg); + result *= 10; + result += (unsigned)(**stringPtr - '0'); + (*stringPtr)++ ; + } + if ((**stringPtr=='K') || (**stringPtr=='M')) { + unsigned const maxK = ((unsigned)(-1)) >> 10; + if (result > maxK) ERROR_OUT(errorMsg); + result <<= 10; + if (**stringPtr=='M') { + if (result > maxK) ERROR_OUT(errorMsg); + result <<= 10; + } + (*stringPtr)++; /* skip `K` or `M` */ + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +static int longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + + +/*_******************************************************* +* Command line +*********************************************************/ + +static int usage(const char* exename) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] file1 file2 ... fileX\n", exename); + DISPLAY( "Arguments :\n"); + DISPLAY( " -H/-h : Help (this text + advanced options)\n"); + return 0; +} + +static int usage_advanced(const char* exename) +{ + usage(exename); + DISPLAY( "\nAdvanced options :\n"); + DISPLAY( " -b# : test only function # \n"); + DISPLAY( " -l# : benchmark functions at that compression level (default : %i)\n", DEFAULT_CLEVEL); + DISPLAY( "--zstd= : custom parameter selection. Format same as zstdcli \n"); + DISPLAY( " -P# : sample compressibility (default : %.1f%%)\n", COMPRESSIBILITY_DEFAULT * 100); + DISPLAY( " -B# : sample size (default : %u)\n", (unsigned)kSampleSizeDefault); + DISPLAY( " -i# : iteration loops [1-9](default : %i)\n", NBLOOPS); + return 0; +} + +static int badusage(const char* exename) +{ + DISPLAY("Wrong parameters\n"); + usage(exename); + return 1; +} + +int main(int argc, const char** argv) +{ + int argNb, filenamesStart=0, result; + const char* const exename = argv[0]; + const char* input_filename = NULL; + U32 benchNb = 0, main_pause = 0; + int cLevel = DEFAULT_CLEVEL; + ZSTD_compressionParameters cparams = ZSTD_getCParams(cLevel, 0, 0); + size_t sampleSize = kSampleSizeDefault; + double compressibility = COMPRESSIBILITY_DEFAULT; + + DISPLAY(WELCOME_MESSAGE); + if (argc<1) return badusage(exename); + + for (argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + CONTROL(argument != NULL); + + if (longCommandWArg(&argument, "--zstd=")) { + for ( ; ;) { + if (longCommandWArg(&argument, "windowLog=") || longCommandWArg(&argument, "wlog=")) { cparams.windowLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "chainLog=") || longCommandWArg(&argument, "clog=")) { cparams.chainLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "hashLog=") || longCommandWArg(&argument, "hlog=")) { cparams.hashLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "searchLog=") || longCommandWArg(&argument, "slog=")) { cparams.searchLog = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "minMatch=") || longCommandWArg(&argument, "mml=")) { cparams.minMatch = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "targetLength=") || longCommandWArg(&argument, "tlen=")) { cparams.targetLength = readU32FromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "strategy=") || longCommandWArg(&argument, "strat=")) { cparams.strategy = (ZSTD_strategy)(readU32FromChar(&argument)); if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevel = (int)readU32FromChar(&argument); cparams = ZSTD_getCParams(cLevel, 0, 0); if (argument[0]==',') { argument++; continue; } else break; } + DISPLAY("invalid compression parameter \n"); + return 1; + } + + /* check end of string */ + if (argument[0] != 0) { + DISPLAY("invalid --zstd= format \n"); + return 1; + } else { + continue; + } + + } else if (argument[0]=='-') { /* Commands (note : aggregated commands are allowed) */ + argument++; + while (argument[0]!=0) { + + switch(argument[0]) + { + /* Display help on usage */ + case 'h': + case 'H': return usage_advanced(exename); + + /* Pause at the end (hidden option) */ + case 'p': main_pause = 1; break; + + /* Select specific algorithm to bench */ + case 'b': + argument++; + benchNb = readU32FromChar(&argument); + break; + + /* Select compression level to use */ + case 'l': + argument++; + cLevel = (int)readU32FromChar(&argument); + cparams = ZSTD_getCParams(cLevel, 0, 0); + break; + + /* Select compressibility of synthetic sample */ + case 'P': + argument++; + compressibility = (double)readU32FromChar(&argument) / 100.; + break; + + /* Select size of synthetic sample */ + case 'B': + argument++; + sampleSize = (size_t)readU32FromChar(&argument); + break; + + /* Modify Nb Iterations */ + case 'i': + argument++; + g_nbIterations = readU32FromChar(&argument); + break; + + /* Unknown command */ + default : return badusage(exename); + } + } + continue; + } + + /* first provided filename is input */ + if (!input_filename) { input_filename=argument; filenamesStart=argNb; continue; } + } + + + + if (filenamesStart==0) /* no input file */ + result = benchSample(benchNb, sampleSize, compressibility, cLevel, cparams); + else + result = benchFiles(benchNb, argv+filenamesStart, argc-filenamesStart, cLevel, cparams); + + if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } + + return result; +} diff --git a/src/zstd/tests/fuzz/.gitignore b/src/zstd/tests/fuzz/.gitignore new file mode 100644 index 000000000..b6fc6e506 --- /dev/null +++ b/src/zstd/tests/fuzz/.gitignore @@ -0,0 +1,20 @@ +# test artefacts +corpora +block_decompress +block_round_trip +dictionary_decompress +dictionary_loader +dictionary_round_trip +dictionary_stream_round_trip +raw_dictionary_round_trip +simple_compress +simple_decompress +simple_round_trip +stream_decompress +stream_round_trip +zstd_frame_info +fuzz-*.log + +# misc +trace +tmp* diff --git a/src/zstd/tests/fuzz/Makefile b/src/zstd/tests/fuzz/Makefile new file mode 100644 index 000000000..1af3dc734 --- /dev/null +++ b/src/zstd/tests/fuzz/Makefile @@ -0,0 +1,212 @@ +# ################################################################ +# Copyright (c) 2016-2020, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +# Optionally user defined flags +CFLAGS ?= -O3 +CXXFLAGS ?= -O3 +CPPFLAGS ?= +LDFLAGS ?= +ARFLAGS ?= +LIB_FUZZING_ENGINE ?= libregression.a +PYTHON ?= python +ifeq ($(shell uname), Darwin) + DOWNLOAD?=curl -L -o +else + DOWNLOAD?=wget -O +endif +CORPORA_URL_PREFIX:=https://github.com/facebook/zstd/releases/download/fuzz-corpora/ + +ZSTDDIR = ../../lib +PRGDIR = ../../programs + +FUZZ_CPPFLAGS := -I$(ZSTDDIR) -I$(ZSTDDIR)/common -I$(ZSTDDIR)/compress \ + -I$(ZSTDDIR)/dictBuilder -I$(ZSTDDIR)/deprecated -I$(ZSTDDIR)/legacy \ + -I$(PRGDIR) -DZSTD_MULTITHREAD -DZSTD_LEGACY_SUPPORT=1 $(CPPFLAGS) +FUZZ_EXTRA_FLAGS := -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \ + -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \ + -Wstrict-prototypes -Wundef \ + -Wvla -Wformat=2 -Winit-self -Wfloat-equal -Wwrite-strings \ + -Wredundant-decls \ + -g -fno-omit-frame-pointer +FUZZ_CFLAGS := $(FUZZ_EXTRA_FLAGS) $(CFLAGS) +FUZZ_CXXFLAGS := $(FUZZ_EXTRA_FLAGS) -std=c++11 $(CXXFLAGS) +FUZZ_LDFLAGS := -pthread $(LDFLAGS) +FUZZ_ARFLAGS := $(ARFLAGS) +FUZZ_TARGET_FLAGS = $(FUZZ_CPPFLAGS) $(FUZZ_CXXFLAGS) $(FUZZ_LDFLAGS) + +FUZZ_ROUND_TRIP_FLAGS := -DFUZZING_ASSERT_VALID_SEQUENCE + +FUZZ_HEADERS := fuzz_helpers.h fuzz.h zstd_helpers.h fuzz_data_producer.h +FUZZ_SRC := $(PRGDIR)/util.c ./fuzz_helpers.c ./zstd_helpers.c ./fuzz_data_producer.c + +ZSTDCOMMON_SRC := $(ZSTDDIR)/common/*.c +ZSTDCOMP_SRC := $(ZSTDDIR)/compress/*.c +ZSTDDECOMP_SRC := $(ZSTDDIR)/decompress/*.c +ZSTDDICT_SRC := $(ZSTDDIR)/dictBuilder/*.c +ZSTDLEGACY_SRC := $(ZSTDDIR)/legacy/*.c +FUZZ_SRC := \ + $(FUZZ_SRC) \ + $(ZSTDDECOMP_SRC) \ + $(ZSTDCOMMON_SRC) \ + $(ZSTDCOMP_SRC) \ + $(ZSTDDICT_SRC) \ + $(ZSTDLEGACY_SRC) +FUZZ_SRC := $(wildcard $(FUZZ_SRC)) + +FUZZ_D_OBJ1 := $(subst $(ZSTDDIR)/common/,d_lib_common_,$(FUZZ_SRC)) +FUZZ_D_OBJ2 := $(subst $(ZSTDDIR)/compress/,d_lib_compress_,$(FUZZ_D_OBJ1)) +FUZZ_D_OBJ3 := $(subst $(ZSTDDIR)/decompress/,d_lib_decompress_,$(FUZZ_D_OBJ2)) +FUZZ_D_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,d_lib_dictBuilder_,$(FUZZ_D_OBJ3)) +FUZZ_D_OBJ5 := $(subst $(ZSTDDIR)/legacy/,d_lib_legacy_,$(FUZZ_D_OBJ4)) +FUZZ_D_OBJ6 := $(subst $(PRGDIR)/,d_prg_,$(FUZZ_D_OBJ5)) +FUZZ_D_OBJ7 := $(subst $\./,d_fuzz_,$(FUZZ_D_OBJ6)) +FUZZ_DECOMPRESS_OBJ := $(FUZZ_D_OBJ7:.c=.o) + +FUZZ_RT_OBJ1 := $(subst $(ZSTDDIR)/common/,rt_lib_common_,$(FUZZ_SRC)) +FUZZ_RT_OBJ2 := $(subst $(ZSTDDIR)/compress/,rt_lib_compress_,$(FUZZ_RT_OBJ1)) +FUZZ_RT_OBJ3 := $(subst $(ZSTDDIR)/decompress/,rt_lib_decompress_,$(FUZZ_RT_OBJ2)) +FUZZ_RT_OBJ4 := $(subst $(ZSTDDIR)/dictBuilder/,rt_lib_dictBuilder_,$(FUZZ_RT_OBJ3)) +FUZZ_RT_OBJ5 := $(subst $(ZSTDDIR)/legacy/,rt_lib_legacy_,$(FUZZ_RT_OBJ4)) +FUZZ_RT_OBJ6 := $(subst $(PRGDIR)/,rt_prg_,$(FUZZ_RT_OBJ5)) +FUZZ_RT_OBJ7 := $(subst $\./,rt_fuzz_,$(FUZZ_RT_OBJ6)) +FUZZ_ROUND_TRIP_OBJ := $(FUZZ_RT_OBJ7:.c=.o) + +.PHONY: default all clean cleanall + +default: all + +FUZZ_TARGETS := \ + simple_round_trip \ + stream_round_trip \ + block_round_trip \ + simple_decompress \ + stream_decompress \ + block_decompress \ + dictionary_round_trip \ + dictionary_decompress \ + zstd_frame_info \ + simple_compress \ + dictionary_loader \ + raw_dictionary_round_trip \ + dictionary_stream_round_trip + +all: $(FUZZ_TARGETS) + +rt_lib_common_%.o: $(ZSTDDIR)/common/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ + +rt_lib_compress_%.o: $(ZSTDDIR)/compress/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ + +rt_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ + +rt_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ + +rt_lib_legacy_%.o: $(ZSTDDIR)/legacy/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ + +rt_prg_%.o: $(PRGDIR)/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ + +rt_fuzz_%.o: %.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $(FUZZ_ROUND_TRIP_FLAGS) $< -c -o $@ + +d_lib_common_%.o: $(ZSTDDIR)/common/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ + +d_lib_compress_%.o: $(ZSTDDIR)/compress/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ + +d_lib_decompress_%.o: $(ZSTDDIR)/decompress/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ + +d_lib_dictBuilder_%.o: $(ZSTDDIR)/dictBuilder/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ + +d_lib_legacy_%.o: $(ZSTDDIR)/legacy/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ + +d_prg_%.o: $(PRGDIR)/%.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ + +d_fuzz_%.o: %.c + $(CC) $(FUZZ_CPPFLAGS) $(FUZZ_CFLAGS) $< -c -o $@ + +simple_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ + +stream_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_stream_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ + +block_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_block_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_block_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ + +simple_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_simple_decompress.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_simple_decompress.o $(LIB_FUZZING_ENGINE) -o $@ + +stream_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_stream_decompress.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_stream_decompress.o $(LIB_FUZZING_ENGINE) -o $@ + +block_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_block_decompress.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_block_decompress.o $(LIB_FUZZING_ENGINE) -o $@ + +dictionary_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ + +raw_dictionary_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_raw_dictionary_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_raw_dictionary_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ + +dictionary_stream_round_trip: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_stream_round_trip.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_stream_round_trip.o $(LIB_FUZZING_ENGINE) -o $@ + +dictionary_decompress: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_dictionary_decompress.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_dictionary_decompress.o $(LIB_FUZZING_ENGINE) -o $@ + +simple_compress: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_compress.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_simple_compress.o $(LIB_FUZZING_ENGINE) -o $@ + +zstd_frame_info: $(FUZZ_HEADERS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_zstd_frame_info.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_DECOMPRESS_OBJ) d_fuzz_zstd_frame_info.o $(LIB_FUZZING_ENGINE) -o $@ + +dictionary_loader: $(FUZZ_HEADERS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_loader.o + $(CXX) $(FUZZ_TARGET_FLAGS) $(FUZZ_ROUND_TRIP_OBJ) rt_fuzz_dictionary_loader.o $(LIB_FUZZING_ENGINE) -o $@ + +libregression.a: $(FUZZ_HEADERS) $(PRGDIR)/util.h $(PRGDIR)/util.c d_fuzz_regression_driver.o + $(AR) $(FUZZ_ARFLAGS) $@ d_fuzz_regression_driver.o + +corpora/%_seed_corpus.zip: + @mkdir -p corpora + $(DOWNLOAD) $@ $(CORPORA_URL_PREFIX)$*_seed_corpus.zip + +corpora/%: corpora/%_seed_corpus.zip + unzip -q $^ -d $@ + +.PHONY: corpora +corpora: $(patsubst %,corpora/%,$(FUZZ_TARGETS)) + +.PHONY: seedcorpora +seedcorpora: $(patsubst %,corpora/%_seed_corpus.zip,$(FUZZ_TARGETS)) + +regressiontest: corpora + CC="$(CC)" CXX="$(CXX)" CFLAGS="$(CFLAGS)" CXXFLAGS="$(CXXFLAGS)" LDFLAGS="$(LDFLAGS)" $(PYTHON) ./fuzz.py build all + $(PYTHON) ./fuzz.py regression all + +clean: + @$(RM) *.a *.o + @$(RM) simple_round_trip stream_round_trip simple_decompress \ + stream_decompress block_decompress block_round_trip \ + simple_compress dictionary_round_trip dictionary_decompress \ + zstd_frame_info + +cleanall: + @$(RM) -r Fuzzer + @$(RM) -r corpora diff --git a/src/zstd/tests/fuzz/README.md b/src/zstd/tests/fuzz/README.md new file mode 100644 index 000000000..71afa4063 --- /dev/null +++ b/src/zstd/tests/fuzz/README.md @@ -0,0 +1,101 @@ +# Fuzzing + +Each fuzzing target can be built with multiple engines. +Zstd provides a fuzz corpus for each target that can be downloaded with +the command: + +``` +make corpora +``` + +It will download each corpus into `./corpora/TARGET`. + +## fuzz.py + +`fuzz.py` is a helper script for building and running fuzzers. +Run `./fuzz.py -h` for the commands and run `./fuzz.py COMMAND -h` for +command specific help. + +### Generating Data + +`fuzz.py` provides a utility to generate seed data for each fuzzer. + +``` +make -C ../tests decodecorpus +./fuzz.py gen TARGET +``` + +By default it outputs 100 samples, each at most 8KB into `corpora/TARGET-seed`, +but that can be configured with the `--number`, `--max-size-log` and `--seed` +flags. + +### Build +It respects the usual build environment variables `CC`, `CFLAGS`, etc. +The environment variables can be overridden with the corresponding flags +`--cc`, `--cflags`, etc. +The specific fuzzing engine is selected with `LIB_FUZZING_ENGINE` or +`--lib-fuzzing-engine`, the default is `libregression.a`. +Alternatively, you can use Clang's built in fuzzing engine with +`--enable-fuzzer`. +It has flags that can easily set up sanitizers `--enable-{a,ub,m}san`, and +coverage instrumentation `--enable-coverage`. +It sets sane defaults which can be overridden with flags `--debug`, +`--enable-ubsan-pointer-overflow`, etc. +Run `./fuzz.py build -h` for help. + +### Running Fuzzers + +`./fuzz.py` can run `libfuzzer`, `afl`, and `regression` tests. +See the help of the relevant command for options. +Flags not parsed by `fuzz.py` are passed to the fuzzing engine. +The command used to run the fuzzer is printed for debugging. + +## LibFuzzer + +``` +# Build the fuzz targets +./fuzz.py build all --enable-fuzzer --enable-asan --enable-ubsan --cc clang --cxx clang++ +# OR equivalently +CC=clang CXX=clang++ ./fuzz.py build all --enable-fuzzer --enable-asan --enable-ubsan +# Run the fuzzer +./fuzz.py libfuzzer TARGET <libfuzzer args like -jobs=4> +``` + +where `TARGET` could be `simple_decompress`, `stream_round_trip`, etc. + +### MSAN + +Fuzzing with `libFuzzer` and `MSAN` is as easy as: + +``` +CC=clang CXX=clang++ ./fuzz.py build all --enable-fuzzer --enable-msan +./fuzz.py libfuzzer TARGET <libfuzzer args> +``` + +`fuzz.py` respects the environment variables / flags `MSAN_EXTRA_CPPFLAGS`, +`MSAN_EXTRA_CFLAGS`, `MSAN_EXTRA_CXXFLAGS`, `MSAN_EXTRA_LDFLAGS` to easily pass +the extra parameters only for MSAN. + +## AFL + +The default `LIB_FUZZING_ENGINE` is `libregression.a`, which produces a binary +that AFL can use. + +``` +# Build the fuzz targets +CC=afl-clang CXX=afl-clang++ ./fuzz.py build all --enable-asan --enable-ubsan +# Run the fuzzer without a memory limit because of ASAN +./fuzz.py afl TARGET -m none +``` + +## Regression Testing + +The regression test supports the `all` target to run all the fuzzers in one +command. + +``` +CC=clang CXX=clang++ ./fuzz.py build all --enable-asan --enable-ubsan +./fuzz.py regression all +CC=clang CXX=clang++ ./fuzz.py build all --enable-msan +./fuzz.py regression all +``` diff --git a/src/zstd/tests/fuzz/block_decompress.c b/src/zstd/tests/fuzz/block_decompress.c new file mode 100644 index 000000000..64d70f005 --- /dev/null +++ b/src/zstd/tests/fuzz/block_decompress.c @@ -0,0 +1,49 @@ +/** + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target attempts to decompress the fuzzed data with the simple + * decompression function to ensure the decompressor never crashes. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include "fuzz_helpers.h" +#include "zstd.h" + +static ZSTD_DCtx *dctx = NULL; +static void* rBuf = NULL; +static size_t bufSize = 0; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + size_t const neededBufSize = ZSTD_BLOCKSIZE_MAX; + + /* Allocate all buffers and contexts if not already allocated */ + if (neededBufSize > bufSize) { + free(rBuf); + rBuf = FUZZ_malloc(neededBufSize); + bufSize = neededBufSize; + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + ZSTD_decompressBegin(dctx); + ZSTD_decompressBlock(dctx, rBuf, neededBufSize, src, size); + +#ifndef STATEFUL_FUZZING + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/block_round_trip.c b/src/zstd/tests/fuzz/block_round_trip.c new file mode 100644 index 000000000..097fc01b8 --- /dev/null +++ b/src/zstd/tests/fuzz/block_round_trip.c @@ -0,0 +1,99 @@ +/** + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target performs a zstd round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; +static void* cBuf = NULL; +static void* rBuf = NULL; +static size_t bufSize = 0; + +static size_t roundTripTest(void *result, size_t resultCapacity, + void *compressed, size_t compressedCapacity, + const void *src, size_t srcSize, + int cLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(cLevel, srcSize, 0); + size_t ret = ZSTD_compressBegin_advanced(cctx, NULL, 0, params, srcSize); + FUZZ_ZASSERT(ret); + + ret = ZSTD_compressBlock(cctx, compressed, compressedCapacity, src, srcSize); + FUZZ_ZASSERT(ret); + if (ret == 0) { + FUZZ_ASSERT(resultCapacity >= srcSize); + if (srcSize > 0) { + memcpy(result, src, srcSize); + } + return srcSize; + } + ZSTD_decompressBegin(dctx); + return ZSTD_decompressBlock(dctx, result, resultCapacity, compressed, ret); +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); + + size_t neededBufSize = size; + if (size > ZSTD_BLOCKSIZE_MAX) + size = ZSTD_BLOCKSIZE_MAX; + + /* Allocate all buffers and contexts if not already allocated */ + if (neededBufSize > bufSize || !cBuf || !rBuf) { + free(cBuf); + free(rBuf); + cBuf = FUZZ_malloc(neededBufSize); + rBuf = FUZZ_malloc(neededBufSize); + bufSize = neededBufSize; + } + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + { + size_t const result = + roundTripTest(rBuf, neededBufSize, cBuf, neededBufSize, src, size, + cLevel); + FUZZ_ZASSERT(result); + FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!"); + } + FUZZ_dataProducer_free(producer); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/dictionary_decompress.c b/src/zstd/tests/fuzz/dictionary_decompress.c new file mode 100644 index 000000000..9944baa15 --- /dev/null +++ b/src/zstd/tests/fuzz/dictionary_decompress.c @@ -0,0 +1,73 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target attempts to decompress the fuzzed data with the dictionary + * decompression function to ensure the decompressor never crashes. It does not + * fuzz the dictionary. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_DCtx *dctx = NULL; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + FUZZ_dict_t dict; + ZSTD_DDict* ddict = NULL; + + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + dict = FUZZ_train(src, size, producer); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { + ddict = ZSTD_createDDict(dict.buff, dict.size); + FUZZ_ASSERT(ddict); + } else { + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( + dctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2))); + else + FUZZ_ZASSERT(ZSTD_DCtx_refPrefix_advanced( + dctx, dict.buff, dict.size, + (ZSTD_dictContentType_e)FUZZ_dataProducer_uint32Range(producer, 0, 2))); + } + + { + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); + void* rBuf = FUZZ_malloc(bufSize); + if (ddict) { + ZSTD_decompress_usingDDict(dctx, rBuf, bufSize, src, size, ddict); + } else { + ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); + } + free(rBuf); + } + free(dict.buff); + FUZZ_dataProducer_free(producer); + ZSTD_freeDDict(ddict); +#ifndef STATEFUL_FUZZING + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/dictionary_loader.c b/src/zstd/tests/fuzz/dictionary_loader.c new file mode 100644 index 000000000..f1fdf4da9 --- /dev/null +++ b/src/zstd/tests/fuzz/dictionary_loader.c @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target makes sure that whenever a compression dictionary can be + * loaded, the data can be round tripped. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +/** + * Compresses the data and returns the compressed size or an error. + */ +static size_t compress(void* compressed, size_t compressedCapacity, + void const* source, size_t sourceSize, + void const* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + int const refPrefix) +{ + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + if (refPrefix) + FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced( + cctx, dict, dictSize, dictContentType)); + else + FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, dictLoadMethod, dictContentType)); + size_t const compressedSize = ZSTD_compress2( + cctx, compressed, compressedCapacity, source, sourceSize); + ZSTD_freeCCtx(cctx); + return compressedSize; +} + +static size_t decompress(void* result, size_t resultCapacity, + void const* compressed, size_t compressedSize, + void const* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + int const refPrefix) +{ + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + if (refPrefix) + FUZZ_ZASSERT(ZSTD_DCtx_refPrefix_advanced( + dctx, dict, dictSize, dictContentType)); + else + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( + dctx, dict, dictSize, dictLoadMethod, dictContentType)); + size_t const resultSize = ZSTD_decompressDCtx( + dctx, result, resultCapacity, compressed, compressedSize); + FUZZ_ZASSERT(resultSize); + ZSTD_freeDCtx(dctx); + return resultSize; +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + int const refPrefix = FUZZ_dataProducer_uint32Range(producer, 0, 1) != 0; + ZSTD_dictLoadMethod_e const dlm = + size = FUZZ_dataProducer_uint32Range(producer, 0, 1); + ZSTD_dictContentType_e const dct = + FUZZ_dataProducer_uint32Range(producer, 0, 2); + size = FUZZ_dataProducer_remainingBytes(producer); + + DEBUGLOG(2, "Dict load method %d", dlm); + DEBUGLOG(2, "Dict content type %d", dct); + DEBUGLOG(2, "Dict size %u", (unsigned)size); + + void* const rBuf = FUZZ_malloc(size); + size_t const cBufSize = ZSTD_compressBound(size); + void* const cBuf = FUZZ_malloc(cBufSize); + + size_t const cSize = + compress(cBuf, cBufSize, src, size, src, size, dlm, dct, refPrefix); + /* compression failing is okay */ + if (ZSTD_isError(cSize)) { + FUZZ_ASSERT_MSG(dct != ZSTD_dct_rawContent, "Raw must always succeed!"); + goto out; + } + size_t const rSize = + decompress(rBuf, size, cBuf, cSize, src, size, dlm, dct, refPrefix); + FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!"); + +out: + free(cBuf); + free(rBuf); + FUZZ_dataProducer_free(producer); + return 0; +} diff --git a/src/zstd/tests/fuzz/dictionary_round_trip.c b/src/zstd/tests/fuzz/dictionary_round_trip.c new file mode 100644 index 000000000..7b7771e48 --- /dev/null +++ b/src/zstd/tests/fuzz/dictionary_round_trip.c @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target performs a zstd round-trip test (compress & decompress) with + * a dictionary, compares the result with the original, and calls abort() on + * corruption. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; + +static size_t roundTripTest(void *result, size_t resultCapacity, + void *compressed, size_t compressedCapacity, + const void *src, size_t srcSize, + FUZZ_dataProducer_t *producer) +{ + ZSTD_dictContentType_e dictContentType = ZSTD_dct_auto; + FUZZ_dict_t dict = FUZZ_train(src, srcSize, producer); + int const refPrefix = FUZZ_dataProducer_uint32Range(producer, 0, 1) != 0; + size_t cSize; + if (FUZZ_dataProducer_uint32Range(producer, 0, 15) == 0) { + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); + + cSize = ZSTD_compress_usingDict(cctx, + compressed, compressedCapacity, + src, srcSize, + dict.buff, dict.size, + cLevel); + } else { + dictContentType = FUZZ_dataProducer_uint32Range(producer, 0, 2); + FUZZ_setRandomParameters(cctx, srcSize, producer); + /* Disable checksum so we can use sizes smaller than compress bound. */ + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); + if (refPrefix) + FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced( + cctx, dict.buff, dict.size, + dictContentType)); + else + FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( + cctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + dictContentType)); + cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + } + FUZZ_ZASSERT(cSize); + if (refPrefix) + FUZZ_ZASSERT(ZSTD_DCtx_refPrefix_advanced( + dctx, dict.buff, dict.size, + dictContentType)); + else + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( + dctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + dictContentType)); + { + size_t const ret = ZSTD_decompressDCtx( + dctx, result, resultCapacity, compressed, cSize); + free(dict.buff); + return ret; + } +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + size_t const rBufSize = size; + void* rBuf = FUZZ_malloc(rBufSize); + size_t cBufSize = ZSTD_compressBound(size); + void *cBuf; + /* Half of the time fuzz with a 1 byte smaller output size. + * This will still succeed because we force the checksum to be disabled, + * giving us 4 bytes of overhead. + */ + cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); + cBuf = FUZZ_malloc(cBufSize); + + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + { + size_t const result = + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer); + FUZZ_ZASSERT(result); + FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!"); + } + free(rBuf); + free(cBuf); + FUZZ_dataProducer_free(producer); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/dictionary_stream_round_trip.c b/src/zstd/tests/fuzz/dictionary_stream_round_trip.c new file mode 100644 index 000000000..67e8c69ef --- /dev/null +++ b/src/zstd/tests/fuzz/dictionary_stream_round_trip.c @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target performs a zstd round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; +static uint8_t* cBuf = NULL; +static uint8_t* rBuf = NULL; +static size_t bufSize = 0; + +static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity, + FUZZ_dataProducer_t *producer) +{ + ZSTD_outBuffer buffer = { dst, 0, 0 }; + + FUZZ_ASSERT(capacity > 0); + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, capacity)); + FUZZ_ASSERT(buffer.size <= capacity); + + return buffer; +} + +static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, + FUZZ_dataProducer_t *producer) +{ + ZSTD_inBuffer buffer = { *src, 0, 0 }; + + FUZZ_ASSERT(*size > 0); + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size)); + FUZZ_ASSERT(buffer.size <= *size); + *src += buffer.size; + *size -= buffer.size; + + return buffer; +} + +static size_t compress(uint8_t *dst, size_t capacity, + const uint8_t *src, size_t srcSize, + const uint8_t* dict, size_t dictSize, + FUZZ_dataProducer_t *producer, int refPrefix, + ZSTD_dictContentType_e dictContentType) +{ + size_t dstSize = 0; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + FUZZ_setRandomParameters(cctx, srcSize, producer); + + /* Disable checksum so we can use sizes smaller than compress bound. */ + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); + if (refPrefix) + FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced( + cctx, dict, dictSize, + dictContentType)); + else + FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + dictContentType)); + + while (srcSize > 0) { + ZSTD_inBuffer in = makeInBuffer(&src, &srcSize, producer); + /* Mode controls the action. If mode == -1 we pick a new mode */ + int mode = -1; + while (in.pos < in.size || mode != -1) { + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); + /* Previous action finished, pick a new mode. */ + if (mode == -1) mode = FUZZ_dataProducer_uint32Range(producer, 0, 9); + switch (mode) { + case 0: /* fall-through */ + case 1: /* fall-through */ + case 2: { + size_t const ret = + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush); + FUZZ_ZASSERT(ret); + if (ret == 0) + mode = -1; + break; + } + case 3: { + size_t ret = + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); + FUZZ_ZASSERT(ret); + /* Reset the compressor when the frame is finished */ + if (ret == 0) { + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + if (FUZZ_dataProducer_uint32Range(producer, 0, 7) == 0) { + size_t const remaining = in.size - in.pos; + FUZZ_setRandomParameters(cctx, remaining, producer); + } + mode = -1; + } + break; + } + case 4: { + ZSTD_inBuffer nullIn = { NULL, 0, 0 }; + ZSTD_outBuffer nullOut = { NULL, 0, 0 }; + size_t const ret = ZSTD_compressStream2(cctx, &nullOut, &nullIn, ZSTD_e_continue); + FUZZ_ZASSERT(ret); + } + /* fall-through */ + default: { + size_t const ret = + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); + FUZZ_ZASSERT(ret); + mode = -1; + } + } + dst += out.pos; + dstSize += out.pos; + capacity -= out.pos; + } + } + for (;;) { + ZSTD_inBuffer in = {NULL, 0, 0}; + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); + size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); + FUZZ_ZASSERT(ret); + + dst += out.pos; + dstSize += out.pos; + capacity -= out.pos; + if (ret == 0) + break; + } + return dstSize; +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + size_t neededBufSize; + + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + neededBufSize = ZSTD_compressBound(size) * 15; + + /* Allocate all buffers and contexts if not already allocated */ + if (neededBufSize > bufSize) { + free(cBuf); + free(rBuf); + cBuf = (uint8_t*)FUZZ_malloc(neededBufSize); + rBuf = (uint8_t*)FUZZ_malloc(neededBufSize); + bufSize = neededBufSize; + } + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + { + ZSTD_dictContentType_e dictContentType = FUZZ_dataProducer_uint32Range(producer, 0, 2); + FUZZ_dict_t dict = FUZZ_train(src, size, producer); + int const refPrefix = FUZZ_dataProducer_uint32Range(producer, 0, 1) != 0; + + size_t const cSize = compress(cBuf, neededBufSize, src, size, dict.buff, dict.size, producer, refPrefix, dictContentType); + + if (refPrefix) + FUZZ_ZASSERT(ZSTD_DCtx_refPrefix_advanced( + dctx, dict.buff, dict.size, + dictContentType)); + else + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( + dctx, dict.buff, dict.size, + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + dictContentType)); + size_t const rSize = + ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize); + FUZZ_ZASSERT(rSize); + FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!"); + free(dict.buff); + } + + FUZZ_dataProducer_free(producer); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/fuzz.h b/src/zstd/tests/fuzz/fuzz.h new file mode 100644 index 000000000..8ee964536 --- /dev/null +++ b/src/zstd/tests/fuzz/fuzz.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * Fuzz target interface. + * Fuzz targets have some common parameters passed as macros during compilation. + * Check the documentation for each individual fuzzer for more parameters. + * + * @param STATEFUL_FUZZING: + * Define this to reuse state between fuzzer runs. This can be useful to + * test code paths which are only executed when contexts are reused. + * WARNING: Makes reproducing crashes much harder. + * Default: Not defined. + * @param DEBUGLEVEL: + * This is a parameter for the zstd library. Defining `DEBUGLEVEL=1` + * enables assert() statements in the zstd library. Higher levels enable + * logging, so aren't recommended. Defining `DEBUGLEVEL=1` is + * recommended. + * @param MEM_FORCE_MEMORY_ACCESS: + * This flag controls how the zstd library accesses unaligned memory. + * It can be undefined, or 0 through 2. If it is undefined, it selects + * the method to use based on the compiler. If testing with UBSAN set + * MEM_FORCE_MEMORY_ACCESS=0 to use the standard compliant method. + * @param FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + * This is the canonical flag to enable deterministic builds for fuzzing. + * Changes to zstd for fuzzing are gated behind this define. + * It is recommended to define this when building zstd for fuzzing. + */ + +#ifndef FUZZ_H +#define FUZZ_H + +#include <stddef.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/zstd/tests/fuzz/fuzz.py b/src/zstd/tests/fuzz/fuzz.py new file mode 100755 index 000000000..6875d1d60 --- /dev/null +++ b/src/zstd/tests/fuzz/fuzz.py @@ -0,0 +1,887 @@ +#!/usr/bin/env python + +# ################################################################ +# Copyright (c) 2016-2020, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ########################################################################## + +import argparse +import contextlib +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile + + +def abs_join(a, *p): + return os.path.abspath(os.path.join(a, *p)) + + +class InputType(object): + RAW_DATA = 1 + COMPRESSED_DATA = 2 + DICTIONARY_DATA = 3 + + +class FrameType(object): + ZSTD = 1 + BLOCK = 2 + + +class TargetInfo(object): + def __init__(self, input_type, frame_type=FrameType.ZSTD): + self.input_type = input_type + self.frame_type = frame_type + + +# Constants +FUZZ_DIR = os.path.abspath(os.path.dirname(__file__)) +CORPORA_DIR = abs_join(FUZZ_DIR, 'corpora') +TARGET_INFO = { + 'simple_round_trip': TargetInfo(InputType.RAW_DATA), + 'stream_round_trip': TargetInfo(InputType.RAW_DATA), + 'block_round_trip': TargetInfo(InputType.RAW_DATA, FrameType.BLOCK), + 'simple_decompress': TargetInfo(InputType.COMPRESSED_DATA), + 'stream_decompress': TargetInfo(InputType.COMPRESSED_DATA), + 'block_decompress': TargetInfo(InputType.COMPRESSED_DATA, FrameType.BLOCK), + 'dictionary_round_trip': TargetInfo(InputType.RAW_DATA), + 'dictionary_decompress': TargetInfo(InputType.COMPRESSED_DATA), + 'zstd_frame_info': TargetInfo(InputType.COMPRESSED_DATA), + 'simple_compress': TargetInfo(InputType.RAW_DATA), + 'dictionary_loader': TargetInfo(InputType.DICTIONARY_DATA), + 'raw_dictionary_round_trip': TargetInfo(InputType.RAW_DATA), + 'dictionary_stream_round_trip': TargetInfo(InputType.RAW_DATA), +} +TARGETS = list(TARGET_INFO.keys()) +ALL_TARGETS = TARGETS + ['all'] +FUZZ_RNG_SEED_SIZE = 4 + +# Standard environment variables +CC = os.environ.get('CC', 'cc') +CXX = os.environ.get('CXX', 'c++') +CPPFLAGS = os.environ.get('CPPFLAGS', '') +CFLAGS = os.environ.get('CFLAGS', '-O3') +CXXFLAGS = os.environ.get('CXXFLAGS', CFLAGS) +LDFLAGS = os.environ.get('LDFLAGS', '') +MFLAGS = os.environ.get('MFLAGS', '-j') + +# Fuzzing environment variables +LIB_FUZZING_ENGINE = os.environ.get('LIB_FUZZING_ENGINE', 'libregression.a') +AFL_FUZZ = os.environ.get('AFL_FUZZ', 'afl-fuzz') +DECODECORPUS = os.environ.get('DECODECORPUS', + abs_join(FUZZ_DIR, '..', 'decodecorpus')) +ZSTD = os.environ.get('ZSTD', abs_join(FUZZ_DIR, '..', '..', 'zstd')) + +# Sanitizer environment variables +MSAN_EXTRA_CPPFLAGS = os.environ.get('MSAN_EXTRA_CPPFLAGS', '') +MSAN_EXTRA_CFLAGS = os.environ.get('MSAN_EXTRA_CFLAGS', '') +MSAN_EXTRA_CXXFLAGS = os.environ.get('MSAN_EXTRA_CXXFLAGS', '') +MSAN_EXTRA_LDFLAGS = os.environ.get('MSAN_EXTRA_LDFLAGS', '') + + +def create(r): + d = os.path.abspath(r) + if not os.path.isdir(d): + os.makedirs(d) + return d + + +def check(r): + d = os.path.abspath(r) + if not os.path.isdir(d): + return None + return d + + +@contextlib.contextmanager +def tmpdir(): + dirpath = tempfile.mkdtemp() + try: + yield dirpath + finally: + shutil.rmtree(dirpath, ignore_errors=True) + + +def parse_targets(in_targets): + targets = set() + for target in in_targets: + if not target: + continue + if target == 'all': + targets = targets.union(TARGETS) + elif target in TARGETS: + targets.add(target) + else: + raise RuntimeError('{} is not a valid target'.format(target)) + return list(targets) + + +def targets_parser(args, description): + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + 'TARGET', + nargs='*', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS))) + args, extra = parser.parse_known_args(args) + args.extra = extra + + args.TARGET = parse_targets(args.TARGET) + + return args + + +def parse_env_flags(args, flags): + """ + Look for flags set by environment variables. + """ + san_flags = ','.join(re.findall('-fsanitize=((?:[a-z]+,?)+)', flags)) + nosan_flags = ','.join(re.findall('-fno-sanitize=((?:[a-z]+,?)+)', flags)) + + def set_sanitizer(sanitizer, default, san, nosan): + if sanitizer in san and sanitizer in nosan: + raise RuntimeError('-fno-sanitize={s} and -fsanitize={s} passed'. + format(s=sanitizer)) + if sanitizer in san: + return True + if sanitizer in nosan: + return False + return default + + san = set(san_flags.split(',')) + nosan = set(nosan_flags.split(',')) + + args.asan = set_sanitizer('address', args.asan, san, nosan) + args.msan = set_sanitizer('memory', args.msan, san, nosan) + args.ubsan = set_sanitizer('undefined', args.ubsan, san, nosan) + + args.sanitize = args.asan or args.msan or args.ubsan + + return args + + +def compiler_version(cc, cxx): + """ + Determines the compiler and version. + Only works for clang and gcc. + """ + cc_version_bytes = subprocess.check_output([cc, "--version"]) + cxx_version_bytes = subprocess.check_output([cxx, "--version"]) + compiler = None + version = None + if b'clang' in cc_version_bytes: + assert(b'clang' in cxx_version_bytes) + compiler = 'clang' + elif b'gcc' in cc_version_bytes: + assert(b'gcc' in cxx_version_bytes or b'g++' in cxx_version_bytes) + compiler = 'gcc' + if compiler is not None: + version_regex = b'([0-9])+\.([0-9])+\.([0-9])+' + version_match = re.search(version_regex, cc_version_bytes) + version = tuple(int(version_match.group(i)) for i in range(1, 4)) + return compiler, version + + +def overflow_ubsan_flags(cc, cxx): + compiler, version = compiler_version(cc, cxx) + if compiler == 'gcc': + return ['-fno-sanitize=signed-integer-overflow'] + if compiler == 'clang' and version >= (5, 0, 0): + return ['-fno-sanitize=pointer-overflow'] + return [] + + +def build_parser(args): + description = """ + Cleans the repository and builds a fuzz target (or all). + Many flags default to environment variables (default says $X='y'). + Options that aren't enabling features default to the correct values for + zstd. + Enable sanitizers with --enable-*san. + For regression testing just build. + For libFuzzer set LIB_FUZZING_ENGINE and pass --enable-coverage. + For AFL set CC and CXX to AFL's compilers and set + LIB_FUZZING_ENGINE='libregression.a'. + """ + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + '--lib-fuzzing-engine', + dest='lib_fuzzing_engine', + type=str, + default=LIB_FUZZING_ENGINE, + help=('The fuzzing engine to use e.g. /path/to/libFuzzer.a ' + "(default: $LIB_FUZZING_ENGINE='{})".format(LIB_FUZZING_ENGINE))) + + fuzz_group = parser.add_mutually_exclusive_group() + fuzz_group.add_argument( + '--enable-coverage', + dest='coverage', + action='store_true', + help='Enable coverage instrumentation (-fsanitize-coverage)') + fuzz_group.add_argument( + '--enable-fuzzer', + dest='fuzzer', + action='store_true', + help=('Enable clang fuzzer (-fsanitize=fuzzer). When enabled ' + 'LIB_FUZZING_ENGINE is ignored') + ) + + parser.add_argument( + '--enable-asan', dest='asan', action='store_true', help='Enable UBSAN') + parser.add_argument( + '--enable-ubsan', + dest='ubsan', + action='store_true', + help='Enable UBSAN') + parser.add_argument( + '--enable-ubsan-pointer-overflow', + dest='ubsan_pointer_overflow', + action='store_true', + help='Enable UBSAN pointer overflow check (known failure)') + parser.add_argument( + '--enable-msan', dest='msan', action='store_true', help='Enable MSAN') + parser.add_argument( + '--enable-msan-track-origins', dest='msan_track_origins', + action='store_true', help='Enable MSAN origin tracking') + parser.add_argument( + '--msan-extra-cppflags', + dest='msan_extra_cppflags', + type=str, + default=MSAN_EXTRA_CPPFLAGS, + help="Extra CPPFLAGS for MSAN (default: $MSAN_EXTRA_CPPFLAGS='{}')". + format(MSAN_EXTRA_CPPFLAGS)) + parser.add_argument( + '--msan-extra-cflags', + dest='msan_extra_cflags', + type=str, + default=MSAN_EXTRA_CFLAGS, + help="Extra CFLAGS for MSAN (default: $MSAN_EXTRA_CFLAGS='{}')".format( + MSAN_EXTRA_CFLAGS)) + parser.add_argument( + '--msan-extra-cxxflags', + dest='msan_extra_cxxflags', + type=str, + default=MSAN_EXTRA_CXXFLAGS, + help="Extra CXXFLAGS for MSAN (default: $MSAN_EXTRA_CXXFLAGS='{}')". + format(MSAN_EXTRA_CXXFLAGS)) + parser.add_argument( + '--msan-extra-ldflags', + dest='msan_extra_ldflags', + type=str, + default=MSAN_EXTRA_LDFLAGS, + help="Extra LDFLAGS for MSAN (default: $MSAN_EXTRA_LDFLAGS='{}')". + format(MSAN_EXTRA_LDFLAGS)) + parser.add_argument( + '--enable-sanitize-recover', + dest='sanitize_recover', + action='store_true', + help='Non-fatal sanitizer errors where possible') + parser.add_argument( + '--debug', + dest='debug', + type=int, + default=1, + help='Set DEBUGLEVEL (default: 1)') + parser.add_argument( + '--force-memory-access', + dest='memory_access', + type=int, + default=0, + help='Set MEM_FORCE_MEMORY_ACCESS (default: 0)') + parser.add_argument( + '--fuzz-rng-seed-size', + dest='fuzz_rng_seed_size', + type=int, + default=4, + help='Set FUZZ_RNG_SEED_SIZE (default: 4)') + parser.add_argument( + '--disable-fuzzing-mode', + dest='fuzzing_mode', + action='store_false', + help='Do not define FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION') + parser.add_argument( + '--enable-stateful-fuzzing', + dest='stateful_fuzzing', + action='store_true', + help='Reuse contexts between runs (makes reproduction impossible)') + parser.add_argument( + '--cc', + dest='cc', + type=str, + default=CC, + help="CC (default: $CC='{}')".format(CC)) + parser.add_argument( + '--cxx', + dest='cxx', + type=str, + default=CXX, + help="CXX (default: $CXX='{}')".format(CXX)) + parser.add_argument( + '--cppflags', + dest='cppflags', + type=str, + default=CPPFLAGS, + help="CPPFLAGS (default: $CPPFLAGS='{}')".format(CPPFLAGS)) + parser.add_argument( + '--cflags', + dest='cflags', + type=str, + default=CFLAGS, + help="CFLAGS (default: $CFLAGS='{}')".format(CFLAGS)) + parser.add_argument( + '--cxxflags', + dest='cxxflags', + type=str, + default=CXXFLAGS, + help="CXXFLAGS (default: $CXXFLAGS='{}')".format(CXXFLAGS)) + parser.add_argument( + '--ldflags', + dest='ldflags', + type=str, + default=LDFLAGS, + help="LDFLAGS (default: $LDFLAGS='{}')".format(LDFLAGS)) + parser.add_argument( + '--mflags', + dest='mflags', + type=str, + default=MFLAGS, + help="Extra Make flags (default: $MFLAGS='{}')".format(MFLAGS)) + parser.add_argument( + 'TARGET', + nargs='*', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(ALL_TARGETS)) + ) + args = parser.parse_args(args) + args = parse_env_flags(args, ' '.join( + [args.cppflags, args.cflags, args.cxxflags, args.ldflags])) + + # Check option sanity + if args.msan and (args.asan or args.ubsan): + raise RuntimeError('MSAN may not be used with any other sanitizers') + if args.msan_track_origins and not args.msan: + raise RuntimeError('--enable-msan-track-origins requires MSAN') + if args.ubsan_pointer_overflow and not args.ubsan: + raise RuntimeError('--enable-ubsan-pointer-overflow requires UBSAN') + if args.sanitize_recover and not args.sanitize: + raise RuntimeError('--enable-sanitize-recover but no sanitizers used') + + return args + + +def build(args): + try: + args = build_parser(args) + except Exception as e: + print(e) + return 1 + # The compilation flags we are setting + targets = args.TARGET + cc = args.cc + cxx = args.cxx + cppflags = shlex.split(args.cppflags) + cflags = shlex.split(args.cflags) + ldflags = shlex.split(args.ldflags) + cxxflags = shlex.split(args.cxxflags) + mflags = shlex.split(args.mflags) + # Flags to be added to both cflags and cxxflags + common_flags = [] + + cppflags += [ + '-DDEBUGLEVEL={}'.format(args.debug), + '-DMEM_FORCE_MEMORY_ACCESS={}'.format(args.memory_access), + '-DFUZZ_RNG_SEED_SIZE={}'.format(args.fuzz_rng_seed_size), + ] + + # Set flags for options + assert not (args.fuzzer and args.coverage) + if args.coverage: + common_flags += [ + '-fsanitize-coverage=trace-pc-guard,indirect-calls,trace-cmp' + ] + if args.fuzzer: + common_flags += ['-fsanitize=fuzzer'] + args.lib_fuzzing_engine = '' + + mflags += ['LIB_FUZZING_ENGINE={}'.format(args.lib_fuzzing_engine)] + + if args.sanitize_recover: + recover_flags = ['-fsanitize-recover=all'] + else: + recover_flags = ['-fno-sanitize-recover=all'] + if args.sanitize: + common_flags += recover_flags + + if args.msan: + msan_flags = ['-fsanitize=memory'] + if args.msan_track_origins: + msan_flags += ['-fsanitize-memory-track-origins'] + common_flags += msan_flags + # Append extra MSAN flags (it might require special setup) + cppflags += [args.msan_extra_cppflags] + cflags += [args.msan_extra_cflags] + cxxflags += [args.msan_extra_cxxflags] + ldflags += [args.msan_extra_ldflags] + + if args.asan: + common_flags += ['-fsanitize=address'] + + if args.ubsan: + ubsan_flags = ['-fsanitize=undefined'] + if not args.ubsan_pointer_overflow: + ubsan_flags += overflow_ubsan_flags(cc, cxx) + common_flags += ubsan_flags + + if args.stateful_fuzzing: + cppflags += ['-DSTATEFUL_FUZZING'] + + if args.fuzzing_mode: + cppflags += ['-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION'] + + if args.lib_fuzzing_engine == 'libregression.a': + targets = ['libregression.a'] + targets + + # Append the common flags + cflags += common_flags + cxxflags += common_flags + + # Prepare the flags for Make + cc_str = "CC={}".format(cc) + cxx_str = "CXX={}".format(cxx) + cppflags_str = "CPPFLAGS={}".format(' '.join(cppflags)) + cflags_str = "CFLAGS={}".format(' '.join(cflags)) + cxxflags_str = "CXXFLAGS={}".format(' '.join(cxxflags)) + ldflags_str = "LDFLAGS={}".format(' '.join(ldflags)) + + # Print the flags + print('MFLAGS={}'.format(' '.join(mflags))) + print(cc_str) + print(cxx_str) + print(cppflags_str) + print(cflags_str) + print(cxxflags_str) + print(ldflags_str) + + # Clean and build + clean_cmd = ['make', 'clean'] + mflags + print(' '.join(clean_cmd)) + subprocess.check_call(clean_cmd) + build_cmd = [ + 'make', + cc_str, + cxx_str, + cppflags_str, + cflags_str, + cxxflags_str, + ldflags_str, + ] + mflags + targets + print(' '.join(build_cmd)) + subprocess.check_call(build_cmd) + return 0 + + +def libfuzzer_parser(args): + description = """ + Runs a libfuzzer binary. + Passes all extra arguments to libfuzzer. + The fuzzer should have been build with LIB_FUZZING_ENGINE pointing to + libFuzzer.a. + Generates output in the CORPORA directory, puts crashes in the ARTIFACT + directory, and takes extra input from the SEED directory. + To merge AFL's output pass the SEED as AFL's output directory and pass + '-merge=1'. + """ + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + '--corpora', + type=str, + help='Override the default corpora dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET'))) + parser.add_argument( + '--artifact', + type=str, + help='Override the default artifact dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET-crash'))) + parser.add_argument( + '--seed', + type=str, + help='Override the default seed dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET-seed'))) + parser.add_argument( + 'TARGET', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(TARGETS))) + args, extra = parser.parse_known_args(args) + args.extra = extra + + if args.TARGET and args.TARGET not in TARGETS: + raise RuntimeError('{} is not a valid target'.format(args.TARGET)) + + return args + + +def libfuzzer(target, corpora=None, artifact=None, seed=None, extra_args=None): + if corpora is None: + corpora = abs_join(CORPORA_DIR, target) + if artifact is None: + artifact = abs_join(CORPORA_DIR, '{}-crash'.format(target)) + if seed is None: + seed = abs_join(CORPORA_DIR, '{}-seed'.format(target)) + if extra_args is None: + extra_args = [] + + target = abs_join(FUZZ_DIR, target) + + corpora = [create(corpora)] + artifact = create(artifact) + seed = check(seed) + + corpora += [artifact] + if seed is not None: + corpora += [seed] + + cmd = [target, '-artifact_prefix={}/'.format(artifact)] + cmd += corpora + extra_args + print(' '.join(cmd)) + subprocess.check_call(cmd) + + +def libfuzzer_cmd(args): + try: + args = libfuzzer_parser(args) + except Exception as e: + print(e) + return 1 + libfuzzer(args.TARGET, args.corpora, args.artifact, args.seed, args.extra) + return 0 + + +def afl_parser(args): + description = """ + Runs an afl-fuzz job. + Passes all extra arguments to afl-fuzz. + The fuzzer should have been built with CC/CXX set to the AFL compilers, + and with LIB_FUZZING_ENGINE='libregression.a'. + Takes input from CORPORA and writes output to OUTPUT. + Uses AFL_FUZZ as the binary (set from flag or environment variable). + """ + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + '--corpora', + type=str, + help='Override the default corpora dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET'))) + parser.add_argument( + '--output', + type=str, + help='Override the default AFL output dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET-afl'))) + parser.add_argument( + '--afl-fuzz', + type=str, + default=AFL_FUZZ, + help='AFL_FUZZ (default: $AFL_FUZZ={})'.format(AFL_FUZZ)) + parser.add_argument( + 'TARGET', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(TARGETS))) + args, extra = parser.parse_known_args(args) + args.extra = extra + + if args.TARGET and args.TARGET not in TARGETS: + raise RuntimeError('{} is not a valid target'.format(args.TARGET)) + + if not args.corpora: + args.corpora = abs_join(CORPORA_DIR, args.TARGET) + if not args.output: + args.output = abs_join(CORPORA_DIR, '{}-afl'.format(args.TARGET)) + + return args + + +def afl(args): + try: + args = afl_parser(args) + except Exception as e: + print(e) + return 1 + target = abs_join(FUZZ_DIR, args.TARGET) + + corpora = create(args.corpora) + output = create(args.output) + + cmd = [args.afl_fuzz, '-i', corpora, '-o', output] + args.extra + cmd += [target, '@@'] + print(' '.join(cmd)) + subprocess.call(cmd) + return 0 + + +def regression(args): + try: + description = """ + Runs one or more regression tests. + The fuzzer should have been built with with + LIB_FUZZING_ENGINE='libregression.a'. + Takes input from CORPORA. + """ + args = targets_parser(args, description) + except Exception as e: + print(e) + return 1 + for target in args.TARGET: + corpora = create(abs_join(CORPORA_DIR, target)) + target = abs_join(FUZZ_DIR, target) + cmd = [target, corpora] + print(' '.join(cmd)) + subprocess.check_call(cmd) + return 0 + + +def gen_parser(args): + description = """ + Generate a seed corpus appropriate for TARGET with data generated with + decodecorpus. + The fuzz inputs are prepended with a seed before the zstd data, so the + output of decodecorpus shouldn't be used directly. + Generates NUMBER samples prepended with FUZZ_RNG_SEED_SIZE random bytes and + puts the output in SEED. + DECODECORPUS is the decodecorpus binary, and must already be built. + """ + parser = argparse.ArgumentParser(prog=args.pop(0), description=description) + parser.add_argument( + '--number', + '-n', + type=int, + default=100, + help='Number of samples to generate') + parser.add_argument( + '--max-size-log', + type=int, + default=18, + help='Maximum sample size to generate') + parser.add_argument( + '--seed', + type=str, + help='Override the default seed dir (default: {})'.format( + abs_join(CORPORA_DIR, 'TARGET-seed'))) + parser.add_argument( + '--decodecorpus', + type=str, + default=DECODECORPUS, + help="decodecorpus binary (default: $DECODECORPUS='{}')".format( + DECODECORPUS)) + parser.add_argument( + '--zstd', + type=str, + default=ZSTD, + help="zstd binary (default: $ZSTD='{}')".format(ZSTD)) + parser.add_argument( + '--fuzz-rng-seed-size', + type=int, + default=4, + help="FUZZ_RNG_SEED_SIZE used for generate the samples (must match)" + ) + parser.add_argument( + 'TARGET', + type=str, + help='Fuzz target(s) to build {{{}}}'.format(', '.join(TARGETS))) + args, extra = parser.parse_known_args(args) + args.extra = extra + + if args.TARGET and args.TARGET not in TARGETS: + raise RuntimeError('{} is not a valid target'.format(args.TARGET)) + + if not args.seed: + args.seed = abs_join(CORPORA_DIR, '{}-seed'.format(args.TARGET)) + + if not os.path.isfile(args.decodecorpus): + raise RuntimeError("{} is not a file run 'make -C {} decodecorpus'". + format(args.decodecorpus, abs_join(FUZZ_DIR, '..'))) + + return args + + +def gen(args): + try: + args = gen_parser(args) + except Exception as e: + print(e) + return 1 + + seed = create(args.seed) + with tmpdir() as compressed, tmpdir() as decompressed, tmpdir() as dict: + info = TARGET_INFO[args.TARGET] + + if info.input_type == InputType.DICTIONARY_DATA: + number = max(args.number, 1000) + else: + number = args.number + cmd = [ + args.decodecorpus, + '-n{}'.format(args.number), + '-p{}/'.format(compressed), + '-o{}'.format(decompressed), + ] + + if info.frame_type == FrameType.BLOCK: + cmd += [ + '--gen-blocks', + '--max-block-size-log={}'.format(min(args.max_size_log, 17)) + ] + else: + cmd += ['--max-content-size-log={}'.format(args.max_size_log)] + + print(' '.join(cmd)) + subprocess.check_call(cmd) + + if info.input_type == InputType.RAW_DATA: + print('using decompressed data in {}'.format(decompressed)) + samples = decompressed + elif info.input_type == InputType.COMPRESSED_DATA: + print('using compressed data in {}'.format(compressed)) + samples = compressed + else: + assert info.input_type == InputType.DICTIONARY_DATA + print('making dictionary data from {}'.format(decompressed)) + samples = dict + min_dict_size_log = 9 + max_dict_size_log = max(min_dict_size_log + 1, args.max_size_log) + for dict_size_log in range(min_dict_size_log, max_dict_size_log): + dict_size = 1 << dict_size_log + cmd = [ + args.zstd, + '--train', + '-r', decompressed, + '--maxdict={}'.format(dict_size), + '-o', abs_join(dict, '{}.zstd-dict'.format(dict_size)) + ] + print(' '.join(cmd)) + subprocess.check_call(cmd) + + # Copy the samples over and prepend the RNG seeds + for name in os.listdir(samples): + samplename = abs_join(samples, name) + outname = abs_join(seed, name) + with open(samplename, 'rb') as sample: + with open(outname, 'wb') as out: + CHUNK_SIZE = 131072 + chunk = sample.read(CHUNK_SIZE) + while len(chunk) > 0: + out.write(chunk) + chunk = sample.read(CHUNK_SIZE) + return 0 + + +def minimize(args): + try: + description = """ + Runs a libfuzzer fuzzer with -merge=1 to build a minimal corpus in + TARGET_seed_corpus. All extra args are passed to libfuzzer. + """ + args = targets_parser(args, description) + except Exception as e: + print(e) + return 1 + + for target in args.TARGET: + # Merge the corpus + anything else into the seed_corpus + corpus = abs_join(CORPORA_DIR, target) + seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target)) + extra_args = [corpus, "-merge=1"] + args.extra + libfuzzer(target, corpora=seed_corpus, extra_args=extra_args) + seeds = set(os.listdir(seed_corpus)) + # Copy all crashes directly into the seed_corpus if not already present + crashes = abs_join(CORPORA_DIR, '{}-crash'.format(target)) + for crash in os.listdir(crashes): + if crash not in seeds: + shutil.copy(abs_join(crashes, crash), seed_corpus) + seeds.add(crash) + + +def zip_cmd(args): + try: + description = """ + Zips up the seed corpus. + """ + args = targets_parser(args, description) + except Exception as e: + print(e) + return 1 + + for target in args.TARGET: + # Zip the seed_corpus + seed_corpus = abs_join(CORPORA_DIR, "{}_seed_corpus".format(target)) + zip_file = "{}.zip".format(seed_corpus) + cmd = ["zip", "-r", "-q", "-j", "-9", zip_file, "."] + print(' '.join(cmd)) + subprocess.check_call(cmd, cwd=seed_corpus) + + +def list_cmd(args): + print("\n".join(TARGETS)) + + +def short_help(args): + name = args[0] + print("Usage: {} [OPTIONS] COMMAND [ARGS]...\n".format(name)) + + +def help(args): + short_help(args) + print("\tfuzzing helpers (select a command and pass -h for help)\n") + print("Options:") + print("\t-h, --help\tPrint this message") + print("") + print("Commands:") + print("\tbuild\t\tBuild a fuzzer") + print("\tlibfuzzer\tRun a libFuzzer fuzzer") + print("\tafl\t\tRun an AFL fuzzer") + print("\tregression\tRun a regression test") + print("\tgen\t\tGenerate a seed corpus for a fuzzer") + print("\tminimize\tMinimize the test corpora") + print("\tzip\t\tZip the minimized corpora up") + print("\tlist\t\tList the available targets") + + +def main(): + args = sys.argv + if len(args) < 2: + help(args) + return 1 + if args[1] == '-h' or args[1] == '--help' or args[1] == '-H': + help(args) + return 1 + command = args.pop(1) + args[0] = "{} {}".format(args[0], command) + if command == "build": + return build(args) + if command == "libfuzzer": + return libfuzzer_cmd(args) + if command == "regression": + return regression(args) + if command == "afl": + return afl(args) + if command == "gen": + return gen(args) + if command == "minimize": + return minimize(args) + if command == "zip": + return zip_cmd(args) + if command == "list": + return list_cmd(args) + short_help(args) + print("Error: No such command {} (pass -h for help)".format(command)) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/zstd/tests/fuzz/fuzz_data_producer.c b/src/zstd/tests/fuzz/fuzz_data_producer.c new file mode 100644 index 000000000..6518af309 --- /dev/null +++ b/src/zstd/tests/fuzz/fuzz_data_producer.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "fuzz_data_producer.h" + +struct FUZZ_dataProducer_s{ + const uint8_t *data; + size_t size; +}; + +FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size) { + FUZZ_dataProducer_t *producer = FUZZ_malloc(sizeof(FUZZ_dataProducer_t)); + + producer->data = data; + producer->size = size; + return producer; +} + +void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer) { free(producer); } + +uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t min, + uint32_t max) { + FUZZ_ASSERT(min <= max); + + uint32_t range = max - min; + uint32_t rolling = range; + uint32_t result = 0; + + while (rolling > 0 && producer->size > 0) { + uint8_t next = *(producer->data + producer->size - 1); + producer->size -= 1; + result = (result << 8) | next; + rolling >>= 8; + } + + if (range == 0xffffffff) { + return result; + } + + return min + result % (range + 1); +} + +uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer) { + return FUZZ_dataProducer_uint32Range(producer, 0, 0xffffffff); +} + +int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, + int32_t min, int32_t max) +{ + FUZZ_ASSERT(min <= max); + + if (min < 0) + return (int)FUZZ_dataProducer_uint32Range(producer, 0, max - min) + min; + + return FUZZ_dataProducer_uint32Range(producer, min, max); +} + +size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer){ + return producer->size; +} + +size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize) +{ + newSize = newSize > producer->size ? producer->size : newSize; + + size_t remaining = producer->size - newSize; + producer->data = producer->data + remaining; + producer->size = newSize; + return remaining; +} + +size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer) +{ + size_t producerSliceSize = FUZZ_dataProducer_uint32Range( + producer, 0, producer->size); + return FUZZ_dataProducer_contract(producer, producerSliceSize); +} diff --git a/src/zstd/tests/fuzz/fuzz_data_producer.h b/src/zstd/tests/fuzz/fuzz_data_producer.h new file mode 100644 index 000000000..41e0b52d5 --- /dev/null +++ b/src/zstd/tests/fuzz/fuzz_data_producer.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * Helper APIs for generating random data from input data stream. + The producer reads bytes from the end of the input and appends them together + to generate a random number in the requested range. If it runs out of input + data, it will keep returning the same value (min) over and over again. + + */ + +#ifndef FUZZ_DATA_PRODUCER_H +#define FUZZ_DATA_PRODUCER_H + +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +#include "fuzz_helpers.h" + +/* Struct used for maintaining the state of the data */ +typedef struct FUZZ_dataProducer_s FUZZ_dataProducer_t; + +/* Returns a data producer state struct. Use for producer initialization. */ +FUZZ_dataProducer_t *FUZZ_dataProducer_create(const uint8_t *data, size_t size); + +/* Frees the data producer */ +void FUZZ_dataProducer_free(FUZZ_dataProducer_t *producer); + +/* Returns value between [min, max] */ +uint32_t FUZZ_dataProducer_uint32Range(FUZZ_dataProducer_t *producer, uint32_t min, + uint32_t max); + +/* Returns a uint32 value */ +uint32_t FUZZ_dataProducer_uint32(FUZZ_dataProducer_t *producer); + +/* Returns a signed value between [min, max] */ +int32_t FUZZ_dataProducer_int32Range(FUZZ_dataProducer_t *producer, + int32_t min, int32_t max); + +/* Returns the size of the remaining bytes of data in the producer */ +size_t FUZZ_dataProducer_remainingBytes(FUZZ_dataProducer_t *producer); + +/* Restricts the producer to only the last newSize bytes of data. +If newSize > current data size, nothing happens. Returns the number of bytes +the producer won't use anymore, after contracting. */ +size_t FUZZ_dataProducer_contract(FUZZ_dataProducer_t *producer, size_t newSize); + +/* Restricts the producer to use only the last X bytes of data, where X is + a random number in the interval [0, data_size]. Returns the size of the + remaining data the producer won't use anymore (the prefix). */ +size_t FUZZ_dataProducer_reserveDataPrefix(FUZZ_dataProducer_t *producer); +#endif // FUZZ_DATA_PRODUCER_H diff --git a/src/zstd/tests/fuzz/fuzz_helpers.c b/src/zstd/tests/fuzz/fuzz_helpers.c new file mode 100644 index 000000000..b80dc7571 --- /dev/null +++ b/src/zstd/tests/fuzz/fuzz_helpers.c @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#include "fuzz_helpers.h" + +#include <stddef.h> +#include <stdlib.h> +#include <string.h> + +void* FUZZ_malloc(size_t size) +{ + if (size > 0) { + void* const mem = malloc(size); + FUZZ_ASSERT(mem); + return mem; + } + return NULL; +} + +int FUZZ_memcmp(void const* lhs, void const* rhs, size_t size) +{ + if (size == 0) { + return 0; + } + return memcmp(lhs, rhs, size); +}
\ No newline at end of file diff --git a/src/zstd/tests/fuzz/fuzz_helpers.h b/src/zstd/tests/fuzz/fuzz_helpers.h new file mode 100644 index 000000000..cde2c4ea7 --- /dev/null +++ b/src/zstd/tests/fuzz/fuzz_helpers.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * Helper functions for fuzzing. + */ + +#ifndef FUZZ_HELPERS_H +#define FUZZ_HELPERS_H + +#include "debug.h" +#include "fuzz.h" +#include "xxhash.h" +#include "zstd.h" +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#define MAX(a, b) ((a) > (b) ? (a) : (b)) + +#define FUZZ_QUOTE_IMPL(str) #str +#define FUZZ_QUOTE(str) FUZZ_QUOTE_IMPL(str) + +/** + * Asserts for fuzzing that are always enabled. + */ +#define FUZZ_ASSERT_MSG(cond, msg) \ + ((cond) ? (void)0 \ + : (fprintf(stderr, "%s: %u: Assertion: `%s' failed. %s\n", __FILE__, \ + __LINE__, FUZZ_QUOTE(cond), (msg)), \ + abort())) +#define FUZZ_ASSERT(cond) FUZZ_ASSERT_MSG((cond), ""); +#define FUZZ_ZASSERT(code) \ + FUZZ_ASSERT_MSG(!ZSTD_isError(code), ZSTD_getErrorName(code)) + +#if defined(__GNUC__) +#define FUZZ_STATIC static __inline __attribute__((unused)) +#elif defined(__cplusplus) || \ + (defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +#define FUZZ_STATIC static inline +#elif defined(_MSC_VER) +#define FUZZ_STATIC static __inline +#else +#define FUZZ_STATIC static +#endif + +/** + * malloc except return NULL for zero sized data and FUZZ_ASSERT + * that malloc doesn't fail. + */ +void* FUZZ_malloc(size_t size); + +/** + * memcmp but accepts NULL. + */ +int FUZZ_memcmp(void const* lhs, void const* rhs, size_t size); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/zstd/tests/fuzz/raw_dictionary_round_trip.c b/src/zstd/tests/fuzz/raw_dictionary_round_trip.c new file mode 100644 index 000000000..08e5fd9ed --- /dev/null +++ b/src/zstd/tests/fuzz/raw_dictionary_round_trip.c @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target performs a zstd round-trip test (compress & decompress) with + * a raw content dictionary, compares the result with the original, and calls + * abort() on corruption. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; + +static size_t roundTripTest(void *result, size_t resultCapacity, + void *compressed, size_t compressedCapacity, + const void *src, size_t srcSize, + const void *dict, size_t dictSize, + FUZZ_dataProducer_t *producer) +{ + ZSTD_dictContentType_e const dictContentType = ZSTD_dct_rawContent; + int const refPrefix = FUZZ_dataProducer_uint32Range(producer, 0, 1) != 0; + size_t cSize; + + FUZZ_setRandomParameters(cctx, srcSize, producer); + /* Disable checksum so we can use sizes smaller than compress bound. */ + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 0)); + if (refPrefix) + FUZZ_ZASSERT(ZSTD_CCtx_refPrefix_advanced( + cctx, dict, dictSize, + ZSTD_dct_rawContent)); + else + FUZZ_ZASSERT(ZSTD_CCtx_loadDictionary_advanced( + cctx, dict, dictSize, + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + ZSTD_dct_rawContent)); + cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + FUZZ_ZASSERT(cSize); + + if (refPrefix) + FUZZ_ZASSERT(ZSTD_DCtx_refPrefix_advanced( + dctx, dict, dictSize, + dictContentType)); + else + FUZZ_ZASSERT(ZSTD_DCtx_loadDictionary_advanced( + dctx, dict, dictSize, + (ZSTD_dictLoadMethod_e)FUZZ_dataProducer_uint32Range(producer, 0, 1), + dictContentType)); + { + size_t const ret = ZSTD_decompressDCtx( + dctx, result, resultCapacity, compressed, cSize); + return ret; + } +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + uint8_t const* const srcBuf = src; + size_t const srcSize = FUZZ_dataProducer_uint32Range(producer, 0, size); + uint8_t const* const dictBuf = srcBuf + srcSize; + size_t const dictSize = size - srcSize; + size_t const decompSize = srcSize; + void* const decompBuf = FUZZ_malloc(decompSize); + size_t compSize = ZSTD_compressBound(srcSize); + void* compBuf; + /* Half of the time fuzz with a 1 byte smaller output size. + * This will still succeed because we force the checksum to be disabled, + * giving us 4 bytes of overhead. + */ + compSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); + compBuf = FUZZ_malloc(compSize); + + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + { + size_t const result = + roundTripTest(decompBuf, decompSize, compBuf, compSize, srcBuf, srcSize, dictBuf, dictSize, producer); + FUZZ_ZASSERT(result); + FUZZ_ASSERT_MSG(result == srcSize, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, decompBuf, srcSize), "Corruption!"); + } + free(decompBuf); + free(compBuf); + FUZZ_dataProducer_free(producer); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/regression_driver.c b/src/zstd/tests/fuzz/regression_driver.c new file mode 100644 index 000000000..8180ca822 --- /dev/null +++ b/src/zstd/tests/fuzz/regression_driver.c @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "fuzz.h" +#include "fuzz_helpers.h" +#include "util.h" +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +int main(int argc, char const **argv) { + size_t const kMaxFileSize = (size_t)1 << 27; + int const kFollowLinks = 1; + FileNamesTable* files; + const char** const fnTable = argv + 1; + uint8_t *buffer = NULL; + size_t bufferSize = 0; + unsigned i; + unsigned numFilesTested = 0; + int ret = 0; + + { + unsigned const numFiles = (unsigned)(argc - 1); +#ifdef UTIL_HAS_CREATEFILELIST + files = UTIL_createExpandedFNT(fnTable, numFiles, kFollowLinks); +#else + files = UTIL_createFNT_fromROTable(fnTable, numFiles); + assert(numFiles == files->tableSize); +#endif + } + if (!files) { + fprintf(stderr, "ERROR: Failed to create file names table\n"); + return 1; + } + if (files->tableSize == 0) + fprintf(stderr, "WARNING: No files passed to %s\n", argv[0]); + for (i = 0; i < files->tableSize; ++i) { + char const *fileName = files->fileNames[i]; + DEBUGLOG(3, "Running %s", fileName); + size_t const fileSize = UTIL_getFileSize(fileName); + size_t readSize; + FILE *file; + + /* Check that it is a regular file, and that the fileSize is valid. + * If it is not a regular file, then it may have been deleted since we + * constructed the list, so just skip it, but return an error exit code. + */ + if (!UTIL_isRegularFile(fileName)) { + ret = 1; + continue; + } + FUZZ_ASSERT_MSG(fileSize <= kMaxFileSize, fileName); + /* Ensure we have a large enough buffer allocated */ + if (fileSize > bufferSize) { + free(buffer); + buffer = (uint8_t *)malloc(fileSize); + FUZZ_ASSERT_MSG(buffer, fileName); + bufferSize = fileSize; + } + /* Open the file */ + file = fopen(fileName, "rb"); + FUZZ_ASSERT_MSG(file, fileName); + /* Read the file */ + readSize = fread(buffer, 1, fileSize, file); + FUZZ_ASSERT_MSG(readSize == fileSize, fileName); + /* Close the file */ + fclose(file); + /* Run the fuzz target */ + LLVMFuzzerTestOneInput(buffer, fileSize); + ++numFilesTested; + } + fprintf(stderr, "Tested %u files: ", numFilesTested); + if (ret == 0) { + fprintf(stderr, "Success!\n"); + } else { + fprintf(stderr, "Failure!\n"); + } + free(buffer); + UTIL_freeFileNamesTable(files); + return ret; +} diff --git a/src/zstd/tests/fuzz/simple_compress.c b/src/zstd/tests/fuzz/simple_compress.c new file mode 100644 index 000000000..b64f373eb --- /dev/null +++ b/src/zstd/tests/fuzz/simple_compress.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target attempts to comprss the fuzzed data with the simple + * compression function with an output buffer that may be too small to + * ensure that the compressor never crashes. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include "fuzz_helpers.h" +#include "zstd.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_CCtx *cctx = NULL; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + size_t const maxSize = ZSTD_compressBound(size); + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, maxSize); + + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); + + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + + void *rBuf = FUZZ_malloc(bufSize); + ZSTD_compressCCtx(cctx, rBuf, bufSize, src, size, cLevel); + free(rBuf); + FUZZ_dataProducer_free(producer); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/simple_decompress.c b/src/zstd/tests/fuzz/simple_decompress.c new file mode 100644 index 000000000..c3903ce8b --- /dev/null +++ b/src/zstd/tests/fuzz/simple_decompress.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target attempts to decompress the fuzzed data with the simple + * decompression function to ensure the decompressor never crashes. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include "fuzz_helpers.h" +#include "zstd.h" +#include "fuzz_data_producer.h" + +static ZSTD_DCtx *dctx = NULL; + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + size_t const bufSize = FUZZ_dataProducer_uint32Range(producer, 0, 10 * size); + void *rBuf = FUZZ_malloc(bufSize); + + ZSTD_decompressDCtx(dctx, rBuf, bufSize, src, size); + free(rBuf); + + FUZZ_dataProducer_free(producer); + +#ifndef STATEFUL_FUZZING + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/simple_round_trip.c b/src/zstd/tests/fuzz/simple_round_trip.c new file mode 100644 index 000000000..2f008d06a --- /dev/null +++ b/src/zstd/tests/fuzz/simple_round_trip.c @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target performs a zstd round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +static ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; + +static size_t roundTripTest(void *result, size_t resultCapacity, + void *compressed, size_t compressedCapacity, + const void *src, size_t srcSize, + FUZZ_dataProducer_t *producer) +{ + size_t cSize; + size_t dSize; + int targetCBlockSize = 0; + if (FUZZ_dataProducer_uint32Range(producer, 0, 1)) { + FUZZ_setRandomParameters(cctx, srcSize, producer); + cSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + FUZZ_ZASSERT(ZSTD_CCtx_getParameter(cctx, ZSTD_c_targetCBlockSize, &targetCBlockSize)); + } else { + int const cLevel = FUZZ_dataProducer_int32Range(producer, kMinClevel, kMaxClevel); + + cSize = ZSTD_compressCCtx( + cctx, compressed, compressedCapacity, src, srcSize, cLevel); + } + FUZZ_ZASSERT(cSize); + dSize = ZSTD_decompressDCtx(dctx, result, resultCapacity, compressed, cSize); + FUZZ_ZASSERT(dSize); + /* When superblock is enabled make sure we don't expand the block more than expected. */ + if (targetCBlockSize != 0) { + size_t normalCSize; + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 0)); + normalCSize = ZSTD_compress2(cctx, compressed, compressedCapacity, src, srcSize); + FUZZ_ZASSERT(normalCSize); + { + size_t const bytesPerBlock = 3 /* block header */ + + 5 /* Literal header */ + + 6 /* Huffman jump table */ + + 3 /* number of sequences */ + + 1 /* symbol compression modes */; + size_t const expectedExpansion = bytesPerBlock * (1 + (normalCSize / MAX(1, targetCBlockSize))); + size_t const allowedExpansion = (srcSize >> 3) + 5 * expectedExpansion + 10; + FUZZ_ASSERT(cSize <= normalCSize + allowedExpansion); + } + } + return dSize; +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + size_t const rBufSize = size; + void* rBuf = FUZZ_malloc(rBufSize); + size_t cBufSize = ZSTD_compressBound(size); + void* cBuf; + + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + /* Half of the time fuzz with a 1 byte smaller output size. + * This will still succeed because we don't use a dictionary, so the dictID + * field is empty, giving us 4 bytes of overhead. + */ + cBufSize -= FUZZ_dataProducer_uint32Range(producer, 0, 1); + + cBuf = FUZZ_malloc(cBufSize); + + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + { + size_t const result = + roundTripTest(rBuf, rBufSize, cBuf, cBufSize, src, size, producer); + FUZZ_ZASSERT(result); + FUZZ_ASSERT_MSG(result == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!"); + } + free(rBuf); + free(cBuf); + FUZZ_dataProducer_free(producer); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/stream_decompress.c b/src/zstd/tests/fuzz/stream_decompress.c new file mode 100644 index 000000000..25901b1eb --- /dev/null +++ b/src/zstd/tests/fuzz/stream_decompress.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target attempts to decompress the fuzzed data with the simple + * decompression function to ensure the decompressor never crashes. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include "fuzz_helpers.h" +#include "zstd.h" +#include "fuzz_data_producer.h" + +static size_t const kBufSize = ZSTD_BLOCKSIZE_MAX; + +static ZSTD_DStream *dstream = NULL; +static void* buf = NULL; +uint32_t seed; + +static ZSTD_outBuffer makeOutBuffer(FUZZ_dataProducer_t *producer, uint32_t min) +{ + ZSTD_outBuffer buffer = { buf, 0, 0 }; + + buffer.size = (FUZZ_dataProducer_uint32Range(producer, min, kBufSize)); + FUZZ_ASSERT(buffer.size <= kBufSize); + + if (buffer.size == 0) { + buffer.dst = NULL; + } + + return buffer; +} + +static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, + FUZZ_dataProducer_t *producer, + uint32_t min) +{ + ZSTD_inBuffer buffer = { *src, 0, 0 }; + + FUZZ_ASSERT(*size > 0); + buffer.size = (FUZZ_dataProducer_uint32Range(producer, min, *size)); + FUZZ_ASSERT(buffer.size <= *size); + *src += buffer.size; + *size -= buffer.size; + + if (buffer.size == 0) { + buffer.src = NULL; + } + + return buffer; +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + /* Guarantee forward progress by refusing to generate 2 zero sized + * buffers in a row. */ + int prevInWasZero = 0; + int prevOutWasZero = 0; + int stableOutBuffer; + ZSTD_outBuffer out; + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + /* Allocate all buffers and contexts if not already allocated */ + if (!buf) { + buf = FUZZ_malloc(kBufSize); + } + + if (!dstream) { + dstream = ZSTD_createDStream(); + FUZZ_ASSERT(dstream); + } else { + FUZZ_ZASSERT(ZSTD_DCtx_reset(dstream, ZSTD_reset_session_only)); + } + + stableOutBuffer = FUZZ_dataProducer_uint32Range(producer, 0, 10) == 5; + if (stableOutBuffer) { + FUZZ_ZASSERT(ZSTD_DCtx_setParameter(dstream, ZSTD_d_stableOutBuffer, 1)); + out.dst = buf; + out.size = kBufSize; + out.pos = 0; + } + + while (size > 0) { + ZSTD_inBuffer in = makeInBuffer(&src, &size, producer, prevInWasZero ? 1 : 0); + prevInWasZero = in.size == 0; + while (in.pos != in.size) { + if (!stableOutBuffer || prevOutWasZero || FUZZ_dataProducer_uint32Range(producer, 0, 100) == 55) { + out = makeOutBuffer(producer, prevOutWasZero ? 1 : 0); + } + prevOutWasZero = out.size == 0; + size_t const rc = ZSTD_decompressStream(dstream, &out, &in); + if (ZSTD_isError(rc)) goto error; + } + } + +error: +#ifndef STATEFUL_FUZZING + ZSTD_freeDStream(dstream); dstream = NULL; +#endif + FUZZ_dataProducer_free(producer); + return 0; +} diff --git a/src/zstd/tests/fuzz/stream_round_trip.c b/src/zstd/tests/fuzz/stream_round_trip.c new file mode 100644 index 000000000..286d3871b --- /dev/null +++ b/src/zstd/tests/fuzz/stream_round_trip.c @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target performs a zstd round-trip test (compress & decompress), + * compares the result with the original, and calls abort() on corruption. + */ + +#define ZSTD_STATIC_LINKING_ONLY + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" +#include "fuzz_data_producer.h" + +ZSTD_CCtx *cctx = NULL; +static ZSTD_DCtx *dctx = NULL; +static uint8_t* cBuf = NULL; +static uint8_t* rBuf = NULL; +static size_t bufSize = 0; + +static ZSTD_outBuffer makeOutBuffer(uint8_t *dst, size_t capacity, + FUZZ_dataProducer_t *producer) +{ + ZSTD_outBuffer buffer = { dst, 0, 0 }; + + FUZZ_ASSERT(capacity > 0); + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, capacity)); + FUZZ_ASSERT(buffer.size <= capacity); + + return buffer; +} + +static ZSTD_inBuffer makeInBuffer(const uint8_t **src, size_t *size, + FUZZ_dataProducer_t *producer) +{ + ZSTD_inBuffer buffer = { *src, 0, 0 }; + + FUZZ_ASSERT(*size > 0); + buffer.size = (FUZZ_dataProducer_uint32Range(producer, 1, *size)); + FUZZ_ASSERT(buffer.size <= *size); + *src += buffer.size; + *size -= buffer.size; + + return buffer; +} + +static size_t compress(uint8_t *dst, size_t capacity, + const uint8_t *src, size_t srcSize, + FUZZ_dataProducer_t *producer) +{ + size_t dstSize = 0; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + FUZZ_setRandomParameters(cctx, srcSize, producer); + + while (srcSize > 0) { + ZSTD_inBuffer in = makeInBuffer(&src, &srcSize, producer); + /* Mode controls the action. If mode == -1 we pick a new mode */ + int mode = -1; + while (in.pos < in.size || mode != -1) { + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); + /* Previous action finished, pick a new mode. */ + if (mode == -1) mode = FUZZ_dataProducer_uint32Range(producer, 0, 9); + switch (mode) { + case 0: /* fall-through */ + case 1: /* fall-through */ + case 2: { + size_t const ret = + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush); + FUZZ_ZASSERT(ret); + if (ret == 0) + mode = -1; + break; + } + case 3: { + size_t ret = + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); + FUZZ_ZASSERT(ret); + /* Reset the compressor when the frame is finished */ + if (ret == 0) { + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + if (FUZZ_dataProducer_uint32Range(producer, 0, 7) == 0) { + size_t const remaining = in.size - in.pos; + FUZZ_setRandomParameters(cctx, remaining, producer); + } + mode = -1; + } + break; + } + case 4: { + ZSTD_inBuffer nullIn = { NULL, 0, 0 }; + ZSTD_outBuffer nullOut = { NULL, 0, 0 }; + size_t const ret = ZSTD_compressStream2(cctx, &nullOut, &nullIn, ZSTD_e_continue); + FUZZ_ZASSERT(ret); + } + /* fall-through */ + default: { + size_t const ret = + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); + FUZZ_ZASSERT(ret); + mode = -1; + } + } + dst += out.pos; + dstSize += out.pos; + capacity -= out.pos; + } + } + for (;;) { + ZSTD_inBuffer in = {NULL, 0, 0}; + ZSTD_outBuffer out = makeOutBuffer(dst, capacity, producer); + size_t const ret = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); + FUZZ_ZASSERT(ret); + + dst += out.pos; + dstSize += out.pos; + capacity -= out.pos; + if (ret == 0) + break; + } + return dstSize; +} + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + size_t neededBufSize; + + /* Give a random portion of src data to the producer, to use for + parameter generation. The rest will be used for (de)compression */ + FUZZ_dataProducer_t *producer = FUZZ_dataProducer_create(src, size); + size = FUZZ_dataProducer_reserveDataPrefix(producer); + + neededBufSize = ZSTD_compressBound(size) * 15; + + /* Allocate all buffers and contexts if not already allocated */ + if (neededBufSize > bufSize) { + free(cBuf); + free(rBuf); + cBuf = (uint8_t*)FUZZ_malloc(neededBufSize); + rBuf = (uint8_t*)FUZZ_malloc(neededBufSize); + bufSize = neededBufSize; + } + if (!cctx) { + cctx = ZSTD_createCCtx(); + FUZZ_ASSERT(cctx); + } + if (!dctx) { + dctx = ZSTD_createDCtx(); + FUZZ_ASSERT(dctx); + } + + { + size_t const cSize = compress(cBuf, neededBufSize, src, size, producer); + size_t const rSize = + ZSTD_decompressDCtx(dctx, rBuf, neededBufSize, cBuf, cSize); + FUZZ_ZASSERT(rSize); + FUZZ_ASSERT_MSG(rSize == size, "Incorrect regenerated size"); + FUZZ_ASSERT_MSG(!FUZZ_memcmp(src, rBuf, size), "Corruption!"); + } + + FUZZ_dataProducer_free(producer); +#ifndef STATEFUL_FUZZING + ZSTD_freeCCtx(cctx); cctx = NULL; + ZSTD_freeDCtx(dctx); dctx = NULL; +#endif + return 0; +} diff --git a/src/zstd/tests/fuzz/zstd_frame_info.c b/src/zstd/tests/fuzz/zstd_frame_info.c new file mode 100644 index 000000000..876a74e9a --- /dev/null +++ b/src/zstd/tests/fuzz/zstd_frame_info.c @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This fuzz target fuzzes all of the helper functions that consume compressed + * input. + */ + +#include <stddef.h> +#include <stdlib.h> +#include <stdio.h> +#include "fuzz_helpers.h" +#include "zstd_helpers.h" + +int LLVMFuzzerTestOneInput(const uint8_t *src, size_t size) +{ + ZSTD_frameHeader zfh; + if (size == 0) { + src = NULL; + } + /* You can fuzz any helper functions here that are fast, and take zstd + * compressed data as input. E.g. don't expect the input to be a dictionary, + * so don't fuzz ZSTD_getDictID_fromDict(). + */ + ZSTD_getFrameContentSize(src, size); + ZSTD_getDecompressedSize(src, size); + ZSTD_findFrameCompressedSize(src, size); + ZSTD_getDictID_fromFrame(src, size); + ZSTD_findDecompressedSize(src, size); + ZSTD_decompressBound(src, size); + ZSTD_frameHeaderSize(src, size); + ZSTD_isFrame(src, size); + ZSTD_getFrameHeader(&zfh, src, size); + ZSTD_getFrameHeader_advanced(&zfh, src, size, ZSTD_f_zstd1); + return 0; +} diff --git a/src/zstd/tests/fuzz/zstd_helpers.c b/src/zstd/tests/fuzz/zstd_helpers.c new file mode 100644 index 000000000..5680bd628 --- /dev/null +++ b/src/zstd/tests/fuzz/zstd_helpers.c @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#define ZSTD_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY + +#include <string.h> + +#include "zstd_helpers.h" +#include "fuzz_helpers.h" +#include "zstd.h" +#include "zdict.h" + +const int kMinClevel = -3; +const int kMaxClevel = 19; + +static void set(ZSTD_CCtx *cctx, ZSTD_cParameter param, int value) +{ + FUZZ_ZASSERT(ZSTD_CCtx_setParameter(cctx, param, value)); +} + +static void setRand(ZSTD_CCtx *cctx, ZSTD_cParameter param, unsigned min, + unsigned max, FUZZ_dataProducer_t *producer) { + unsigned const value = FUZZ_dataProducer_uint32Range(producer, min, max); + set(cctx, param, value); +} + +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer) +{ + /* Select compression parameters */ + ZSTD_compressionParameters cParams; + cParams.windowLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_WINDOWLOG_MIN, 15); + cParams.hashLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_HASHLOG_MIN, 15); + cParams.chainLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_CHAINLOG_MIN, 16); + cParams.searchLog = FUZZ_dataProducer_uint32Range(producer, ZSTD_SEARCHLOG_MIN, 9); + cParams.minMatch = FUZZ_dataProducer_uint32Range(producer, ZSTD_MINMATCH_MIN, + ZSTD_MINMATCH_MAX); + cParams.targetLength = FUZZ_dataProducer_uint32Range(producer, 0, 512); + cParams.strategy = FUZZ_dataProducer_uint32Range(producer, ZSTD_STRATEGY_MIN, ZSTD_STRATEGY_MAX); + return ZSTD_adjustCParams(cParams, srcSize, 0); +} + +ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer) +{ + /* Select frame parameters */ + ZSTD_frameParameters fParams; + fParams.contentSizeFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); + fParams.checksumFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); + fParams.noDictIDFlag = FUZZ_dataProducer_uint32Range(producer, 0, 1); + return fParams; +} + +ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer) +{ + ZSTD_parameters params; + params.cParams = FUZZ_randomCParams(srcSize, producer); + params.fParams = FUZZ_randomFParams(producer); + return params; +} + +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer) +{ + ZSTD_compressionParameters cParams = FUZZ_randomCParams(srcSize, producer); + set(cctx, ZSTD_c_windowLog, cParams.windowLog); + set(cctx, ZSTD_c_hashLog, cParams.hashLog); + set(cctx, ZSTD_c_chainLog, cParams.chainLog); + set(cctx, ZSTD_c_searchLog, cParams.searchLog); + set(cctx, ZSTD_c_minMatch, cParams.minMatch); + set(cctx, ZSTD_c_targetLength, cParams.targetLength); + set(cctx, ZSTD_c_strategy, cParams.strategy); + /* Select frame parameters */ + setRand(cctx, ZSTD_c_contentSizeFlag, 0, 1, producer); + setRand(cctx, ZSTD_c_checksumFlag, 0, 1, producer); + setRand(cctx, ZSTD_c_dictIDFlag, 0, 1, producer); + /* Select long distance matching parameters */ + setRand(cctx, ZSTD_c_enableLongDistanceMatching, 0, 1, producer); + setRand(cctx, ZSTD_c_ldmHashLog, ZSTD_HASHLOG_MIN, 16, producer); + setRand(cctx, ZSTD_c_ldmMinMatch, ZSTD_LDM_MINMATCH_MIN, + ZSTD_LDM_MINMATCH_MAX, producer); + setRand(cctx, ZSTD_c_ldmBucketSizeLog, 0, ZSTD_LDM_BUCKETSIZELOG_MAX, + producer); + setRand(cctx, ZSTD_c_ldmHashRateLog, ZSTD_LDM_HASHRATELOG_MIN, + ZSTD_LDM_HASHRATELOG_MAX, producer); + /* Set misc parameters */ + setRand(cctx, ZSTD_c_nbWorkers, 0, 2, producer); + setRand(cctx, ZSTD_c_rsyncable, 0, 1, producer); + setRand(cctx, ZSTD_c_forceMaxWindow, 0, 1, producer); + setRand(cctx, ZSTD_c_literalCompressionMode, 0, 2, producer); + setRand(cctx, ZSTD_c_forceAttachDict, 0, 2, producer); + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { + setRand(cctx, ZSTD_c_srcSizeHint, ZSTD_SRCSIZEHINT_MIN, 2 * srcSize, producer); + } + if (FUZZ_dataProducer_uint32Range(producer, 0, 1) == 0) { + setRand(cctx, ZSTD_c_targetCBlockSize, ZSTD_TARGETCBLOCKSIZE_MIN, ZSTD_TARGETCBLOCKSIZE_MAX, producer); + } +} + +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer) +{ + size_t const dictSize = MAX(srcSize / 8, 1024); + size_t const totalSampleSize = dictSize * 11; + FUZZ_dict_t dict = { FUZZ_malloc(dictSize), dictSize }; + char* const samples = (char*)FUZZ_malloc(totalSampleSize); + unsigned nbSamples = 100; + size_t* const samplesSizes = (size_t*)FUZZ_malloc(sizeof(size_t) * nbSamples); + size_t pos = 0; + size_t sample = 0; + ZDICT_fastCover_params_t params; + + for (sample = 0; sample < nbSamples; ++sample) { + size_t const remaining = totalSampleSize - pos; + size_t const offset = FUZZ_dataProducer_uint32Range(producer, 0, MAX(srcSize, 1) - 1); + size_t const limit = MIN(srcSize - offset, remaining); + size_t const toCopy = MIN(limit, remaining / (nbSamples - sample)); + memcpy(samples + pos, src + offset, toCopy); + pos += toCopy; + samplesSizes[sample] = toCopy; + } + memset(samples + pos, 0, totalSampleSize - pos); + + memset(¶ms, 0, sizeof(params)); + params.accel = 5; + params.k = 40; + params.d = 8; + params.f = 14; + params.zParams.compressionLevel = 1; + dict.size = ZDICT_trainFromBuffer_fastCover(dict.buff, dictSize, + samples, samplesSizes, nbSamples, params); + if (ZSTD_isError(dict.size)) { + free(dict.buff); + memset(&dict, 0, sizeof(dict)); + } + + free(samplesSizes); + free(samples); + + return dict; +} diff --git a/src/zstd/tests/fuzz/zstd_helpers.h b/src/zstd/tests/fuzz/zstd_helpers.h new file mode 100644 index 000000000..6a4e340d3 --- /dev/null +++ b/src/zstd/tests/fuzz/zstd_helpers.h @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +/** + * Helper functions for fuzzing. + */ + +#ifndef ZSTD_HELPERS_H +#define ZSTD_HELPERS_H + +#define ZSTD_STATIC_LINKING_ONLY + +#include "zstd.h" +#include "fuzz_data_producer.h" +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +extern const int kMinClevel; +extern const int kMaxClevel; + +void FUZZ_setRandomParameters(ZSTD_CCtx *cctx, size_t srcSize, FUZZ_dataProducer_t *producer); + +ZSTD_compressionParameters FUZZ_randomCParams(size_t srcSize, FUZZ_dataProducer_t *producer); +ZSTD_frameParameters FUZZ_randomFParams(FUZZ_dataProducer_t *producer); +ZSTD_parameters FUZZ_randomParams(size_t srcSize, FUZZ_dataProducer_t *producer); + +typedef struct { + void* buff; + size_t size; +} FUZZ_dict_t; + +/* Quickly train a dictionary from a source for fuzzing. + * NOTE: Don't use this to train production dictionaries, it is only optimized + * for speed, and doesn't care about dictionary quality. + */ +FUZZ_dict_t FUZZ_train(void const* src, size_t srcSize, FUZZ_dataProducer_t *producer); + +#ifdef __cplusplus +} +#endif + +#endif /* ZSTD_HELPERS_H */ diff --git a/src/zstd/tests/fuzzer.c b/src/zstd/tests/fuzzer.c new file mode 100644 index 000000000..8ac2864f3 --- /dev/null +++ b/src/zstd/tests/fuzzer.c @@ -0,0 +1,3422 @@ +/* + * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*-************************************ +* Compiler specific +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +/*-************************************ +* Includes +**************************************/ +#include <stdlib.h> /* free */ +#include <stdio.h> /* fgets, sscanf */ +#include <string.h> /* strcmp */ +#undef NDEBUG +#include <assert.h> +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_compressContinue, ZSTD_compressBlock */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ +#include "fse.h" +#include "zstd.h" /* ZSTD_VERSION_STRING */ +#include "zstd_errors.h" /* ZSTD_getErrorCode */ +#include "zstdmt_compress.h" +#define ZDICT_STATIC_LINKING_ONLY +#include "zdict.h" /* ZDICT_trainFromBuffer */ +#include "mem.h" +#include "datagen.h" /* RDG_genBuffer */ +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH64 */ +#include "util.h" +#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_TIME_INITIALIZER, UTIL_clockSpanMicro, UTIL_getTime */ +/* must be included after util.h, due to ERROR macro redefinition issue on Visual Studio */ +#include "zstd_internal.h" /* ZSTD_WORKSPACETOOLARGE_MAXDURATION, ZSTD_WORKSPACETOOLARGE_FACTOR, KB, MB */ + + +/*-************************************ +* Constants +**************************************/ +#define GB *(1U<<30) + +static const int FUZ_compressibility_default = 50; +static const int nbTestsDefault = 30000; + + +/*-************************************ +* Display Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define DISPLAYUPDATE(l, ...) \ + if (g_displayLevel>=l) { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ + { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stderr); } \ + } + + +/*-******************************************************* +* Compile time test +*********************************************************/ +#undef MIN +#undef MAX +/* Declaring the function, to avoid -Wmissing-prototype */ +void FUZ_bug976(void); +void FUZ_bug976(void) +{ /* these constants shall not depend on MIN() macro */ + assert(ZSTD_HASHLOG_MAX < 31); + assert(ZSTD_CHAINLOG_MAX < 31); +} + + +/*-******************************************************* +* Internal functions +*********************************************************/ +#define MIN(a,b) ((a)<(b)?(a):(b)) +#define MAX(a,b) ((a)>(b)?(a):(b)) + +#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static U32 FUZ_rand(U32* src) +{ + static const U32 prime1 = 2654435761U; + static const U32 prime2 = 2246822519U; + U32 rand32 = *src; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + +static U32 FUZ_highbit32(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + while (v32) v32 >>= 1, nbBits++; + return nbBits; +} + + +/*============================================= +* Test macros +=============================================*/ +#define CHECK_Z(f) { \ + size_t const err = f; \ + if (ZSTD_isError(err)) { \ + DISPLAY("Error => %s : %s ", \ + #f, ZSTD_getErrorName(err)); \ + exit(1); \ +} } + +#define CHECK_VAR(var, fn) var = fn; if (ZSTD_isError(var)) { DISPLAYLEVEL(1, "%s : fails : %s \n", #fn, ZSTD_getErrorName(var)); goto _output_error; } +#define CHECK_NEWV(var, fn) size_t const CHECK_VAR(var, fn) +#define CHECK(fn) { CHECK_NEWV(err, fn); } +#define CHECKPLUS(var, fn, more) { CHECK_NEWV(var, fn); more; } + +#define CHECK_OP(op, lhs, rhs) { \ + if (!((lhs) op (rhs))) { \ + DISPLAY("Error L%u => FAILED %s %s %s ", __LINE__, #lhs, #op, #rhs); \ + goto _output_error; \ + } \ +} +#define CHECK_EQ(lhs, rhs) CHECK_OP(==, lhs, rhs) +#define CHECK_LT(lhs, rhs) CHECK_OP(<, lhs, rhs) + + +/*============================================= +* Memory Tests +=============================================*/ +#if defined(__APPLE__) && defined(__MACH__) + +#include <malloc/malloc.h> /* malloc_size */ + +typedef struct { + unsigned long long totalMalloc; + size_t currentMalloc; + size_t peakMalloc; + unsigned nbMalloc; + unsigned nbFree; +} mallocCounter_t; + +static const mallocCounter_t INIT_MALLOC_COUNTER = { 0, 0, 0, 0, 0 }; + +static void* FUZ_mallocDebug(void* counter, size_t size) +{ + mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; + void* const ptr = malloc(size); + if (ptr==NULL) return NULL; + DISPLAYLEVEL(4, "allocating %u KB => effectively %u KB \n", + (unsigned)(size >> 10), (unsigned)(malloc_size(ptr) >> 10)); /* OS-X specific */ + mcPtr->totalMalloc += size; + mcPtr->currentMalloc += size; + if (mcPtr->currentMalloc > mcPtr->peakMalloc) + mcPtr->peakMalloc = mcPtr->currentMalloc; + mcPtr->nbMalloc += 1; + return ptr; +} + +static void FUZ_freeDebug(void* counter, void* address) +{ + mallocCounter_t* const mcPtr = (mallocCounter_t*)counter; + DISPLAYLEVEL(4, "freeing %u KB \n", (unsigned)(malloc_size(address) >> 10)); + mcPtr->nbFree += 1; + mcPtr->currentMalloc -= malloc_size(address); /* OS-X specific */ + free(address); +} + +static void FUZ_displayMallocStats(mallocCounter_t count) +{ + DISPLAYLEVEL(3, "peak:%6u KB, nbMallocs:%2u, total:%6u KB \n", + (unsigned)(count.peakMalloc >> 10), + count.nbMalloc, + (unsigned)(count.totalMalloc >> 10)); +} + +static int FUZ_mallocTests_internal(unsigned seed, double compressibility, unsigned part, + void* inBuffer, size_t inSize, void* outBuffer, size_t outSize) +{ + /* test only played in verbose mode, as they are long */ + if (g_displayLevel<3) return 0; + + /* Create compressible noise */ + if (!inBuffer || !outBuffer) { + DISPLAY("Not enough memory, aborting\n"); + exit(1); + } + RDG_genBuffer(inBuffer, inSize, compressibility, 0. /*auto*/, seed); + + /* simple compression tests */ + if (part <= 1) + { int compressionLevel; + for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { + mallocCounter_t malcount = INIT_MALLOC_COUNTER; + ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); + CHECK_Z( ZSTD_compressCCtx(cctx, outBuffer, outSize, inBuffer, inSize, compressionLevel) ); + ZSTD_freeCCtx(cctx); + DISPLAYLEVEL(3, "compressCCtx level %i : ", compressionLevel); + FUZ_displayMallocStats(malcount); + } } + + /* streaming compression tests */ + if (part <= 2) + { int compressionLevel; + for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { + mallocCounter_t malcount = INIT_MALLOC_COUNTER; + ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; + ZSTD_CCtx* const cstream = ZSTD_createCStream_advanced(cMem); + ZSTD_outBuffer out = { outBuffer, outSize, 0 }; + ZSTD_inBuffer in = { inBuffer, inSize, 0 }; + CHECK_Z( ZSTD_initCStream(cstream, compressionLevel) ); + CHECK_Z( ZSTD_compressStream(cstream, &out, &in) ); + CHECK_Z( ZSTD_endStream(cstream, &out) ); + ZSTD_freeCStream(cstream); + DISPLAYLEVEL(3, "compressStream level %i : ", compressionLevel); + FUZ_displayMallocStats(malcount); + } } + + /* advanced MT API test */ + if (part <= 3) + { int nbThreads; + for (nbThreads=1; nbThreads<=4; nbThreads++) { + int compressionLevel; + for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { + mallocCounter_t malcount = INIT_MALLOC_COUNTER; + ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); + CHECK_Z( ZSTD_compress2(cctx, outBuffer, outSize, inBuffer, inSize) ); + ZSTD_freeCCtx(cctx); + DISPLAYLEVEL(3, "compress_generic,-T%i,end level %i : ", + nbThreads, compressionLevel); + FUZ_displayMallocStats(malcount); + } } } + + /* advanced MT streaming API test */ + if (part <= 4) + { int nbThreads; + for (nbThreads=1; nbThreads<=4; nbThreads++) { + int compressionLevel; + for (compressionLevel=1; compressionLevel<=6; compressionLevel++) { + mallocCounter_t malcount = INIT_MALLOC_COUNTER; + ZSTD_customMem const cMem = { FUZ_mallocDebug, FUZ_freeDebug, &malcount }; + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(cMem); + ZSTD_outBuffer out = { outBuffer, outSize, 0 }; + ZSTD_inBuffer in = { inBuffer, inSize, 0 }; + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbThreads) ); + CHECK_Z( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue) ); + while ( ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end) ) {} + ZSTD_freeCCtx(cctx); + DISPLAYLEVEL(3, "compress_generic,-T%i,continue level %i : ", + nbThreads, compressionLevel); + FUZ_displayMallocStats(malcount); + } } } + + return 0; +} + +static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) +{ + size_t const inSize = 64 MB + 16 MB + 4 MB + 1 MB + 256 KB + 64 KB; /* 85.3 MB */ + size_t const outSize = ZSTD_compressBound(inSize); + void* const inBuffer = malloc(inSize); + void* const outBuffer = malloc(outSize); + int result; + + /* Create compressible noise */ + if (!inBuffer || !outBuffer) { + DISPLAY("Not enough memory, aborting \n"); + exit(1); + } + + result = FUZ_mallocTests_internal(seed, compressibility, part, + inBuffer, inSize, outBuffer, outSize); + + free(inBuffer); + free(outBuffer); + return result; +} + +#else + +static int FUZ_mallocTests(unsigned seed, double compressibility, unsigned part) +{ + (void)seed; (void)compressibility; (void)part; + return 0; +} + +#endif + +static void FUZ_decodeSequences(BYTE* dst, ZSTD_Sequence* seqs, size_t seqsSize, BYTE* src, size_t size) +{ + size_t i; + size_t j; + for(i = 0; i < seqsSize - 1; ++i) { + assert(dst + seqs[i].litLength + seqs[i].matchLength < dst + size); + assert(src + seqs[i].litLength + seqs[i].matchLength < src + size); + + memcpy(dst, src, seqs[i].litLength); + dst += seqs[i].litLength; + src += seqs[i].litLength; + size -= seqs[i].litLength; + + for (j = 0; j < seqs[i].matchLength; ++j) + dst[j] = dst[j - seqs[i].offset]; + dst += seqs[i].matchLength; + src += seqs[i].matchLength; + size -= seqs[i].matchLength; + } + memcpy(dst, src, size); +} + +/*============================================= +* Unit tests +=============================================*/ + +static int basicUnitTests(U32 const seed, double compressibility) +{ + size_t const CNBuffSize = 5 MB; + void* const CNBuffer = malloc(CNBuffSize); + size_t const compressedBufferSize = ZSTD_compressBound(CNBuffSize); + void* const compressedBuffer = malloc(compressedBufferSize); + void* const decodedBuffer = malloc(CNBuffSize); + int testResult = 0; + unsigned testNb=0; + size_t cSize; + + /* Create compressible noise */ + if (!CNBuffer || !compressedBuffer || !decodedBuffer) { + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } + RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); + + /* Basic tests */ + DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName : ", testNb++); + { const char* errorString = ZSTD_getErrorName(0); + DISPLAYLEVEL(3, "OK : %s \n", errorString); + } + + DISPLAYLEVEL(3, "test%3u : ZSTD_getErrorName with wrong value : ", testNb++); + { const char* errorString = ZSTD_getErrorName(499); + DISPLAYLEVEL(3, "OK : %s \n", errorString); + } + + DISPLAYLEVEL(3, "test%3u : min compression level : ", testNb++); + { int const mcl = ZSTD_minCLevel(); + DISPLAYLEVEL(3, "%i (OK) \n", mcl); + } + + DISPLAYLEVEL(3, "test%3u : ZSTD_versionNumber : ", testNb++); + { unsigned const vn = ZSTD_versionNumber(); + DISPLAYLEVEL(3, "%u (OK) \n", vn); + } + + DISPLAYLEVEL(3, "test%3u : compress %u bytes : ", testNb++, (unsigned)CNBuffSize); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + if (cctx==NULL) goto _output_error; + CHECK_VAR(cSize, ZSTD_compressCCtx(cctx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, 1) ); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : size of cctx for level 1 : ", testNb++); + { size_t const cctxSize = ZSTD_sizeof_CCtx(cctx); + DISPLAYLEVEL(3, "%u bytes \n", (unsigned)cctxSize); + } + ZSTD_freeCCtx(cctx); + } + + DISPLAYLEVEL(3, "test%3i : decompress skippable frame -8 size : ", testNb++); + { + char const skippable8[] = "\x50\x2a\x4d\x18\xf8\xff\xff\xff"; + size_t const size = ZSTD_decompress(NULL, 0, skippable8, 8); + if (!ZSTD_isError(size)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_getFrameContentSize test : ", testNb++); + { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); + if (rSize != CNBuffSize) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_getDecompressedSize test : ", testNb++); + { unsigned long long const rSize = ZSTD_getDecompressedSize(compressedBuffer, cSize); + if (rSize != CNBuffSize) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_findDecompressedSize test : ", testNb++); + { unsigned long long const rSize = ZSTD_findDecompressedSize(compressedBuffer, cSize); + if (rSize != CNBuffSize) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : tight ZSTD_decompressBound test : ", testNb++); + { + unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize); + if (bound != CNBuffSize) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with invalid srcSize : ", testNb++); + { + unsigned long long bound = ZSTD_decompressBound(compressedBuffer, cSize - 1); + if (bound != ZSTD_CONTENTSIZE_ERROR) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (r != CNBuffSize) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); + { size_t u; + for (u=0; u<CNBuffSize; u++) { + if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; + } } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_checkCParams : ", testNb++); + { + ZSTD_parameters params = ZSTD_getParams(3, 0, 0); + assert(!ZSTD_checkCParams(params.cParams)); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_createDCtx_advanced and ZSTD_sizeof_DCtx: ", testNb++); + { + ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(ZSTD_defaultCMem); + assert(dctx != NULL); + assert(ZSTD_sizeof_DCtx(dctx) != 0); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : misc unaccounted for zstd symbols : ", testNb++); + { + /* %p takes a void*. In ISO C, it's illegal to cast a function pointer + * to a data pointer. (Although in POSIX you're required to be allowed + * to do it...) So we have to fall back to our trusty friend memcpy. */ + unsigned (* const funcptr_getDictID)(const ZSTD_DDict* ddict) = + ZSTD_getDictID_fromDDict; + ZSTD_DStream* (* const funcptr_createDStream)( + ZSTD_customMem customMem) = ZSTD_createDStream_advanced; + void (* const funcptr_copyDCtx)( + ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx) = ZSTD_copyDCtx; + ZSTD_nextInputType_e (* const funcptr_nextInputType)(ZSTD_DCtx* dctx) = + ZSTD_nextInputType; + const void *voidptr_getDictID; + const void *voidptr_createDStream; + const void *voidptr_copyDCtx; + const void *voidptr_nextInputType; + DEBUG_STATIC_ASSERT(sizeof(funcptr_getDictID) == sizeof(voidptr_getDictID)); + memcpy( + (void*)&voidptr_getDictID, + (const void*)&funcptr_getDictID, + sizeof(void*)); + memcpy( + (void*)&voidptr_createDStream, + (const void*)&funcptr_createDStream, + sizeof(void*)); + memcpy( + (void*)&voidptr_copyDCtx, + (const void*)&funcptr_copyDCtx, + sizeof(void*)); + memcpy( + (void*)&voidptr_nextInputType, + (const void*)&funcptr_nextInputType, + sizeof(void*)); + DISPLAYLEVEL(3, "%p ", voidptr_getDictID); + DISPLAYLEVEL(3, "%p ", voidptr_createDStream); + DISPLAYLEVEL(3, "%p ", voidptr_copyDCtx); + DISPLAYLEVEL(3, "%p ", voidptr_nextInputType); + } + DISPLAYLEVEL(3, ": OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress with null dict : ", testNb++); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + { size_t const r = ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + NULL, 0); + if (r != CNBuffSize) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress with null DDict : ", testNb++); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + { size_t const r = ZSTD_decompress_usingDDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + NULL); + if (r != CNBuffSize) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress with 1 missing byte : ", testNb++); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize-1); + if (!ZSTD_isError(r)) goto _output_error; + if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_srcSize_wrong) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress with 1 too much byte : ", testNb++); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize+1); + if (!ZSTD_isError(r)) goto _output_error; + if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress too large input : ", testNb++); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, compressedBufferSize); + if (!ZSTD_isError(r)) goto _output_error; + if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress into NULL buffer : ", testNb++); + { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, compressedBufferSize); + if (!ZSTD_isError(r)) goto _output_error; + if (ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressBound test with content size missing : ", testNb++); + { /* create compressed buffer with content size missing */ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); + CHECK_VAR(cSize, ZSTD_compress2(cctx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize) ); + ZSTD_freeCCtx(cctx); + } + { /* ensure frame content size is missing */ + ZSTD_frameHeader zfh; + size_t const ret = ZSTD_getFrameHeader(&zfh, compressedBuffer, compressedBufferSize); + if (ret != 0 || zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; + } + { /* ensure CNBuffSize <= decompressBound */ + unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, compressedBufferSize); + if (CNBuffSize > bound) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d: check DCtx size is reduced after many oversized calls : ", testNb++); + { + size_t const largeFrameSrcSize = 200; + size_t const smallFrameSrcSize = 10; + size_t const nbFrames = 256; + + size_t i = 0, consumed = 0, produced = 0, prevDCtxSize = 0; + int sizeReduced = 0; + + BYTE* const dst = (BYTE*)compressedBuffer; + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + + /* create a large frame and then a bunch of small frames */ + size_t srcSize = ZSTD_compress((void*)dst, + compressedBufferSize, CNBuffer, largeFrameSrcSize, 3); + for (i = 0; i < nbFrames; i++) + srcSize += ZSTD_compress((void*)(dst + srcSize), + compressedBufferSize - srcSize, CNBuffer, + smallFrameSrcSize, 3); + + /* decompressStream and make sure that dctx size was reduced at least once */ + while (consumed < srcSize) { + ZSTD_inBuffer in = {(void*)(dst + consumed), MIN(1, srcSize - consumed), 0}; + ZSTD_outBuffer out = {(BYTE*)CNBuffer + produced, CNBuffSize - produced, 0}; + ZSTD_decompressStream(dctx, &out, &in); + consumed += in.pos; + produced += out.pos; + + /* success! size was reduced from the previous frame */ + if (prevDCtxSize > ZSTD_sizeof_DCtx(dctx)) + sizeReduced = 1; + + prevDCtxSize = ZSTD_sizeof_DCtx(dctx); + } + + assert(sizeReduced); + + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ldm fill dict out-of-bounds check", testNb++); + { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + + size_t const size = (1U << 10); + size_t const dstCapacity = ZSTD_compressBound(size); + void* dict = (void*)malloc(size); + void* src = (void*)malloc(size); + void* dst = (void*)malloc(dstCapacity); + + RDG_genBuffer(dict, size, 0.5, 0.5, seed); + RDG_genBuffer(src, size, 0.5, 0.5, seed); + + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + assert(!ZSTD_isError(ZSTD_compress_usingDict(cctx, dst, dstCapacity, src, size, dict, size, 3))); + + ZSTD_freeCCtx(cctx); + free(dict); + free(src); + free(dst); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : testing dict compression with enableLdm and forceMaxWindow : ", testNb++); + { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + void* dict = (void*)malloc(CNBuffSize); + int nbWorkers; + + for (nbWorkers = 0; nbWorkers < 3; ++nbWorkers) { + RDG_genBuffer(dict, CNBuffSize, 0.5, 0.5, seed); + RDG_genBuffer(CNBuffer, CNBuffSize, 0.6, 0.6, seed); + + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, nbWorkers)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_forceMaxWindow, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, CNBuffSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(cSize); + CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, CNBuffSize)); + } + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + free(dict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : testing ldm dictionary gets invalidated : ", testNb++); + { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + void* dict = (void*)malloc(CNBuffSize); + size_t const kWindowLog = 10; + size_t const kWindowSize = (size_t)1 << kWindowLog; + size_t const dictSize = kWindowSize * 10; + size_t const srcSize1 = kWindowSize / 2; + size_t const srcSize2 = kWindowSize * 10; + + if (CNBuffSize < dictSize) goto _output_error; + + RDG_genBuffer(dict, dictSize, 0.5, 0.5, seed); + RDG_genBuffer(CNBuffer, srcSize1 + srcSize2, 0.5, 0.5, seed); + + /* Enable checksum to verify round trip. */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, 1)); + /* Disable content size to skip single-pass decompression. */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, (int)kWindowLog)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmMinMatch, 32)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashRateLog, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmHashLog, 16)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_ldmBucketSizeLog, 3)); + + /* Round trip once with a dictionary. */ + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize1); + CHECK_Z(cSize); + CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, srcSize2); + /* Streaming decompression to catch out of bounds offsets. */ + { + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; + size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); + CHECK_Z(dSize); + if (dSize != 0) goto _output_error; + } + + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); + /* Round trip once with a dictionary. */ + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); + { + ZSTD_inBuffer in = {CNBuffer, srcSize1, 0}; + ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + cSize = out.pos; + } + CHECK_Z(ZSTD_decompress_usingDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, dict, dictSize)); + { + ZSTD_inBuffer in = {CNBuffer, srcSize2, 0}; + ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + cSize = out.pos; + } + /* Streaming decompression to catch out of bounds offsets. */ + { + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + ZSTD_outBuffer out = {decodedBuffer, CNBuffSize, 0}; + size_t const dSize = ZSTD_decompressStream(dctx, &out, &in); + CHECK_Z(dSize); + if (dSize != 0) goto _output_error; + } + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + free(dict); + } + DISPLAYLEVEL(3, "OK \n"); + + /* Note: this test takes 0.5 seconds to run */ + DISPLAYLEVEL(3, "test%3i : testing refPrefx vs refPrefx + ldm (size comparison) : ", testNb++); + { + /* test a big buffer so that ldm can take effect */ + size_t const size = 100 MB; + int const windowLog = 27; + size_t const dstSize = ZSTD_compressBound(size); + + void* dict = (void*)malloc(size); + void* src = (void*)malloc(size); + void* dst = (void*)malloc(dstSize); + void* recon = (void*)malloc(size); + + size_t refPrefixCompressedSize = 0; + size_t refPrefixLdmComrpessedSize = 0; + size_t reconSize = 0; + + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + + /* make dict and src the same uncompressible data */ + RDG_genBuffer(src, size, 0, 0, seed); + memcpy(dict, src, size); + assert(!memcmp(dict, src, size)); + + /* set level 1 and windowLog to cover src */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog)); + + /* compress on level 1 using just refPrefix and no ldm */ + ZSTD_CCtx_refPrefix(cctx, dict, size); + refPrefixCompressedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); + assert(!ZSTD_isError(refPrefixCompressedSize)); + + /* test round trip just refPrefix */ + ZSTD_DCtx_refPrefix(dctx, dict, size); + reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixCompressedSize); + assert(!ZSTD_isError(reconSize)); + assert(reconSize == size); + assert(!memcmp(recon, src, size)); + + /* compress on level 1 using refPrefix and ldm */ + ZSTD_CCtx_refPrefix(cctx, dict, size);; + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)) + refPrefixLdmComrpessedSize = ZSTD_compress2(cctx, dst, dstSize, src, size); + assert(!ZSTD_isError(refPrefixLdmComrpessedSize)); + + /* test round trip refPrefix + ldm*/ + ZSTD_DCtx_refPrefix(dctx, dict, size); + reconSize = ZSTD_decompressDCtx(dctx, recon, size, dst, refPrefixLdmComrpessedSize); + assert(!ZSTD_isError(reconSize)); + assert(reconSize == size); + assert(!memcmp(recon, src, size)); + + /* make sure that refPrefixCompressedSize is significantly greater */ + assert(refPrefixCompressedSize > 10 * refPrefixLdmComrpessedSize); + /* make sure the ldm comrpessed size is less than 1% of original */ + assert((double)refPrefixLdmComrpessedSize / (double)size < 0.01); + + ZSTD_freeDCtx(dctx); + ZSTD_freeCCtx(cctx); + free(recon); + free(dict); + free(src); + free(dst); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d: superblock uncompressible data, too many nocompress superblocks : ", testNb++); + { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + const BYTE* src = (BYTE*)CNBuffer; BYTE* dst = (BYTE*)compressedBuffer; + size_t srcSize = 321656; size_t dstCapacity = ZSTD_compressBound(srcSize); + + /* This is the number of bytes to stream before ending. This value + * was obtained by trial and error :/. */ + + const size_t streamCompressThreshold = 161792; + const size_t streamCompressDelta = 1024; + + /* The first 1/5 of the buffer is compressible and the last 4/5 is + * uncompressible. This is an approximation of the type of data + * the fuzzer generated to catch this bug. Streams like this were making + * zstd generate noCompress superblocks (which are larger than the src + * they come from). Do this enough times, and we'll run out of room + * and throw a dstSize_tooSmall error. */ + + const size_t compressiblePartSize = srcSize/5; + const size_t uncompressiblePartSize = srcSize-compressiblePartSize; + RDG_genBuffer(CNBuffer, compressiblePartSize, 0.5, 0.5, seed); + RDG_genBuffer((BYTE*)CNBuffer+compressiblePartSize, uncompressiblePartSize, 0, 0, seed); + + /* Setting target block size so that superblock is used */ + + assert(cctx != NULL); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 81); + + { size_t read; + for (read = 0; read < streamCompressThreshold; read += streamCompressDelta) { + ZSTD_inBuffer in = {src, streamCompressDelta, 0}; + ZSTD_outBuffer out = {dst, dstCapacity, 0}; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + src += streamCompressDelta; srcSize -= streamCompressDelta; + dst += out.pos; dstCapacity -= out.pos;}} + + /* This is trying to catch a dstSize_tooSmall error */ + + { ZSTD_inBuffer in = {src, srcSize, 0}; + ZSTD_outBuffer out = {dst, dstCapacity, 0}; + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end));} + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d: superblock with no literals : ", testNb++); + /* Generate the same data 20 times over */ + { + size_t const avgChunkSize = CNBuffSize / 20; + size_t b; + for (b = 0; b < CNBuffSize; b += avgChunkSize) { + size_t const chunkSize = MIN(CNBuffSize - b, avgChunkSize); + RDG_genBuffer((char*)CNBuffer + b, chunkSize, compressibility, 0. /* auto */, seed); + } + } + { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const normalCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + size_t const allowedExpansion = (CNBuffSize * 3 / 1000); + size_t superCSize; + CHECK_Z(normalCSize); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 19); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 1000); + superCSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK_Z(superCSize); + if (superCSize > normalCSize + allowedExpansion) { + DISPLAYLEVEL(1, "Superblock too big: %u > %u + %u \n", (U32)superCSize, (U32)normalCSize, (U32)allowedExpansion); + goto _output_error; + } + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0. /*auto*/, seed); + DISPLAYLEVEL(3, "test%3d: superblock enough room for checksum : ", testNb++) + { + /* This tests whether or not we leave enough room for the checksum at the end + * of the dst buffer. The bug that motivated this test was found by the + * stream_round_trip fuzzer but this crashes for the same reason and is + * far more compact than re-creating the stream_round_trip fuzzer's code path */ + ZSTD_CCtx *cctx = ZSTD_createCCtx(); + ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetCBlockSize, 64); + assert(!ZSTD_isError(ZSTD_compress2(cctx, compressedBuffer, 1339, CNBuffer, 1278))); + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : compress a NULL input with each level : ", testNb++); + { int level = -1; + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + if (!cctx) goto _output_error; + for (level = -1; level <= ZSTD_maxCLevel(); ++level) { + CHECK_Z( ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, level) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, level) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, NULL, 0) ); + } + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d : check CCtx size after compressing empty input : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const r = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, NULL, 0, 19); + if (ZSTD_isError(r)) goto _output_error; + if (ZSTD_sizeof_CCtx(cctx) > (1U << 20)) goto _output_error; + ZSTD_freeCCtx(cctx); + cSize = r; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d : decompress empty frame into NULL : ", testNb++); + { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, cSize); + if (ZSTD_isError(r)) goto _output_error; + if (r != 0) goto _output_error; + } + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_outBuffer output; + if (cctx==NULL) goto _output_error; + output.dst = compressedBuffer; + output.size = compressedBufferSize; + output.pos = 0; + CHECK_Z( ZSTD_initCStream(cctx, 1) ); /* content size unknown */ + CHECK_Z( ZSTD_flushStream(cctx, &output) ); /* ensure no possibility to "concatenate" and determine the content size */ + CHECK_Z( ZSTD_endStream(cctx, &output) ); + ZSTD_freeCCtx(cctx); + /* single scan decompression */ + { size_t const r = ZSTD_decompress(NULL, 0, compressedBuffer, output.pos); + if (ZSTD_isError(r)) goto _output_error; + if (r != 0) goto _output_error; + } + /* streaming decompression */ + { ZSTD_DCtx* const dstream = ZSTD_createDStream(); + ZSTD_inBuffer dinput; + ZSTD_outBuffer doutput; + size_t ipos; + if (dstream==NULL) goto _output_error; + dinput.src = compressedBuffer; + dinput.size = 0; + dinput.pos = 0; + doutput.dst = NULL; + doutput.size = 0; + doutput.pos = 0; + CHECK_Z ( ZSTD_initDStream(dstream) ); + for (ipos=1; ipos<=output.pos; ipos++) { + dinput.size = ipos; + CHECK_Z ( ZSTD_decompressStream(dstream, &doutput, &dinput) ); + } + if (doutput.pos != 0) goto _output_error; + ZSTD_freeDStream(dstream); + } + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d : re-use CCtx with expanding block size : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_parameters const params = ZSTD_getParams(1, ZSTD_CONTENTSIZE_UNKNOWN, 0); + assert(params.fParams.contentSizeFlag == 1); /* block size will be adapted if pledgedSrcSize is enabled */ + CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, 1 /*pledgedSrcSize*/) ); + CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); /* creates a block size of 1 */ + + CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */ + { size_t const inSize = 2* 128 KB; + size_t const outSize = ZSTD_compressBound(inSize); + CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, outSize, CNBuffer, inSize) ); + /* will fail if blockSize is not resized */ + } + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d : re-using a CCtx should compress the same : ", testNb++); + { size_t const sampleSize = 30; + int i; + for (i=0; i<20; i++) + ((char*)CNBuffer)[i] = (char)i; /* ensure no match during initial section */ + memcpy((char*)CNBuffer + 20, CNBuffer, 10); /* create one match, starting from beginning of sample, which is the difficult case (see #1241) */ + for (i=1; i<=19; i++) { + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t size1, size2; + DISPLAYLEVEL(5, "l%i ", i); + size1 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); + CHECK_Z(size1); + + size2 = ZSTD_compressCCtx(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize, i); + CHECK_Z(size2); + CHECK_EQ(size1, size2); + + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, i) ); + size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, sampleSize); + CHECK_Z(size2); + CHECK_EQ(size1, size2); + + size2 = ZSTD_compress2(cctx, compressedBuffer, ZSTD_compressBound(sampleSize) - 1, CNBuffer, sampleSize); /* force streaming, as output buffer is not large enough to guarantee success */ + CHECK_Z(size2); + CHECK_EQ(size1, size2); + + { ZSTD_inBuffer inb; + ZSTD_outBuffer outb; + inb.src = CNBuffer; + inb.pos = 0; + inb.size = sampleSize; + outb.dst = compressedBuffer; + outb.pos = 0; + outb.size = ZSTD_compressBound(sampleSize) - 1; /* force streaming, as output buffer is not large enough to guarantee success */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); + assert(inb.pos == inb.size); + CHECK_EQ(size1, outb.pos); + } + + ZSTD_freeCCtx(cctx); + } + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d : btultra2 & 1st block : ", testNb++); + { size_t const sampleSize = 1024; + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_inBuffer inb; + ZSTD_outBuffer outb; + inb.src = CNBuffer; + inb.pos = 0; + inb.size = 0; + outb.dst = compressedBuffer; + outb.pos = 0; + outb.size = compressedBufferSize; + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, ZSTD_maxCLevel()) ); + + inb.size = sampleSize; /* start with something, so that context is already used */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ + assert(inb.pos == inb.size); + outb.pos = 0; /* cancel output */ + + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(cctx, sampleSize) ); + inb.size = 4; /* too small size : compression will be skipped */ + inb.pos = 0; + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); + assert(inb.pos == inb.size); + + inb.size += 5; /* too small size : compression will be skipped */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); + assert(inb.pos == inb.size); + + inb.size += 11; /* small enough to attempt compression */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_flush) ); + assert(inb.pos == inb.size); + + assert(inb.pos < sampleSize); + inb.size = sampleSize; /* large enough to trigger stats_init, but no longer at beginning */ + CHECK_Z( ZSTD_compressStream2(cctx, &outb, &inb, ZSTD_e_end) ); /* will break internal assert if stats_init is not disabled */ + assert(inb.pos == inb.size); + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d : ZSTD_CCtx_getParameter() : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_outBuffer out = {NULL, 0, 0}; + ZSTD_inBuffer in = {NULL, 0, 0}; + int value; + + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); + CHECK_EQ(value, 3); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); + CHECK_EQ(value, 0); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, ZSTD_HASHLOG_MIN)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); + CHECK_EQ(value, 3); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); + CHECK_EQ(value, ZSTD_HASHLOG_MIN); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7)); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); + CHECK_EQ(value, 7); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); + CHECK_EQ(value, ZSTD_HASHLOG_MIN); + /* Start a compression job */ + ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_continue); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); + CHECK_EQ(value, 7); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); + CHECK_EQ(value, ZSTD_HASHLOG_MIN); + /* Reset the CCtx */ + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); + CHECK_EQ(value, 7); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); + CHECK_EQ(value, ZSTD_HASHLOG_MIN); + /* Reset the parameters */ + ZSTD_CCtx_reset(cctx, ZSTD_reset_parameters); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_compressionLevel, &value)); + CHECK_EQ(value, 3); + CHECK_Z(ZSTD_CCtx_getParameter(cctx, ZSTD_c_hashLog, &value)); + CHECK_EQ(value, 0); + + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + /* this test is really too long, and should be made faster */ + DISPLAYLEVEL(3, "test%3d : overflow protection with large windowLog : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_parameters params = ZSTD_getParams(-999, ZSTD_CONTENTSIZE_UNKNOWN, 0); + size_t const nbCompressions = ((1U << 31) / CNBuffSize) + 2; /* ensure U32 overflow protection is triggered */ + size_t cnb; + assert(cctx != NULL); + params.fParams.contentSizeFlag = 0; + params.cParams.windowLog = ZSTD_WINDOWLOG_MAX; + for (cnb = 0; cnb < nbCompressions; ++cnb) { + DISPLAYLEVEL(6, "run %zu / %zu \n", cnb, nbCompressions); + CHECK_Z( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN) ); /* re-use same parameters */ + CHECK_Z( ZSTD_compressEnd(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize) ); + } + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3d : size down context : ", testNb++); + { ZSTD_CCtx* const largeCCtx = ZSTD_createCCtx(); + assert(largeCCtx != NULL); + CHECK_Z( ZSTD_compressBegin(largeCCtx, 19) ); /* streaming implies ZSTD_CONTENTSIZE_UNKNOWN, which maximizes memory usage */ + CHECK_Z( ZSTD_compressEnd(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1) ); + { size_t const largeCCtxSize = ZSTD_sizeof_CCtx(largeCCtx); /* size of context must be measured after compression */ + { ZSTD_CCtx* const smallCCtx = ZSTD_createCCtx(); + assert(smallCCtx != NULL); + CHECK_Z(ZSTD_compressCCtx(smallCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); + { size_t const smallCCtxSize = ZSTD_sizeof_CCtx(smallCCtx); + DISPLAYLEVEL(5, "(large) %zuKB > 32*%zuKB (small) : ", + largeCCtxSize>>10, smallCCtxSize>>10); + assert(largeCCtxSize > 32* smallCCtxSize); /* note : "too large" definition is handled within zstd_compress.c . + * make this test case extreme, so that it doesn't depend on a possibly fluctuating definition */ + } + ZSTD_freeCCtx(smallCCtx); + } + { U32 const maxNbAttempts = 1100; /* nb of usages before triggering size down is handled within zstd_compress.c. + * currently defined as 128x, but could be adjusted in the future. + * make this test long enough so that it's not too much tied to the current definition within zstd_compress.c */ + unsigned u; + for (u=0; u<maxNbAttempts; u++) { + CHECK_Z(ZSTD_compressCCtx(largeCCtx, compressedBuffer, compressedBufferSize, CNBuffer, 1, 1)); + if (ZSTD_sizeof_CCtx(largeCCtx) < largeCCtxSize) break; /* sized down */ + } + DISPLAYLEVEL(5, "size down after %u attempts : ", u); + if (u==maxNbAttempts) goto _output_error; /* no sizedown happened */ + } + } + ZSTD_freeCCtx(largeCCtx); + } + DISPLAYLEVEL(3, "OK \n"); + + /* Static CCtx tests */ +#define STATIC_CCTX_LEVEL 4 + DISPLAYLEVEL(3, "test%3i : create static CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); + { size_t const staticCStreamSize = ZSTD_estimateCStreamSize(STATIC_CCTX_LEVEL); + void* const staticCCtxBuffer = malloc(staticCStreamSize); + size_t const staticDCtxSize = ZSTD_estimateDCtxSize(); + void* const staticDCtxBuffer = malloc(staticDCtxSize); + DISPLAYLEVEL(4, "CStream size = %u, ", (U32)staticCStreamSize); + if (staticCCtxBuffer==NULL || staticDCtxBuffer==NULL) { + free(staticCCtxBuffer); + free(staticDCtxBuffer); + DISPLAY("Not enough memory, aborting\n"); + testResult = 1; + goto _end; + } + { size_t const smallInSize = 32 KB; + ZSTD_compressionParameters const cparams_small = ZSTD_getCParams(STATIC_CCTX_LEVEL, smallInSize, 0); + size_t const smallCCtxSize = ZSTD_estimateCCtxSize_usingCParams(cparams_small); + size_t const staticCCtxSize = ZSTD_estimateCCtxSize(STATIC_CCTX_LEVEL); + ZSTD_CCtx* staticCCtx = ZSTD_initStaticCCtx(staticCCtxBuffer, smallCCtxSize); + ZSTD_DCtx* const staticDCtx = ZSTD_initStaticDCtx(staticDCtxBuffer, staticDCtxSize); + DISPLAYLEVEL(4, "Full CCtx size = %u, ", (U32)staticCCtxSize); + DISPLAYLEVEL(4, "CCtx for 32 KB = %u, ", (U32)smallCCtxSize); + if ((staticCCtx==NULL) || (staticDCtx==NULL)) goto _output_error; + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : compress small input with small static CCtx : ", testNb++); + CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, + compressedBuffer, compressedBufferSize, + CNBuffer, smallInSize, STATIC_CCTX_LEVEL) ); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", + (unsigned)cSize, (double)cSize/smallInSize*100); + + DISPLAYLEVEL(3, "test%3i : compress large input with small static CCtx (must fail) : ", testNb++); + { size_t const r = ZSTD_compressCCtx(staticCCtx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL); + if (ZSTD_getErrorCode((size_t)r) != ZSTD_error_memory_allocation) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : resize context to full CCtx size : ", testNb++); + staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCCtxSize); + DISPLAYLEVEL(4, "staticCCtxBuffer = %p, staticCCtx = %p , ", staticCCtxBuffer, staticCCtx); + if (staticCCtx == NULL) goto _output_error; + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : compress large input with static CCtx : ", testNb++); + CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", + (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : compress small input often enough to trigger context reduce : ", testNb++); + { int nbc; + assert(staticCCtxSize > smallCCtxSize * ZSTD_WORKSPACETOOLARGE_FACTOR); /* ensure size down scenario */ + assert(CNBuffSize > smallInSize + ZSTD_WORKSPACETOOLARGE_MAXDURATION + 3); + for (nbc=0; nbc<ZSTD_WORKSPACETOOLARGE_MAXDURATION+2; nbc++) { + CHECK_Z(ZSTD_compressCCtx(staticCCtx, + compressedBuffer, compressedBufferSize, + (char*)CNBuffer + nbc, smallInSize, + STATIC_CCTX_LEVEL) ); + } } + DISPLAYLEVEL(3, "OK \n") + + DISPLAYLEVEL(3, "test%3i : init CCtx for level %u : ", testNb++, STATIC_CCTX_LEVEL); + CHECK_Z( ZSTD_compressBegin(staticCCtx, STATIC_CCTX_LEVEL) ); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : compression again with static CCtx : ", testNb++); + CHECK_VAR(cSize, ZSTD_compressCCtx(staticCCtx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, STATIC_CCTX_LEVEL) ); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", + (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : simple decompression test with static DCtx : ", testNb++); + { size_t const r = ZSTD_decompressDCtx(staticDCtx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize); + if (r != CNBuffSize) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); + if (memcmp(decodedBuffer, CNBuffer, CNBuffSize)) goto _output_error; + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : init CCtx for too large level (must fail) : ", testNb++); + { size_t const r = ZSTD_compressBegin(staticCCtx, ZSTD_maxCLevel()); + if (!ZSTD_isError(r)) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : init CCtx for small level %u (should work again) : ", testNb++, 1); + CHECK( ZSTD_compressBegin(staticCCtx, 1) ); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : use CStream on CCtx-sized static context (should fail) : ", testNb++); + CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ + { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; + size_t const r = ZSTD_compressStream(staticCCtx, &output, &input); /* now allocates, should fail */ + if (!ZSTD_isError(r)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : resize context to CStream size, then stream compress : ", testNb++); + staticCCtx = ZSTD_initStaticCStream(staticCCtxBuffer, staticCStreamSize); + assert(staticCCtx != NULL); + CHECK_Z( ZSTD_initCStream(staticCCtx, STATIC_CCTX_LEVEL) ); /* note : doesn't allocate */ + { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; + CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : CStream for small level %u : ", testNb++, 1); + CHECK_Z( ZSTD_initCStream(staticCCtx, 1) ); /* note : doesn't allocate */ + { ZSTD_outBuffer output = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_inBuffer input = { CNBuffer, CNBuffSize, 0 }; + CHECK_Z( ZSTD_compressStream(staticCCtx, &output, &input) ); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : init static CStream with dictionary (should fail) : ", testNb++); + { size_t const r = ZSTD_initCStream_usingDict(staticCCtx, CNBuffer, 64 KB, 1); + if (!ZSTD_isError(r)) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : use DStream on DCtx-sized static context (should fail) : ", testNb++); + CHECK_Z( ZSTD_initDStream(staticDCtx) ); + { ZSTD_outBuffer output = { decodedBuffer, CNBuffSize, 0 }; + ZSTD_inBuffer input = { compressedBuffer, ZSTD_FRAMEHEADERSIZE_MAX+1, 0 }; + size_t const r = ZSTD_decompressStream(staticDCtx, &output, &input); + if (!ZSTD_isError(r)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + } + free(staticCCtxBuffer); + free(staticDCtxBuffer); + } + + DISPLAYLEVEL(3, "test%3i : Static context sizes for negative levels : ", testNb++); + { size_t const cctxSizeN1 = ZSTD_estimateCCtxSize(-1); + size_t const cctxSizeP1 = ZSTD_estimateCCtxSize(1); + size_t const cstreamSizeN1 = ZSTD_estimateCStreamSize(-1); + size_t const cstreamSizeP1 = ZSTD_estimateCStreamSize(1); + + if (!(0 < cctxSizeN1 && cctxSizeN1 <= cctxSizeP1)) goto _output_error; + if (!(0 < cstreamSizeN1 && cstreamSizeN1 <= cstreamSizeP1)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + + /* ZSTDMT simple MT compression test */ + DISPLAYLEVEL(3, "test%3i : create ZSTDMT CCtx : ", testNb++); + { ZSTDMT_CCtx* const mtctx = ZSTDMT_createCCtx(2); + if (mtctx==NULL) { + DISPLAY("mtctx : not enough memory, aborting \n"); + testResult = 1; + goto _end; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3u : compress %u bytes with 2 threads : ", testNb++, (unsigned)CNBuffSize); + CHECK_VAR(cSize, ZSTDMT_compressCCtx(mtctx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, + 1) ); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : decompressed size test : ", testNb++); + { unsigned long long const rSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); + if (rSize != CNBuffSize) { + DISPLAY("ZSTD_getFrameContentSize incorrect : %u != %u \n", (unsigned)rSize, (unsigned)CNBuffSize); + goto _output_error; + } } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (r != CNBuffSize) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); + { size_t u; + for (u=0; u<CNBuffSize; u++) { + if (((BYTE*)decodedBuffer)[u] != ((BYTE*)CNBuffer)[u]) goto _output_error; + } } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : compress -T2 with checksum : ", testNb++); + { ZSTD_parameters params = ZSTD_getParams(1, CNBuffSize, 0); + params.fParams.checksumFlag = 1; + params.fParams.contentSizeFlag = 1; + CHECK_VAR(cSize, ZSTDMT_compress_advanced(mtctx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, + NULL, params, 3 /*overlapRLog*/) ); + } + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, (unsigned)CNBuffSize); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (r != CNBuffSize) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + ZSTDMT_freeCCtx(mtctx); + } + + DISPLAYLEVEL(3, "test%3u : compress empty string and decompress with small window log : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + char out[32]; + if (cctx == NULL || dctx == NULL) goto _output_error; + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, 0) ); + CHECK_VAR(cSize, ZSTD_compress2(cctx, out, sizeof(out), NULL, 0) ); + DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)cSize); + + CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 10) ); + { char const* outPtr = out; + ZSTD_inBuffer inBuffer = { outPtr, cSize, 0 }; + ZSTD_outBuffer outBuffer = { NULL, 0, 0 }; + size_t dSize; + CHECK_VAR(dSize, ZSTD_decompressStream(dctx, &outBuffer, &inBuffer) ); + if (dSize != 0) goto _output_error; + } + + ZSTD_freeDCtx(dctx); + ZSTD_freeCCtx(cctx); + } + + DISPLAYLEVEL(3, "test%3i : compress -T2 with/without literals compression : ", testNb++) + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + size_t cSize1, cSize2; + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2) ); + cSize1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK(cSize1); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_literalCompressionMode, ZSTD_lcm_uncompressed) ); + cSize2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, CNBuffSize); + CHECK(cSize2); + CHECK_LT(cSize1, cSize2); + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Multithreaded ZSTD_compress2() with rsyncable : ", testNb++) + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + /* Set rsyncable and don't give the ZSTD_compressBound(CNBuffSize) so + * ZSTDMT is forced to not take the shortcut. + */ + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_rsyncable, 1) ); + CHECK( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize - 1, CNBuffer, CNBuffSize) ); + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : setting multithreaded parameters : ", testNb++) + { ZSTD_CCtx_params* params = ZSTD_createCCtxParams(); + int value; + /* Check that the overlap log and job size are unset. */ + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); + CHECK_EQ(value, 0); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); + CHECK_EQ(value, 0); + /* Set and check the overlap log and job size. */ + CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_overlapLog, 5) ); + CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_jobSize, 2 MB) ); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); + CHECK_EQ(value, 5); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); + CHECK_EQ(value, 2 MB); + /* Set the number of workers and check the overlap log and job size. */ + CHECK( ZSTD_CCtxParams_setParameter(params, ZSTD_c_nbWorkers, 2) ); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_overlapLog, &value) ); + CHECK_EQ(value, 5); + CHECK( ZSTD_CCtxParams_getParameter(params, ZSTD_c_jobSize, &value) ); + CHECK_EQ(value, 2 MB); + ZSTD_freeCCtxParams(params); + + } + DISPLAYLEVEL(3, "OK \n"); + + /* Simple API multiframe test */ + DISPLAYLEVEL(3, "test%3i : compress multiple frames : ", testNb++); + { size_t off = 0; + int i; + int const segs = 4; + /* only use the first half so we don't push against size limit of compressedBuffer */ + size_t const segSize = (CNBuffSize / 2) / segs; + for (i = 0; i < segs; i++) { + CHECK_NEWV(r, ZSTD_compress( + (BYTE*)compressedBuffer + off, CNBuffSize - off, + (BYTE*)CNBuffer + segSize * (size_t)i, segSize, + 5) ); + off += r; + if (i == segs/2) { + /* insert skippable frame */ + const U32 skipLen = 129 KB; + MEM_writeLE32((BYTE*)compressedBuffer + off, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32((BYTE*)compressedBuffer + off + 4, skipLen); + off += skipLen + ZSTD_SKIPPABLEHEADERSIZE; + } + } + cSize = off; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : get decompressed size of multiple frames : ", testNb++); + { unsigned long long const r = ZSTD_findDecompressedSize(compressedBuffer, cSize); + if (r != CNBuffSize / 2) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : get tight decompressed bound of multiple frames : ", testNb++); + { unsigned long long const bound = ZSTD_decompressBound(compressedBuffer, cSize); + if (bound != CNBuffSize / 2) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress multiple frames : ", testNb++); + { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, CNBuffSize, compressedBuffer, cSize)); + if (r != CNBuffSize / 2) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); + if (memcmp(decodedBuffer, CNBuffer, CNBuffSize / 2) != 0) goto _output_error; + DISPLAYLEVEL(3, "OK \n"); + + /* Dictionary and CCtx Duplication tests */ + { ZSTD_CCtx* const ctxOrig = ZSTD_createCCtx(); + ZSTD_CCtx* const ctxDuplicated = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + static const size_t dictSize = 551; + assert(dctx != NULL); assert(ctxOrig != NULL); assert(ctxDuplicated != NULL); + + DISPLAYLEVEL(3, "test%3i : copy context too soon : ", testNb++); + { size_t const copyResult = ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0); + if (!ZSTD_isError(copyResult)) goto _output_error; } /* error must be detected */ + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : load dictionary into context : ", testNb++); + CHECK( ZSTD_compressBegin_usingDict(ctxOrig, CNBuffer, dictSize, 2) ); + CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, 0) ); /* Begin_usingDict implies unknown srcSize, so match that */ + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : compress with flat dictionary : ", testNb++); + cSize = 0; + CHECKPLUS(r, ZSTD_compressEnd(ctxOrig, + compressedBuffer, compressedBufferSize, + (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), + cSize += r); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : frame built with flat dictionary should be decompressible : ", testNb++); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + CNBuffer, dictSize), + if (r != CNBuffSize - dictSize) goto _output_error); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : compress with duplicated context : ", testNb++); + { size_t const cSizeOrig = cSize; + cSize = 0; + CHECKPLUS(r, ZSTD_compressEnd(ctxDuplicated, + compressedBuffer, compressedBufferSize, + (const char*)CNBuffer + dictSize, CNBuffSize - dictSize), + cSize += r); + if (cSize != cSizeOrig) goto _output_error; /* should be identical ==> same size */ + } + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : frame built with duplicated context should be decompressible : ", testNb++); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + CNBuffer, dictSize), + if (r != CNBuffSize - dictSize) goto _output_error); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress with DDict : ", testNb++); + { ZSTD_DDict* const ddict = ZSTD_createDDict(CNBuffer, dictSize); + size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); + if (r != CNBuffSize - dictSize) goto _output_error; + DISPLAYLEVEL(3, "OK (size of DDict : %u) \n", (unsigned)ZSTD_sizeof_DDict(ddict)); + ZSTD_freeDDict(ddict); + } + + DISPLAYLEVEL(3, "test%3i : decompress with static DDict : ", testNb++); + { size_t const ddictBufferSize = ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); + void* const ddictBuffer = malloc(ddictBufferSize); + if (ddictBuffer == NULL) goto _output_error; + { const ZSTD_DDict* const ddict = ZSTD_initStaticDDict(ddictBuffer, ddictBufferSize, CNBuffer, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); + size_t const r = ZSTD_decompress_usingDDict(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, ddict); + if (r != CNBuffSize - dictSize) goto _output_error; + } + free(ddictBuffer); + DISPLAYLEVEL(3, "OK (size of static DDict : %u) \n", (unsigned)ddictBufferSize); + } + + DISPLAYLEVEL(3, "test%3i : check content size on duplicated context : ", testNb++); + { size_t const testSize = CNBuffSize / 3; + { ZSTD_parameters p = ZSTD_getParams(2, testSize, dictSize); + p.fParams.contentSizeFlag = 1; + CHECK( ZSTD_compressBegin_advanced(ctxOrig, CNBuffer, dictSize, p, testSize-1) ); + } + CHECK( ZSTD_copyCCtx(ctxDuplicated, ctxOrig, testSize) ); + + CHECK_VAR(cSize, ZSTD_compressEnd(ctxDuplicated, compressedBuffer, ZSTD_compressBound(testSize), + (const char*)CNBuffer + dictSize, testSize) ); + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize)) goto _output_error; + if ((zfh.frameContentSize != testSize) && (zfh.frameContentSize != 0)) goto _output_error; + } } + DISPLAYLEVEL(3, "OK \n"); + + if ((int)(compressibility * 100 + 0.1) == FUZ_compressibility_default) { /* test only valid with known input */ + size_t const flatdictSize = 22 KB; + size_t const contentSize = 9 KB; + const void* const dict = (const char*)CNBuffer; + const void* const contentStart = (const char*)dict + flatdictSize; + size_t const target_nodict_cSize[22+1] = { 3840, 3770, 3870, 3830, 3770, + 3770, 3770, 3770, 3750, 3750, + 3740, 3670, 3670, 3660, 3660, + 3660, 3660, 3660, 3660, 3660, + 3660, 3660, 3660 }; + size_t const target_wdict_cSize[22+1] = { 2830, 2890, 2890, 2820, 2940, + 2950, 2950, 2920, 2900, 2890, + 2910, 2910, 2910, 2770, 2760, + 2750, 2750, 2750, 2750, 2750, + 2750, 2750, 2750 }; + int l = 1; + int const maxLevel = ZSTD_maxCLevel(); + + DISPLAYLEVEL(3, "test%3i : flat-dictionary efficiency test : \n", testNb++); + assert(maxLevel == 22); + RDG_genBuffer(CNBuffer, flatdictSize + contentSize, compressibility, 0., seed); + DISPLAYLEVEL(4, "content hash : %016llx; dict hash : %016llx \n", XXH64(contentStart, contentSize, 0), XXH64(dict, flatdictSize, 0)); + + for ( ; l <= maxLevel; l++) { + size_t const nodict_cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, + contentStart, contentSize, l); + if (nodict_cSize > target_nodict_cSize[l]) { + DISPLAYLEVEL(1, "error : compression at level %i worse than expected (%u > %u) \n", + l, (unsigned)nodict_cSize, (unsigned)target_nodict_cSize[l]); + goto _output_error; + } + DISPLAYLEVEL(4, "level %i : max expected %u >= reached %u \n", + l, (unsigned)target_nodict_cSize[l], (unsigned)nodict_cSize); + } + for ( l=1 ; l <= maxLevel; l++) { + size_t const wdict_cSize = ZSTD_compress_usingDict(ctxOrig, + compressedBuffer, compressedBufferSize, + contentStart, contentSize, + dict, flatdictSize, + l); + if (wdict_cSize > target_wdict_cSize[l]) { + DISPLAYLEVEL(1, "error : compression with dictionary at level %i worse than expected (%u > %u) \n", + l, (unsigned)wdict_cSize, (unsigned)target_wdict_cSize[l]); + goto _output_error; + } + DISPLAYLEVEL(4, "level %i with dictionary : max expected %u >= reached %u \n", + l, (unsigned)target_wdict_cSize[l], (unsigned)wdict_cSize); + } + + DISPLAYLEVEL(4, "compression efficiency tests OK \n"); + } + + ZSTD_freeCCtx(ctxOrig); + ZSTD_freeCCtx(ctxDuplicated); + ZSTD_freeDCtx(dctx); + } + + /* Dictionary and dictBuilder tests */ + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const dictBufferCapacity = 16 KB; + void* const dictBuffer = malloc(dictBufferCapacity); + size_t const totalSampleSize = 1 MB; + size_t const sampleUnitSize = 8 KB; + U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); + size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); + size_t dictSize; + U32 dictID; + size_t dictHeaderSize; + + if (dictBuffer==NULL || samplesSizes==NULL) { + free(dictBuffer); + free(samplesSizes); + goto _output_error; + } + + DISPLAYLEVEL(3, "test%3i : dictBuilder on cyclic data : ", testNb++); + assert(compressedBufferSize >= totalSampleSize); + { U32 u; for (u=0; u<totalSampleSize; u++) ((BYTE*)decodedBuffer)[u] = (BYTE)u; } + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + { size_t const sDictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, + decodedBuffer, samplesSizes, nbSamples); + if (ZDICT_isError(sDictSize)) goto _output_error; + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)sDictSize); + } + + DISPLAYLEVEL(3, "test%3i : dictBuilder : ", testNb++); + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + dictSize = ZDICT_trainFromBuffer(dictBuffer, dictBufferCapacity, + CNBuffer, samplesSizes, nbSamples); + if (ZDICT_isError(dictSize)) goto _output_error; + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); + + DISPLAYLEVEL(3, "test%3i : Multithreaded COVER dictBuilder : ", testNb++); + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + { ZDICT_cover_params_t coverParams; + memset(&coverParams, 0, sizeof(coverParams)); + coverParams.steps = 8; + coverParams.nbThreads = 4; + dictSize = ZDICT_optimizeTrainFromBuffer_cover( + dictBuffer, dictBufferCapacity, + CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ + &coverParams); + if (ZDICT_isError(dictSize)) goto _output_error; + } + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); + + DISPLAYLEVEL(3, "test%3i : COVER dictBuilder with shrinkDict: ", testNb++); + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + { ZDICT_cover_params_t coverParams; + memset(&coverParams, 0, sizeof(coverParams)); + coverParams.steps = 8; + coverParams.nbThreads = 4; + coverParams.shrinkDict = 1; + coverParams.shrinkDictMaxRegression = 1; + dictSize = ZDICT_optimizeTrainFromBuffer_cover( + dictBuffer, dictBufferCapacity, + CNBuffer, samplesSizes, nbSamples/8, /* less samples for faster tests */ + &coverParams); + if (ZDICT_isError(dictSize)) goto _output_error; + } + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); + + DISPLAYLEVEL(3, "test%3i : Multithreaded FASTCOVER dictBuilder : ", testNb++); + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + { ZDICT_fastCover_params_t fastCoverParams; + memset(&fastCoverParams, 0, sizeof(fastCoverParams)); + fastCoverParams.steps = 8; + fastCoverParams.nbThreads = 4; + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( + dictBuffer, dictBufferCapacity, + CNBuffer, samplesSizes, nbSamples, + &fastCoverParams); + if (ZDICT_isError(dictSize)) goto _output_error; + } + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); + + DISPLAYLEVEL(3, "test%3i : FASTCOVER dictBuilder with shrinkDict: ", testNb++); + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + { ZDICT_fastCover_params_t fastCoverParams; + memset(&fastCoverParams, 0, sizeof(fastCoverParams)); + fastCoverParams.steps = 8; + fastCoverParams.nbThreads = 4; + fastCoverParams.shrinkDict = 1; + fastCoverParams.shrinkDictMaxRegression = 1; + dictSize = ZDICT_optimizeTrainFromBuffer_fastCover( + dictBuffer, dictBufferCapacity, + CNBuffer, samplesSizes, nbSamples, + &fastCoverParams); + if (ZDICT_isError(dictSize)) goto _output_error; + } + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); + + DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); + dictID = ZDICT_getDictID(dictBuffer, dictSize); + if (dictID==0) goto _output_error; + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); + + DISPLAYLEVEL(3, "test%3i : check dict header size no error : ", testNb++); + dictHeaderSize = ZDICT_getDictHeaderSize(dictBuffer, dictSize); + if (dictHeaderSize==0) goto _output_error; + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); + + DISPLAYLEVEL(3, "test%3i : check dict header size correctness : ", testNb++); + { unsigned char const dictBufferFixed[144] = { 0x37, 0xa4, 0x30, 0xec, 0x63, 0x00, 0x00, 0x00, 0x08, 0x10, 0x00, 0x1f, + 0x0f, 0x00, 0x28, 0xe5, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x80, 0x0f, 0x9e, 0x0f, 0x00, 0x00, 0x24, 0x40, 0x80, 0x00, 0x01, + 0x02, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0xde, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, + 0x08, 0x08, 0x08, 0x08, 0xbc, 0xe1, 0x4b, 0x92, 0x0e, 0xb4, 0x7b, 0x18, + 0x86, 0x61, 0x18, 0xc6, 0x18, 0x63, 0x8c, 0x31, 0xc6, 0x18, 0x63, 0x8c, + 0x31, 0x66, 0x66, 0x66, 0x66, 0xb6, 0x6d, 0x01, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x20, 0x73, 0x6f, 0x64, 0x61, + 0x6c, 0x65, 0x73, 0x20, 0x74, 0x6f, 0x72, 0x74, 0x6f, 0x72, 0x20, 0x65, + 0x6c, 0x65, 0x69, 0x66, 0x65, 0x6e, 0x64, 0x2e, 0x20, 0x41, 0x6c, 0x69 }; + dictHeaderSize = ZDICT_getDictHeaderSize(dictBufferFixed, 144); + if (dictHeaderSize != 115) goto _output_error; + } + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictHeaderSize); + + DISPLAYLEVEL(3, "test%3i : compress with dictionary : ", testNb++); + cSize = ZSTD_compress_usingDict(cctx, compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, + dictBuffer, dictSize, 4); + if (ZSTD_isError(cSize)) goto _output_error; + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : retrieve dictID from dictionary : ", testNb++); + { U32 const did = ZSTD_getDictID_fromDict(dictBuffer, dictSize); + if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); + { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); + if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : estimate CDict size : ", testNb++); + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); + size_t const estimatedSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byRef); + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)estimatedSize); + } + + DISPLAYLEVEL(3, "test%3i : compress with CDict ", testNb++); + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, + ZSTD_dlm_byRef, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + assert(cdict != NULL); + DISPLAYLEVEL(3, "(size : %u) : ", (unsigned)ZSTD_sizeof_CDict(cdict)); + cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, cdict); + ZSTD_freeCDict(cdict); + if (ZSTD_isError(cSize)) goto _output_error; + } + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : retrieve dictID from frame : ", testNb++); + { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); + if (did != dictID) goto _output_error; /* non-conformant (content-only) dictionary */ + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : frame built with dictionary should be decompressible : ", testNb++); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : compress with static CDict : ", testNb++); + { int const maxLevel = ZSTD_maxCLevel(); + int level; + for (level = 1; level <= maxLevel; ++level) { + ZSTD_compressionParameters const cParams = ZSTD_getCParams(level, CNBuffSize, dictSize); + size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); + void* const cdictBuffer = malloc(cdictSize); + if (cdictBuffer==NULL) goto _output_error; + { const ZSTD_CDict* const cdict = ZSTD_initStaticCDict( + cdictBuffer, cdictSize, + dictBuffer, dictSize, + ZSTD_dlm_byCopy, ZSTD_dct_auto, + cParams); + if (cdict == NULL) { + DISPLAY("ZSTD_initStaticCDict failed "); + goto _output_error; + } + cSize = ZSTD_compress_usingCDict(cctx, + compressedBuffer, compressedBufferSize, + CNBuffer, MIN(10 KB, CNBuffSize), cdict); + if (ZSTD_isError(cSize)) { + DISPLAY("ZSTD_compress_usingCDict failed "); + goto _output_error; + } } + free(cdictBuffer); + } } + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compress_usingCDict_advanced, no contentSize, no dictID : ", testNb++); + { ZSTD_frameParameters const fParams = { 0 /* frameSize */, 1 /* checksum */, 1 /* noDictID*/ }; + ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); + assert(cdict != NULL); + cSize = ZSTD_compress_usingCDict_advanced(cctx, + compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, + cdict, fParams); + ZSTD_freeCDict(cdict); + if (ZSTD_isError(cSize)) goto _output_error; + } + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : try retrieving contentSize from frame : ", testNb++); + { U64 const contentSize = ZSTD_getFrameContentSize(compressedBuffer, cSize); + if (contentSize != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; + } + DISPLAYLEVEL(3, "OK (unknown)\n"); + + DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_compress_advanced, no dictID : ", testNb++); + { ZSTD_parameters p = ZSTD_getParams(3, CNBuffSize, dictSize); + p.fParams.noDictIDFlag = 1; + cSize = ZSTD_compress_advanced(cctx, compressedBuffer, compressedBufferSize, + CNBuffer, CNBuffSize, + dictBuffer, dictSize, p); + if (ZSTD_isError(cSize)) goto _output_error; + } + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBuffSize*100); + + DISPLAYLEVEL(3, "test%3i : frame built without dictID should be decompressible : ", testNb++); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); assert(dctx != NULL); + CHECKPLUS(r, ZSTD_decompress_usingDict(dctx, + decodedBuffer, CNBuffSize, + compressedBuffer, cSize, + dictBuffer, dictSize), + if (r != CNBuffSize) goto _output_error); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : dictionary containing only header should return error : ", testNb++); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + assert(dctx != NULL); + { const size_t ret = ZSTD_decompress_usingDict( + dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize, + "\x37\xa4\x30\xec\x11\x22\x33\x44", 8); + if (ZSTD_getErrorCode(ret) != ZSTD_error_dictionary_corrupted) + goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a good dictionary : ", testNb++); + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); + if (cdict==NULL) goto _output_error; + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Building cdict w/ ZSTD_dct_fullDict on a rawContent (must fail) : ", testNb++); + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBuffSize, dictSize); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced((const char*)dictBuffer+1, dictSize-1, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); + if (cdict!=NULL) goto _output_error; + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_auto should fail : ", testNb++); + { + size_t ret; + MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); + /* Either operation is allowed to fail, but one must fail. */ + ret = ZSTD_CCtx_loadDictionary_advanced( + cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_auto); + if (!ZSTD_isError(ret)) { + ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); + if (!ZSTD_isError(ret)) goto _output_error; + } + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading rawContent starting with dict header w/ ZSTD_dct_rawContent should pass : ", testNb++); + { + size_t ret; + MEM_writeLE32((char*)dictBuffer+2, ZSTD_MAGIC_DICTIONARY); + ret = ZSTD_CCtx_loadDictionary_advanced( + cctx, (const char*)dictBuffer+2, dictSize-2, ZSTD_dlm_byRef, ZSTD_dct_rawContent); + if (ZSTD_isError(ret)) goto _output_error; + ret = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)); + if (ZSTD_isError(ret)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_CCtx_refCDict() then set parameters : ", testNb++); + { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 1) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, 12 )); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading dictionary before setting parameters is the same as loading after : ", testNb++); + { + size_t size1, size2; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); + size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size1)) goto _output_error; + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); + CHECK_Z( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 7) ); + size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size2)) goto _output_error; + + if (size1 != size2) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the prefix : ", testNb++); + { + CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a dictionary clears the cdict : ", testNb++); + { + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the prefix : ", testNb++); + { + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a cdict clears the dictionary : ", testNb++); + { + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the dictionary : ", testNb++); + { + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loading a prefix clears the cdict : ", testNb++); + { + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, 1); + CHECK_Z( ZSTD_CCtx_refCDict(cctx, cdict) ); + CHECK_Z( ZSTD_CCtx_refPrefix(cctx, (const char*)dictBuffer, dictSize) ); + CHECK_Z( ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100)) ); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loaded dictionary persists across reset session : ", testNb++); + { + size_t size1, size2; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); + size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size1)) goto _output_error; + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_only); + size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size2)) goto _output_error; + + if (size1 != size2) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Loaded dictionary is cleared after resetting parameters : ", testNb++); + { + size_t size1, size2; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, CNBuffer, MIN(CNBuffSize, 10 KB)) ); + size1 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size1)) goto _output_error; + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + size2 = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + if (ZSTD_isError(size2)) goto _output_error; + + if (size1 == size2) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(cctx, dictBuffer, dictSize) ); + cSize = ZSTD_compress2(cctx, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBuffSize, 100 KB)); + CHECK_Z(cSize); + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with dictionary : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t ret; + /* We should fail to decompress without a dictionary. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + /* We should succeed to decompress with the dictionary. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictBuffer, dictSize) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* The dictionary should presist across calls. */ + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* When we reset the context the dictionary is cleared. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with ddict : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictSize); + size_t ret; + /* We should succeed to decompress with the ddict. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* The ddict should presist across calls. */ + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* When we reset the context the ddict is cleared. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + ZSTD_freeDCtx(dctx); + ZSTD_freeDDict(ddict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t ret; + /* We should succeed to decompress with the prefix. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictBuffer, dictSize, ZSTD_dct_auto) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + /* The prefix should be cleared after the first compression. */ + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Dictionary with non-default repcodes : ", testNb++); + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, + CNBuffer, samplesSizes, nbSamples); + if (ZDICT_isError(dictSize)) goto _output_error; + /* Set all the repcodes to non-default */ + { + BYTE* dictPtr = (BYTE*)dictBuffer; + BYTE* dictLimit = dictPtr + dictSize - 12; + /* Find the repcodes */ + while (dictPtr < dictLimit && + (MEM_readLE32(dictPtr) != 1 || MEM_readLE32(dictPtr + 4) != 4 || + MEM_readLE32(dictPtr + 8) != 8)) { + ++dictPtr; + } + if (dictPtr >= dictLimit) goto _output_error; + MEM_writeLE32(dictPtr + 0, 10); + MEM_writeLE32(dictPtr + 4, 10); + MEM_writeLE32(dictPtr + 8, 10); + /* Set the last 8 bytes to 'x' */ + memset((BYTE*)dictBuffer + dictSize - 8, 'x', 8); + } + /* The optimal parser checks all the repcodes. + * Make sure at least one is a match >= targetLength so that it is + * immediately chosen. This will make sure that the compressor and + * decompressor agree on at least one of the repcodes. + */ + { size_t dSize; + BYTE data[1024]; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + ZSTD_compressionParameters const cParams = ZSTD_getCParams(19, CNBuffSize, dictSize); + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictSize, + ZSTD_dlm_byRef, ZSTD_dct_auto, + cParams, ZSTD_defaultCMem); + assert(dctx != NULL); assert(cdict != NULL); + memset(data, 'x', sizeof(data)); + cSize = ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, + data, sizeof(data), cdict); + ZSTD_freeCDict(cdict); + if (ZSTD_isError(cSize)) { DISPLAYLEVEL(5, "Compression error %s : ", ZSTD_getErrorName(cSize)); goto _output_error; } + dSize = ZSTD_decompress_usingDict(dctx, decodedBuffer, sizeof(data), compressedBuffer, cSize, dictBuffer, dictSize); + if (ZSTD_isError(dSize)) { DISPLAYLEVEL(5, "Decompression error %s : ", ZSTD_getErrorName(dSize)); goto _output_error; } + if (memcmp(data, decodedBuffer, sizeof(data))) { DISPLAYLEVEL(5, "Data corruption : "); goto _output_error; } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + ZSTD_freeCCtx(cctx); + free(dictBuffer); + free(samplesSizes); + } + + /* COVER dictionary builder tests */ + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t dictSize = 16 KB; + size_t optDictSize = dictSize; + void* dictBuffer = malloc(dictSize); + size_t const totalSampleSize = 1 MB; + size_t const sampleUnitSize = 8 KB; + U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); + size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); + U32 seed32 = seed; + ZDICT_cover_params_t params; + U32 dictID; + + if (dictBuffer==NULL || samplesSizes==NULL) { + free(dictBuffer); + free(samplesSizes); + goto _output_error; + } + + DISPLAYLEVEL(3, "test%3i : ZDICT_trainFromBuffer_cover : ", testNb++); + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + memset(¶ms, 0, sizeof(params)); + params.d = 1 + (FUZ_rand(&seed32) % 16); + params.k = params.d + (FUZ_rand(&seed32) % 256); + dictSize = ZDICT_trainFromBuffer_cover(dictBuffer, dictSize, + CNBuffer, samplesSizes, nbSamples, + params); + if (ZDICT_isError(dictSize)) goto _output_error; + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)dictSize); + + DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); + dictID = ZDICT_getDictID(dictBuffer, dictSize); + if (dictID==0) goto _output_error; + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); + + DISPLAYLEVEL(3, "test%3i : ZDICT_optimizeTrainFromBuffer_cover : ", testNb++); + memset(¶ms, 0, sizeof(params)); + params.steps = 4; + optDictSize = ZDICT_optimizeTrainFromBuffer_cover(dictBuffer, optDictSize, + CNBuffer, samplesSizes, + nbSamples / 4, ¶ms); + if (ZDICT_isError(optDictSize)) goto _output_error; + DISPLAYLEVEL(3, "OK, created dictionary of size %u \n", (unsigned)optDictSize); + + DISPLAYLEVEL(3, "test%3i : check dictID : ", testNb++); + dictID = ZDICT_getDictID(dictBuffer, optDictSize); + if (dictID==0) goto _output_error; + DISPLAYLEVEL(3, "OK : %u \n", (unsigned)dictID); + + ZSTD_freeCCtx(cctx); + free(dictBuffer); + free(samplesSizes); + } + + /* Decompression defense tests */ + DISPLAYLEVEL(3, "test%3i : Check input length for magic number : ", testNb++); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 3); /* too small input */ + if (!ZSTD_isError(r)) goto _output_error; + if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Check magic Number : ", testNb++); + ((char*)(CNBuffer))[0] = 1; + { size_t const r = ZSTD_decompress(decodedBuffer, CNBuffSize, CNBuffer, 4); + if (!ZSTD_isError(r)) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + /* content size verification test */ + DISPLAYLEVEL(3, "test%3i : Content size verification : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const srcSize = 5000; + size_t const wrongSrcSize = (srcSize + 1000); + ZSTD_parameters params = ZSTD_getParams(1, wrongSrcSize, 0); + params.fParams.contentSizeFlag = 1; + CHECK( ZSTD_compressBegin_advanced(cctx, NULL, 0, params, wrongSrcSize) ); + { size_t const result = ZSTD_compressEnd(cctx, decodedBuffer, CNBuffSize, CNBuffer, srcSize); + if (!ZSTD_isError(result)) goto _output_error; + if (ZSTD_getErrorCode(result) != ZSTD_error_srcSize_wrong) goto _output_error; + DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(result)); + } + ZSTD_freeCCtx(cctx); + } + + /* negative compression level test : ensure simple API and advanced API produce same result */ + DISPLAYLEVEL(3, "test%3i : negative compression level : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const srcSize = CNBuffSize / 5; + int const compressionLevel = -1; + + assert(cctx != NULL); + { ZSTD_parameters const params = ZSTD_getParams(compressionLevel, srcSize, 0); + size_t const cSize_1pass = ZSTD_compress_advanced(cctx, + compressedBuffer, compressedBufferSize, + CNBuffer, srcSize, + NULL, 0, + params); + if (ZSTD_isError(cSize_1pass)) goto _output_error; + + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, compressionLevel) ); + { size_t const compressionResult = ZSTD_compress2(cctx, + compressedBuffer, compressedBufferSize, + CNBuffer, srcSize); + DISPLAYLEVEL(5, "simple=%zu vs %zu=advanced : ", cSize_1pass, compressionResult); + if (ZSTD_isError(compressionResult)) goto _output_error; + if (compressionResult != cSize_1pass) goto _output_error; + } } + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + /* parameters order test */ + { size_t const inputSize = CNBuffSize / 2; + U64 xxh64; + + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + DISPLAYLEVEL(3, "test%3i : parameters in order : ", testNb++); + assert(cctx != NULL); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); + { size_t const compressedSize = ZSTD_compress2(cctx, + compressedBuffer, ZSTD_compressBound(inputSize), + CNBuffer, inputSize); + CHECK(compressedSize); + cSize = compressedSize; + xxh64 = XXH64(compressedBuffer, compressedSize, 0); + } + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); + ZSTD_freeCCtx(cctx); + } + + { ZSTD_CCtx* cctx = ZSTD_createCCtx(); + DISPLAYLEVEL(3, "test%3i : parameters disordered : ", testNb++); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 18) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1) ); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, 2) ); + { size_t const result = ZSTD_compress2(cctx, + compressedBuffer, ZSTD_compressBound(inputSize), + CNBuffer, inputSize); + CHECK(result); + if (result != cSize) goto _output_error; /* must result in same compressed result, hence same size */ + if (XXH64(compressedBuffer, result, 0) != xxh64) goto _output_error; /* must result in exactly same content, hence same hash */ + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)result); + } + ZSTD_freeCCtx(cctx); + } + } + + /* advanced parameters for decompression */ + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + assert(dctx != NULL); + + DISPLAYLEVEL(3, "test%3i : get dParameter bounds ", testNb++); + { ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + CHECK(bounds.error); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : wrong dParameter : ", testNb++); + { size_t const sr = ZSTD_DCtx_setParameter(dctx, (ZSTD_dParameter)999999, 0); + if (!ZSTD_isError(sr)) goto _output_error; + } + { ZSTD_bounds const bounds = ZSTD_dParam_getBounds((ZSTD_dParameter)999998); + if (!ZSTD_isError(bounds.error)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : out of bound dParameter : ", testNb++); + { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_windowLogMax, 9999); + if (!ZSTD_isError(sr)) goto _output_error; + } + { size_t const sr = ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (ZSTD_format_e)888); + if (!ZSTD_isError(sr)) goto _output_error; + } + DISPLAYLEVEL(3, "OK \n"); + + ZSTD_freeDCtx(dctx); + } + + + /* custom formats tests */ + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + size_t const inputSize = CNBuffSize / 2; /* won't cause pb with small dict size */ + assert(dctx != NULL); assert(cctx != NULL); + + /* basic block compression */ + DISPLAYLEVEL(3, "test%3i : magic-less format test : ", testNb++); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); + { ZSTD_inBuffer in = { CNBuffer, inputSize, 0 }; + ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(inputSize), 0 }; + size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); + if (result != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + cSize = out.pos; + } + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)inputSize, (unsigned)cSize); + + DISPLAYLEVEL(3, "test%3i : decompress normally (should fail) : ", testNb++); + { size_t const decodeResult = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (ZSTD_getErrorCode(decodeResult) != ZSTD_error_prefix_unknown) goto _output_error; + DISPLAYLEVEL(3, "OK : %s \n", ZSTD_getErrorName(decodeResult)); + } + + DISPLAYLEVEL(3, "test%3i : decompress of magic-less frame : ", testNb++); + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); + { ZSTD_frameHeader zfh; + size_t const zfhrt = ZSTD_getFrameHeader_advanced(&zfh, compressedBuffer, cSize, ZSTD_f_zstd1_magicless); + if (zfhrt != 0) goto _output_error; + } + /* one shot */ + { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (result != inputSize) goto _output_error; + DISPLAYLEVEL(3, "one-shot OK, "); + } + /* streaming */ + { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; + ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; + size_t const result = ZSTD_decompressStream(dctx, &out, &in); + if (result != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + if (out.pos != inputSize) goto _output_error; + DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); + } + + /* basic block compression */ + DISPLAYLEVEL(3, "test%3i : empty magic-less format test : ", testNb++); + CHECK( ZSTD_CCtx_setParameter(cctx, ZSTD_c_format, ZSTD_f_zstd1_magicless) ); + { ZSTD_inBuffer in = { CNBuffer, 0, 0 }; + ZSTD_outBuffer out = { compressedBuffer, ZSTD_compressBound(0), 0 }; + size_t const result = ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end); + if (result != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + cSize = out.pos; + } + DISPLAYLEVEL(3, "OK (compress : %u -> %u bytes)\n", (unsigned)0, (unsigned)cSize); + + DISPLAYLEVEL(3, "test%3i : decompress of empty magic-less frame : ", testNb++); + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK( ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, ZSTD_f_zstd1_magicless) ); + /* one shot */ + { size_t const result = ZSTD_decompressDCtx(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize); + if (result != 0) goto _output_error; + DISPLAYLEVEL(3, "one-shot OK, "); + } + /* streaming */ + { ZSTD_inBuffer in = { compressedBuffer, cSize, 0 }; + ZSTD_outBuffer out = { decodedBuffer, CNBuffSize, 0 }; + size_t const result = ZSTD_decompressStream(dctx, &out, &in); + if (result != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + if (out.pos != 0) goto _output_error; + DISPLAYLEVEL(3, "streaming OK : regenerated %u bytes \n", (unsigned)out.pos); + } + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + + /* block API tests */ + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + static const size_t dictSize = 65 KB; + static const size_t blockSize = 100 KB; /* won't cause pb with small dict size */ + size_t cSize2; + assert(cctx != NULL); assert(dctx != NULL); + + /* basic block compression */ + DISPLAYLEVEL(3, "test%3i : Block compression test : ", testNb++); + CHECK( ZSTD_compressBegin(cctx, 5) ); + CHECK( ZSTD_getBlockSize(cctx) >= blockSize); + CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize) ); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Block decompression test : ", testNb++); + CHECK( ZSTD_decompressBegin(dctx) ); + { CHECK_NEWV(r, ZSTD_decompressBlock(dctx, decodedBuffer, CNBuffSize, compressedBuffer, cSize) ); + if (r != blockSize) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + /* very long stream of block compression */ + DISPLAYLEVEL(3, "test%3i : Huge block streaming compression test : ", testNb++); + CHECK( ZSTD_compressBegin(cctx, -199) ); /* we just want to quickly overflow internal U32 index */ + CHECK( ZSTD_getBlockSize(cctx) >= blockSize); + { U64 const toCompress = 5000000000ULL; /* > 4 GB */ + U64 compressed = 0; + while (compressed < toCompress) { + size_t const blockCSize = ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), CNBuffer, blockSize); + assert(blockCSize != 0); + if (ZSTD_isError(blockCSize)) goto _output_error; + compressed += blockCSize; + } } + DISPLAYLEVEL(3, "OK \n"); + + /* dictionary block compression */ + DISPLAYLEVEL(3, "test%3i : Dictionary Block compression test : ", testNb++); + CHECK( ZSTD_compressBegin_usingDict(cctx, CNBuffer, dictSize, 5) ); + CHECK_VAR(cSize, ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize)); + RDG_genBuffer((char*)CNBuffer+dictSize+blockSize, blockSize, 0.0, 0.0, seed); /* create a non-compressible second block */ + { CHECK_NEWV(r, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize+blockSize, blockSize) ); /* for cctx history consistency */ + assert(r == 0); /* non-compressible block */ } + memcpy((char*)compressedBuffer+cSize, (char*)CNBuffer+dictSize+blockSize, blockSize); /* send non-compressed block (without header) */ + CHECK_VAR(cSize2, ZSTD_compressBlock(cctx, (char*)compressedBuffer+cSize+blockSize, ZSTD_compressBound(blockSize), + (char*)CNBuffer+dictSize+2*blockSize, blockSize)); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Dictionary Block decompression test : ", testNb++); + CHECK( ZSTD_decompressBegin_usingDict(dctx, CNBuffer, dictSize) ); + { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, decodedBuffer, blockSize, compressedBuffer, cSize) ); + if (r != blockSize) { + DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); + goto _output_error; + } } + memcpy((char*)decodedBuffer+blockSize, (char*)compressedBuffer+cSize, blockSize); + ZSTD_insertBlock(dctx, (char*)decodedBuffer+blockSize, blockSize); /* insert non-compressed block into dctx history */ + { CHECK_NEWV( r, ZSTD_decompressBlock(dctx, (char*)decodedBuffer+2*blockSize, blockSize, (char*)compressedBuffer+cSize+blockSize, cSize2) ); + if (r != blockSize) { + DISPLAYLEVEL(1, "ZSTD_decompressBlock() with _usingDict() and after insertBlock() fails : %u, instead of %u expected \n", (unsigned)r, (unsigned)blockSize); + goto _output_error; + } } + assert(memcpy((char*)CNBuffer+dictSize, decodedBuffer, blockSize*3)); /* ensure regenerated content is identical to origin */ + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : Block compression with CDict : ", testNb++); + { ZSTD_CDict* const cdict = ZSTD_createCDict(CNBuffer, dictSize, 3); + if (cdict==NULL) goto _output_error; + CHECK( ZSTD_compressBegin_usingCDict(cctx, cdict) ); + CHECK( ZSTD_compressBlock(cctx, compressedBuffer, ZSTD_compressBound(blockSize), (char*)CNBuffer+dictSize, blockSize) ); + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + } + + /* long rle test */ + { size_t sampleSize = 0; + size_t expectedCompressedSize = 39; /* block 1, 2: compressed, block 3: RLE, zstd 1.4.4 */ + DISPLAYLEVEL(3, "test%3i : Long RLE test : ", testNb++); + memset((char*)CNBuffer+sampleSize, 'B', 256 KB - 1); + sampleSize += 256 KB - 1; + memset((char*)CNBuffer+sampleSize, 'A', 96 KB); + sampleSize += 96 KB; + cSize = ZSTD_compress(compressedBuffer, ZSTD_compressBound(sampleSize), CNBuffer, sampleSize, 1); + if (ZSTD_isError(cSize) || cSize > expectedCompressedSize) goto _output_error; + { CHECK_NEWV(regenSize, ZSTD_decompress(decodedBuffer, sampleSize, compressedBuffer, cSize)); + if (regenSize!=sampleSize) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + } + + DISPLAYLEVEL(3, "test%3i : ZSTD_getSequences decode from sequences test : ", testNb++); + { + size_t srcSize = 100 KB; + BYTE* src = (BYTE*)CNBuffer; + BYTE* decoded = (BYTE*)compressedBuffer; + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_Sequence* seqs = (ZSTD_Sequence*)malloc(srcSize * sizeof(ZSTD_Sequence)); + size_t seqsSize; + + if (seqs == NULL) goto _output_error; + assert(cctx != NULL); + + /* Populate src with random data */ + RDG_genBuffer(CNBuffer, srcSize, compressibility, 0., seed); + + /* get the sequences */ + seqsSize = ZSTD_getSequences(cctx, seqs, srcSize, src, srcSize); + + /* "decode" and compare the sequences */ + FUZ_decodeSequences(decoded, seqs, seqsSize, src, srcSize); + assert(!memcmp(CNBuffer, compressedBuffer, srcSize)); + + ZSTD_freeCCtx(cctx); + free(seqs); + } + + /* Multiple blocks of zeros test */ + #define LONGZEROSLENGTH 1000000 /* 1MB of zeros */ + DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, LONGZEROSLENGTH); + memset(CNBuffer, 0, LONGZEROSLENGTH); + CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(LONGZEROSLENGTH), CNBuffer, LONGZEROSLENGTH, 1) ); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/LONGZEROSLENGTH*100); + + DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, LONGZEROSLENGTH); + { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, LONGZEROSLENGTH, compressedBuffer, cSize) ); + if (r != LONGZEROSLENGTH) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + /* All zeroes test (test bug #137) */ + #define ZEROESLENGTH 100 + DISPLAYLEVEL(3, "test%3i : compress %u zeroes : ", testNb++, ZEROESLENGTH); + memset(CNBuffer, 0, ZEROESLENGTH); + CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(ZEROESLENGTH), CNBuffer, ZEROESLENGTH, 1) ); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/ZEROESLENGTH*100); + + DISPLAYLEVEL(3, "test%3i : decompress %u zeroes : ", testNb++, ZEROESLENGTH); + { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, ZEROESLENGTH, compressedBuffer, cSize) ); + if (r != ZEROESLENGTH) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + /* nbSeq limit test */ + #define _3BYTESTESTLENGTH 131000 + #define NB3BYTESSEQLOG 9 + #define NB3BYTESSEQ (1 << NB3BYTESSEQLOG) + #define NB3BYTESSEQMASK (NB3BYTESSEQ-1) + /* creates a buffer full of 3-bytes sequences */ + { BYTE _3BytesSeqs[NB3BYTESSEQ][3]; + U32 rSeed = 1; + + /* create batch of 3-bytes sequences */ + { int i; + for (i=0; i < NB3BYTESSEQ; i++) { + _3BytesSeqs[i][0] = (BYTE)(FUZ_rand(&rSeed) & 255); + _3BytesSeqs[i][1] = (BYTE)(FUZ_rand(&rSeed) & 255); + _3BytesSeqs[i][2] = (BYTE)(FUZ_rand(&rSeed) & 255); + } } + + /* randomly fills CNBuffer with prepared 3-bytes sequences */ + { int i; + for (i=0; i < _3BYTESTESTLENGTH; i += 3) { /* note : CNBuffer size > _3BYTESTESTLENGTH+3 */ + U32 const id = FUZ_rand(&rSeed) & NB3BYTESSEQMASK; + ((BYTE*)CNBuffer)[i+0] = _3BytesSeqs[id][0]; + ((BYTE*)CNBuffer)[i+1] = _3BytesSeqs[id][1]; + ((BYTE*)CNBuffer)[i+2] = _3BytesSeqs[id][2]; + } } } + DISPLAYLEVEL(3, "test%3i : growing nbSeq : ", testNb++); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const maxNbSeq = _3BYTESTESTLENGTH / 3; + size_t const bound = ZSTD_compressBound(_3BYTESTESTLENGTH); + size_t nbSeq = 1; + while (nbSeq <= maxNbSeq) { + CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, nbSeq * 3, 19)); + /* Check every sequence for the first 100, then skip more rapidly. */ + if (nbSeq < 100) { + ++nbSeq; + } else { + nbSeq += (nbSeq >> 2); + } + } + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : compress lots 3-bytes sequences : ", testNb++); + CHECK_VAR(cSize, ZSTD_compress(compressedBuffer, ZSTD_compressBound(_3BYTESTESTLENGTH), + CNBuffer, _3BYTESTESTLENGTH, 19) ); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/_3BYTESTESTLENGTH*100); + + DISPLAYLEVEL(3, "test%3i : decompress lots 3-bytes sequence : ", testNb++); + { CHECK_NEWV(r, ZSTD_decompress(decodedBuffer, _3BYTESTESTLENGTH, compressedBuffer, cSize) ); + if (r != _3BYTESTESTLENGTH) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + + DISPLAYLEVEL(3, "test%3i : growing literals buffer : ", testNb++); + RDG_genBuffer(CNBuffer, CNBuffSize, 0.0, 0.1, seed); + { ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + size_t const bound = ZSTD_compressBound(CNBuffSize); + size_t size = 1; + while (size <= CNBuffSize) { + CHECK(ZSTD_compressCCtx(cctx, compressedBuffer, bound, CNBuffer, size, 3)); + /* Check every size for the first 100, then skip more rapidly. */ + if (size < 100) { + ++size; + } else { + size += (size >> 2); + } + } + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : incompressible data and ill suited dictionary : ", testNb++); + { /* Train a dictionary on low characters */ + size_t dictSize = 16 KB; + void* const dictBuffer = malloc(dictSize); + size_t const totalSampleSize = 1 MB; + size_t const sampleUnitSize = 8 KB; + U32 const nbSamples = (U32)(totalSampleSize / sampleUnitSize); + size_t* const samplesSizes = (size_t*) malloc(nbSamples * sizeof(size_t)); + if (!dictBuffer || !samplesSizes) goto _output_error; + { U32 u; for (u=0; u<nbSamples; u++) samplesSizes[u] = sampleUnitSize; } + dictSize = ZDICT_trainFromBuffer(dictBuffer, dictSize, CNBuffer, samplesSizes, nbSamples); + if (ZDICT_isError(dictSize)) goto _output_error; + /* Reverse the characters to make the dictionary ill suited */ + { U32 u; + for (u = 0; u < CNBuffSize; ++u) { + ((BYTE*)CNBuffer)[u] = 255 - ((BYTE*)CNBuffer)[u]; + } + } + { /* Compress the data */ + size_t const inputSize = 500; + size_t const outputSize = ZSTD_compressBound(inputSize); + void* const outputBuffer = malloc(outputSize); + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + if (!outputBuffer || !cctx) goto _output_error; + CHECK(ZSTD_compress_usingDict(cctx, outputBuffer, outputSize, CNBuffer, inputSize, dictBuffer, dictSize, 1)); + free(outputBuffer); + ZSTD_freeCCtx(cctx); + } + + free(dictBuffer); + free(samplesSizes); + } + DISPLAYLEVEL(3, "OK \n"); + + + /* findFrameCompressedSize on skippable frames */ + DISPLAYLEVEL(3, "test%3i : frame compressed size of skippable frame : ", testNb++); + { const char* frame = "\x50\x2a\x4d\x18\x05\x0\x0\0abcde"; + size_t const frameSrcSize = 13; + if (ZSTD_findFrameCompressedSize(frame, frameSrcSize) != frameSrcSize) goto _output_error; } + DISPLAYLEVEL(3, "OK \n"); + + /* error string tests */ + DISPLAYLEVEL(3, "test%3i : testing ZSTD error code strings : ", testNb++); + if (strcmp("No error detected", ZSTD_getErrorName((ZSTD_ErrorCode)(0-ZSTD_error_no_error))) != 0) goto _output_error; + if (strcmp("No error detected", ZSTD_getErrorString(ZSTD_error_no_error)) != 0) goto _output_error; + if (strcmp("Unspecified error code", ZSTD_getErrorString((ZSTD_ErrorCode)(0-ZSTD_error_GENERIC))) != 0) goto _output_error; + if (strcmp("Error (generic)", ZSTD_getErrorName((size_t)0-ZSTD_error_GENERIC)) != 0) goto _output_error; + if (strcmp("Error (generic)", ZSTD_getErrorString(ZSTD_error_GENERIC)) != 0) goto _output_error; + if (strcmp("No error detected", ZSTD_getErrorName(ZSTD_error_GENERIC)) != 0) goto _output_error; + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : testing ZSTD dictionary sizes : ", testNb++); + RDG_genBuffer(CNBuffer, CNBuffSize, compressibility, 0., seed); + { + size_t const size = MIN(128 KB, CNBuffSize); + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_CDict* const lgCDict = ZSTD_createCDict(CNBuffer, size, 1); + ZSTD_CDict* const smCDict = ZSTD_createCDict(CNBuffer, 1 KB, 1); + ZSTD_frameHeader lgHeader; + ZSTD_frameHeader smHeader; + + CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, lgCDict)); + CHECK_Z(ZSTD_getFrameHeader(&lgHeader, compressedBuffer, compressedBufferSize)); + CHECK_Z(ZSTD_compress_usingCDict(cctx, compressedBuffer, compressedBufferSize, CNBuffer, size, smCDict)); + CHECK_Z(ZSTD_getFrameHeader(&smHeader, compressedBuffer, compressedBufferSize)); + + if (lgHeader.windowSize != smHeader.windowSize) goto _output_error; + + ZSTD_freeCDict(smCDict); + ZSTD_freeCDict(lgCDict); + ZSTD_freeCCtx(cctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : testing FSE_normalizeCount() PR#1255: ", testNb++); + { + short norm[32]; + unsigned count[32]; + unsigned const tableLog = 5; + size_t const nbSeq = 32; + unsigned const maxSymbolValue = 31; + size_t i; + + for (i = 0; i < 32; ++i) + count[i] = 1; + /* Calling FSE_normalizeCount() on a uniform distribution should not + * cause a division by zero. + */ + FSE_normalizeCount(norm, tableLog, count, nbSeq, maxSymbolValue); + } + DISPLAYLEVEL(3, "OK \n"); +#ifdef ZSTD_MULTITHREAD + DISPLAYLEVEL(3, "test%3i : passing wrong full dict should fail on compressStream2 refPrefix ", testNb++); + { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + /* A little more than ZSTDMT_JOBSIZE_MIN */ + size_t const srcSize = 1 MB + 5; + size_t const dstSize = ZSTD_compressBound(srcSize); + void* const src = CNBuffer; + void* const dst = compressedBuffer; + void* dict = (void*)malloc(srcSize); + + RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); + RDG_genBuffer(dict, srcSize, compressibility, 0., seed); + + /* Make sure there is no ZSTD_MAGIC_NUMBER */ + memset(dict, 0, sizeof(U32)); + + /* something more than 1 */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); + /* lie and claim this is a full dict */ + CHECK_Z(ZSTD_CCtx_refPrefix_advanced(cctx, dict, srcSize, ZSTD_dct_fullDict)); + + { + ZSTD_outBuffer out = {dst, dstSize, 0}; + ZSTD_inBuffer in = {src, srcSize, 0}; + + /* should fail because its not a full dict like we said it was */ + assert(ZSTD_isError(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush))); + } + + ZSTD_freeCCtx(cctx); + free(dict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : small dictionary with multithreading and LDM ", testNb++); + { + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + /* A little more than ZSTDMT_JOBSIZE_MIN */ + size_t const srcSize = 1 MB + 5; + size_t const dictSize = 10; + size_t const dstSize = ZSTD_compressBound(srcSize); + void* const src = CNBuffer; + void* const dst = compressedBuffer; + void* dict = (void*)malloc(dictSize); + + RDG_genBuffer(src, srcSize, compressibility, 0.5, seed); + RDG_genBuffer(dict, dictSize, compressibility, 0., seed); + + /* Make sure there is no ZSTD_MAGIC_NUMBER */ + memset(dict, 0, sizeof(U32)); + + /* Enable MT, LDM, and use refPrefix() for a small dict */ + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_nbWorkers, 2)); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1)); + CHECK_Z(ZSTD_CCtx_refPrefix(cctx, dict, dictSize)); + + CHECK_Z(ZSTD_compress2(cctx, dst, dstSize, src, srcSize)); + + ZSTD_freeCCtx(cctx); + free(dict); + } + DISPLAYLEVEL(3, "OK \n"); +#endif + + /* note : this test is rather long, it would be great to find a way to speed up its execution */ + DISPLAYLEVEL(3, "test%3i : table cleanliness through index reduction : ", testNb++); + { + int cLevel; + size_t approxIndex = 0; + size_t maxIndex = ((3U << 29) + (1U << ZSTD_WINDOWLOG_MAX)); /* ZSTD_CURRENT_MAX from zstd_compress_internal.h */ + + /* Provision enough space in a static context so that we can do all + * this without ever reallocating, which would reset the indices. */ + size_t const staticCCtxSize = ZSTD_estimateCStreamSize(22); + void* const staticCCtxBuffer = malloc(staticCCtxSize); + ZSTD_CCtx* const cctx = ZSTD_initStaticCCtx(staticCCtxBuffer, staticCCtxSize); + + /* bump the indices so the following compressions happen at high + * indices. */ + { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); + while (approxIndex <= (maxIndex / 4) * 3) { + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + approxIndex += in.pos; + CHECK(in.pos == in.size); + in.pos = 0; + out.pos = 0; + } + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + } + + /* spew a bunch of stuff into the table area */ + for (cLevel = 1; cLevel <= 22; cLevel++) { + ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; + ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + approxIndex += in.pos; + } + + /* now crank the indices so we overflow */ + { ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize, 0 }; + ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, -500)); + while (approxIndex <= maxIndex) { + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + approxIndex += in.pos; + CHECK(in.pos == in.size); + in.pos = 0; + out.pos = 0; + } + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + } + + /* do a bunch of compressions again in low indices and ensure we don't + * hit untracked invalid indices */ + for (cLevel = 1; cLevel <= 22; cLevel++) { + ZSTD_outBuffer out = { compressedBuffer, compressedBufferSize / (unsigned)cLevel, 0 }; + ZSTD_inBuffer in = { CNBuffer, CNBuffSize, 0 }; + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(cctx, ZSTD_c_compressionLevel, cLevel)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_flush)); + CHECK_Z(ZSTD_compressStream2(cctx, &out, &in, ZSTD_e_end)); + approxIndex += in.pos; + } + + free(staticCCtxBuffer); + } + DISPLAYLEVEL(3, "OK \n"); + +_end: + free(CNBuffer); + free(compressedBuffer); + free(decodedBuffer); + return testResult; + +_output_error: + testResult = 1; + DISPLAY("Error detected in Unit tests ! \n"); + goto _end; +} + + +static size_t findDiff(const void* buf1, const void* buf2, size_t max) +{ + const BYTE* b1 = (const BYTE*)buf1; + const BYTE* b2 = (const BYTE*)buf2; + size_t u; + for (u=0; u<max; u++) { + if (b1[u] != b2[u]) break; + } + return u; +} + + +static ZSTD_parameters FUZ_makeParams(ZSTD_compressionParameters cParams, ZSTD_frameParameters fParams) +{ + ZSTD_parameters params; + params.cParams = cParams; + params.fParams = fParams; + return params; +} + +static size_t FUZ_rLogLength(U32* seed, U32 logLength) +{ + size_t const lengthMask = ((size_t)1 << logLength) - 1; + return (lengthMask+1) + (FUZ_rand(seed) & lengthMask); +} + +static size_t FUZ_randomLength(U32* seed, U32 maxLog) +{ + U32 const logLength = FUZ_rand(seed) % maxLog; + return FUZ_rLogLength(seed, logLength); +} + +#undef CHECK +#define CHECK(cond, ...) { \ + if (cond) { \ + DISPLAY("Error => "); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ + goto _output_error; \ +} } + +#undef CHECK_Z +#define CHECK_Z(f) { \ + size_t const err = f; \ + if (ZSTD_isError(err)) { \ + DISPLAY("Error => %s : %s ", \ + #f, ZSTD_getErrorName(err)); \ + DISPLAY(" (seed %u, test nb %u) \n", (unsigned)seed, testNb); \ + goto _output_error; \ +} } + + +static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, U32 const maxDurationS, double compressibility, int bigTests) +{ + static const U32 maxSrcLog = 23; + static const U32 maxSampleLog = 22; + size_t const srcBufferSize = (size_t)1<<maxSrcLog; + size_t const dstBufferSize = (size_t)1<<maxSampleLog; + size_t const cBufferSize = ZSTD_compressBound(dstBufferSize); + BYTE* cNoiseBuffer[5]; + BYTE* const cBuffer = (BYTE*) malloc (cBufferSize); + BYTE* const dstBuffer = (BYTE*) malloc (dstBufferSize); + BYTE* const mirrorBuffer = (BYTE*) malloc (dstBufferSize); + ZSTD_CCtx* const refCtx = ZSTD_createCCtx(); + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + U32 result = 0; + unsigned testNb = 0; + U32 coreSeed = seed; + UTIL_time_t const startClock = UTIL_getTime(); + U64 const maxClockSpan = maxDurationS * SEC_TO_MICRO; + int const cLevelLimiter = bigTests ? 3 : 2; + + /* allocation */ + cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); + CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] + || !dstBuffer || !mirrorBuffer || !cBuffer || !refCtx || !ctx || !dctx, + "Not enough memory, fuzzer tests cancelled"); + + /* Create initial samples */ + RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ + RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ + RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); + RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ + RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ + + /* catch up testNb */ + for (testNb=1; testNb < startTest; testNb++) FUZ_rand(&coreSeed); + + /* main test loop */ + for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < maxClockSpan); testNb++ ) { + BYTE* srcBuffer; /* jumping pointer */ + U32 lseed; + size_t sampleSize, maxTestSize, totalTestSize; + size_t cSize, totalCSize, totalGenSize; + U64 crcOrig; + BYTE* sampleBuffer; + const BYTE* dict; + size_t dictSize; + + /* notification */ + if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } + else { DISPLAYUPDATE(2, "\r%6u ", testNb); } + + FUZ_rand(&coreSeed); + { U32 const prime1 = 2654435761U; lseed = coreSeed ^ prime1; } + + /* srcBuffer selection [0-4] */ + { U32 buffNb = FUZ_rand(&lseed) & 0x7F; + if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ + else { + buffNb >>= 3; + if (buffNb & 7) { + const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ + buffNb = tnb[buffNb >> 3]; + } else { + const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ + buffNb = tnb[buffNb >> 3]; + } } + srcBuffer = cNoiseBuffer[buffNb]; + } + + /* select src segment */ + sampleSize = FUZ_randomLength(&lseed, maxSampleLog); + + /* create sample buffer (to catch read error with valgrind & sanitizers) */ + sampleBuffer = (BYTE*)malloc(sampleSize); + CHECK(sampleBuffer==NULL, "not enough memory for sample buffer"); + { size_t const sampleStart = FUZ_rand(&lseed) % (srcBufferSize - sampleSize); + memcpy(sampleBuffer, srcBuffer + sampleStart, sampleSize); } + crcOrig = XXH64(sampleBuffer, sampleSize, 0); + + /* compression tests */ + { int const cLevelPositive = (int) + ( FUZ_rand(&lseed) % + ((U32)ZSTD_maxCLevel() - (FUZ_highbit32((U32)sampleSize) / (U32)cLevelLimiter)) ) + + 1; + int const cLevel = ((FUZ_rand(&lseed) & 15) == 3) ? + - (int)((FUZ_rand(&lseed) & 7) + 1) : /* test negative cLevel */ + cLevelPositive; + DISPLAYLEVEL(5, "fuzzer t%u: Simple compression test (level %i) \n", testNb, cLevel); + cSize = ZSTD_compressCCtx(ctx, cBuffer, cBufferSize, sampleBuffer, sampleSize, cLevel); + CHECK(ZSTD_isError(cSize), "ZSTD_compressCCtx failed : %s", ZSTD_getErrorName(cSize)); + + /* compression failure test : too small dest buffer */ + assert(cSize > 3); + { const size_t missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; + const size_t tooSmallSize = cSize - missing; + const unsigned endMark = 0x4DC2B1A9; + memcpy(dstBuffer+tooSmallSize, &endMark, sizeof(endMark)); + DISPLAYLEVEL(5, "fuzzer t%u: compress into too small buffer of size %u (missing %u bytes) \n", + testNb, (unsigned)tooSmallSize, (unsigned)missing); + { size_t const errorCode = ZSTD_compressCCtx(ctx, dstBuffer, tooSmallSize, sampleBuffer, sampleSize, cLevel); + CHECK(!ZSTD_isError(errorCode), "ZSTD_compressCCtx should have failed ! (buffer too small : %u < %u)", (unsigned)tooSmallSize, (unsigned)cSize); } + { unsigned endCheck; memcpy(&endCheck, dstBuffer+tooSmallSize, sizeof(endCheck)); + CHECK(endCheck != endMark, "ZSTD_compressCCtx : dst buffer overflow (check.%08X != %08X.mark)", endCheck, endMark); } + } } + + /* frame header decompression test */ + { ZSTD_frameHeader zfh; + CHECK_Z( ZSTD_getFrameHeader(&zfh, cBuffer, cSize) ); + CHECK(zfh.frameContentSize != sampleSize, "Frame content size incorrect"); + } + + /* Decompressed size test */ + { unsigned long long const rSize = ZSTD_findDecompressedSize(cBuffer, cSize); + CHECK(rSize != sampleSize, "decompressed size incorrect"); + } + + /* successful decompression test */ + DISPLAYLEVEL(5, "fuzzer t%u: simple decompression test \n", testNb); + { size_t const margin = (FUZ_rand(&lseed) & 1) ? 0 : (FUZ_rand(&lseed) & 31) + 1; + size_t const dSize = ZSTD_decompress(dstBuffer, sampleSize + margin, cBuffer, cSize); + CHECK(dSize != sampleSize, "ZSTD_decompress failed (%s) (srcSize : %u ; cSize : %u)", ZSTD_getErrorName(dSize), (unsigned)sampleSize, (unsigned)cSize); + { U64 const crcDest = XXH64(dstBuffer, sampleSize, 0); + CHECK(crcOrig != crcDest, "decompression result corrupted (pos %u / %u)", (unsigned)findDiff(sampleBuffer, dstBuffer, sampleSize), (unsigned)sampleSize); + } } + + free(sampleBuffer); /* no longer useful after this point */ + + /* truncated src decompression test */ + DISPLAYLEVEL(5, "fuzzer t%u: decompression of truncated source \n", testNb); + { size_t const missing = (FUZ_rand(&lseed) % (cSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ + size_t const tooSmallSize = cSize - missing; + void* cBufferTooSmall = malloc(tooSmallSize); /* valgrind will catch read overflows */ + CHECK(cBufferTooSmall == NULL, "not enough memory !"); + memcpy(cBufferTooSmall, cBuffer, tooSmallSize); + { size_t const errorCode = ZSTD_decompress(dstBuffer, dstBufferSize, cBufferTooSmall, tooSmallSize); + CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed ! (truncated src buffer)"); } + free(cBufferTooSmall); + } + + /* too small dst decompression test */ + DISPLAYLEVEL(5, "fuzzer t%u: decompress into too small dst buffer \n", testNb); + if (sampleSize > 3) { + size_t const missing = (FUZ_rand(&lseed) % (sampleSize-2)) + 1; /* no problem, as cSize > 4 (frameHeaderSizer) */ + size_t const tooSmallSize = sampleSize - missing; + static const BYTE token = 0xA9; + dstBuffer[tooSmallSize] = token; + { size_t const errorCode = ZSTD_decompress(dstBuffer, tooSmallSize, cBuffer, cSize); + CHECK(!ZSTD_isError(errorCode), "ZSTD_decompress should have failed : %u > %u (dst buffer too small)", (unsigned)errorCode, (unsigned)tooSmallSize); } + CHECK(dstBuffer[tooSmallSize] != token, "ZSTD_decompress : dst buffer overflow"); + } + + /* noisy src decompression test */ + if (cSize > 6) { + /* insert noise into src */ + { U32 const maxNbBits = FUZ_highbit32((U32)(cSize-4)); + size_t pos = 4; /* preserve magic number (too easy to detect) */ + for (;;) { + /* keep some original src */ + { U32 const nbBits = FUZ_rand(&lseed) % maxNbBits; + size_t const mask = (1<<nbBits) - 1; + size_t const skipLength = FUZ_rand(&lseed) & mask; + pos += skipLength; + } + if (pos >= cSize) break; + /* add noise */ + { U32 const nbBitsCodes = FUZ_rand(&lseed) % maxNbBits; + U32 const nbBits = nbBitsCodes ? nbBitsCodes-1 : 0; + size_t const mask = (1<<nbBits) - 1; + size_t const rNoiseLength = (FUZ_rand(&lseed) & mask) + 1; + size_t const noiseLength = MIN(rNoiseLength, cSize-pos); + size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseLength); + memcpy(cBuffer + pos, srcBuffer + noiseStart, noiseLength); + pos += noiseLength; + } } } + + /* decompress noisy source */ + DISPLAYLEVEL(5, "fuzzer t%u: decompress noisy source \n", testNb); + { U32 const endMark = 0xA9B1C3D6; + memcpy(dstBuffer+sampleSize, &endMark, 4); + { size_t const decompressResult = ZSTD_decompress(dstBuffer, sampleSize, cBuffer, cSize); + /* result *may* be an unlikely success, but even then, it must strictly respect dst buffer boundaries */ + CHECK((!ZSTD_isError(decompressResult)) && (decompressResult>sampleSize), + "ZSTD_decompress on noisy src : result is too large : %u > %u (dst buffer)", (unsigned)decompressResult, (unsigned)sampleSize); + } + { U32 endCheck; memcpy(&endCheck, dstBuffer+sampleSize, 4); + CHECK(endMark!=endCheck, "ZSTD_decompress on noisy src : dst buffer overflow"); + } } } /* noisy src decompression test */ + + /*===== Bufferless streaming compression test, scattered segments and dictionary =====*/ + DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming compression test \n", testNb); + { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; + int const cLevel = (FUZ_rand(&lseed) % + (ZSTD_maxCLevel() - + (MAX(testLog, dictLog) / cLevelLimiter))) + + 1; + maxTestSize = FUZ_rLogLength(&lseed, testLog); + if (maxTestSize >= dstBufferSize) maxTestSize = dstBufferSize-1; + + dictSize = FUZ_rLogLength(&lseed, dictLog); /* needed also for decompression */ + dict = srcBuffer + (FUZ_rand(&lseed) % (srcBufferSize - dictSize)); + + DISPLAYLEVEL(6, "fuzzer t%u: Compressing up to <=%u bytes at level %i with dictionary size %u \n", + testNb, (unsigned)maxTestSize, cLevel, (unsigned)dictSize); + + if (FUZ_rand(&lseed) & 0xF) { + CHECK_Z ( ZSTD_compressBegin_usingDict(refCtx, dict, dictSize, cLevel) ); + } else { + ZSTD_compressionParameters const cPar = ZSTD_getCParams(cLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize); + ZSTD_frameParameters const fPar = { FUZ_rand(&lseed)&1 /* contentSizeFlag */, + !(FUZ_rand(&lseed)&3) /* contentChecksumFlag*/, + 0 /*NodictID*/ }; /* note : since dictionary is fake, dictIDflag has no impact */ + ZSTD_parameters const p = FUZ_makeParams(cPar, fPar); + CHECK_Z ( ZSTD_compressBegin_advanced(refCtx, dict, dictSize, p, 0) ); + } + CHECK_Z( ZSTD_copyCCtx(ctx, refCtx, 0) ); + } + + { U32 const nbChunks = (FUZ_rand(&lseed) & 127) + 2; + U32 n; + XXH64_state_t xxhState; + XXH64_reset(&xxhState, 0); + for (totalTestSize=0, cSize=0, n=0 ; n<nbChunks ; n++) { + size_t const segmentSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const segmentStart = FUZ_rand(&lseed) % (srcBufferSize - segmentSize); + + if (cBufferSize-cSize < ZSTD_compressBound(segmentSize)) break; /* avoid invalid dstBufferTooSmall */ + if (totalTestSize+segmentSize > maxTestSize) break; + + { size_t const compressResult = ZSTD_compressContinue(ctx, cBuffer+cSize, cBufferSize-cSize, srcBuffer+segmentStart, segmentSize); + CHECK (ZSTD_isError(compressResult), "multi-segments compression error : %s", ZSTD_getErrorName(compressResult)); + cSize += compressResult; + } + XXH64_update(&xxhState, srcBuffer+segmentStart, segmentSize); + memcpy(mirrorBuffer + totalTestSize, srcBuffer+segmentStart, segmentSize); + totalTestSize += segmentSize; + } + + { size_t const flushResult = ZSTD_compressEnd(ctx, cBuffer+cSize, cBufferSize-cSize, NULL, 0); + CHECK (ZSTD_isError(flushResult), "multi-segments epilogue error : %s", ZSTD_getErrorName(flushResult)); + cSize += flushResult; + } + crcOrig = XXH64_digest(&xxhState); + } + + /* streaming decompression test */ + DISPLAYLEVEL(5, "fuzzer t%u: Bufferless streaming decompression test \n", testNb); + /* ensure memory requirement is good enough (should always be true) */ + { ZSTD_frameHeader zfh; + CHECK( ZSTD_getFrameHeader(&zfh, cBuffer, ZSTD_FRAMEHEADERSIZE_MAX), + "ZSTD_getFrameHeader(): error retrieving frame information"); + { size_t const roundBuffSize = ZSTD_decodingBufferSize_min(zfh.windowSize, zfh.frameContentSize); + CHECK_Z(roundBuffSize); + CHECK((roundBuffSize > totalTestSize) && (zfh.frameContentSize!=ZSTD_CONTENTSIZE_UNKNOWN), + "ZSTD_decodingBufferSize_min() requires more memory (%u) than necessary (%u)", + (unsigned)roundBuffSize, (unsigned)totalTestSize ); + } } + if (dictSize<8) dictSize=0, dict=NULL; /* disable dictionary */ + CHECK_Z( ZSTD_decompressBegin_usingDict(dctx, dict, dictSize) ); + totalCSize = 0; + totalGenSize = 0; + while (totalCSize < cSize) { + size_t const inSize = ZSTD_nextSrcSizeToDecompress(dctx); + size_t const genSize = ZSTD_decompressContinue(dctx, dstBuffer+totalGenSize, dstBufferSize-totalGenSize, cBuffer+totalCSize, inSize); + CHECK (ZSTD_isError(genSize), "ZSTD_decompressContinue error : %s", ZSTD_getErrorName(genSize)); + totalGenSize += genSize; + totalCSize += inSize; + } + CHECK (ZSTD_nextSrcSizeToDecompress(dctx) != 0, "frame not fully decoded"); + CHECK (totalGenSize != totalTestSize, "streaming decompressed data : wrong size") + CHECK (totalCSize != cSize, "compressed data should be fully read") + { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); + CHECK(crcOrig != crcDest, "streaming decompressed data corrupted (pos %u / %u)", + (unsigned)findDiff(mirrorBuffer, dstBuffer, totalTestSize), (unsigned)totalTestSize); + } + } /* for ( ; (testNb <= nbTests) */ + DISPLAY("\r%u fuzzer tests completed \n", testNb-1); + +_cleanup: + ZSTD_freeCCtx(refCtx); + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + free(cNoiseBuffer[0]); + free(cNoiseBuffer[1]); + free(cNoiseBuffer[2]); + free(cNoiseBuffer[3]); + free(cNoiseBuffer[4]); + free(cBuffer); + free(dstBuffer); + free(mirrorBuffer); + return result; + +_output_error: + result = 1; + goto _cleanup; +} + + +/*_******************************************************* +* Command line +*********************************************************/ +static int FUZ_usage(const char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -i# : Number of tests (default:%i)\n", nbTestsDefault); + DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n"); + DISPLAY( " -s# : Select seed (default:prompt user)\n"); + DISPLAY( " -t# : Select starting test number (default:0)\n"); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_compressibility_default); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -p : pause at the end\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + +/*! readU32FromChar() : + @return : unsigned integer value read from input in `char` format + allows and interprets K, KB, KiB, M, MB and MiB suffix. + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : function result can overflow if digit string > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + */ +static int longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + +int main(int argc, const char** argv) +{ + U32 seed = 0; + int seedset = 0; + int argNb; + int nbTests = nbTestsDefault; + int testNb = 0; + int proba = FUZ_compressibility_default; + int result = 0; + U32 mainPause = 0; + U32 maxDuration = 0; + int bigTests = 1; + U32 memTestsOnly = 0; + const char* const programName = argv[0]; + + /* Check command line */ + for (argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + if(!argument) continue; /* Protection if argument empty */ + + /* Handle commands. Aggregated commands are allowed */ + if (argument[0]=='-') { + + if (longCommandWArg(&argument, "--memtest=")) { memTestsOnly = readU32FromChar(&argument); continue; } + + if (!strcmp(argument, "--memtest")) { memTestsOnly=1; continue; } + if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } + + argument++; + while (*argument!=0) { + switch(*argument) + { + case 'h': + return FUZ_usage(programName); + + case 'v': + argument++; + g_displayLevel++; + break; + + case 'q': + argument++; + g_displayLevel--; + break; + + case 'p': /* pause at the end */ + argument++; + mainPause = 1; + break; + + case 'i': + argument++; maxDuration = 0; + nbTests = (int)readU32FromChar(&argument); + break; + + case 'T': + argument++; + nbTests = 0; + maxDuration = readU32FromChar(&argument); + if (*argument=='s') argument++; /* seconds */ + if (*argument=='m') maxDuration *= 60, argument++; /* minutes */ + if (*argument=='n') argument++; + break; + + case 's': + argument++; + seedset = 1; + seed = readU32FromChar(&argument); + break; + + case 't': + argument++; + testNb = (int)readU32FromChar(&argument); + break; + + case 'P': /* compressibility % */ + argument++; + proba = (int)readU32FromChar(&argument); + if (proba>100) proba = 100; + break; + + default: + return (FUZ_usage(programName), 1); + } } } } /* for (argNb=1; argNb<argc; argNb++) */ + + /* Get Seed */ + DISPLAY("Starting zstd tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); + + if (!seedset) { + time_t const t = time(NULL); + U32 const h = XXH32(&t, sizeof(t), 1); + seed = h % 10000; + } + + DISPLAY("Seed = %u\n", (unsigned)seed); + if (proba!=FUZ_compressibility_default) DISPLAY("Compressibility : %i%%\n", proba); + + if (memTestsOnly) { + g_displayLevel = MAX(3, g_displayLevel); + return FUZ_mallocTests(seed, ((double)proba) / 100, memTestsOnly); + } + + if (nbTests < testNb) nbTests = testNb; + + if (testNb==0) + result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */ + if (!result) + result = fuzzerTests(seed, nbTests, testNb, maxDuration, ((double)proba) / 100, bigTests); + if (mainPause) { + int unused; + DISPLAY("Press Enter \n"); + unused = getchar(); + (void)unused; + } + return result; +} diff --git a/src/zstd/tests/golden-compression/huffman-compressed-larger b/src/zstd/tests/golden-compression/huffman-compressed-larger Binary files differnew file mode 100644 index 000000000..f594f1ae9 --- /dev/null +++ b/src/zstd/tests/golden-compression/huffman-compressed-larger diff --git a/src/zstd/tests/golden-decompression/rle-first-block.zst b/src/zstd/tests/golden-decompression/rle-first-block.zst Binary files differnew file mode 100644 index 000000000..fd067edd7 --- /dev/null +++ b/src/zstd/tests/golden-decompression/rle-first-block.zst diff --git a/src/zstd/tests/gzip/Makefile b/src/zstd/tests/gzip/Makefile new file mode 100644 index 000000000..73f62f0eb --- /dev/null +++ b/src/zstd/tests/gzip/Makefile @@ -0,0 +1,45 @@ +# ################################################################ +# Copyright (c) 2017-2020, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +PRGDIR = ../../programs +VOID = /dev/null +export PATH := .:$(PATH) + +.PHONY: all +#all: test-gzip-env +all: test-helin-segv test-hufts test-keep test-list test-memcpy-abuse test-mixed +all: test-null-suffix-clobber test-stdin test-trailing-nul test-unpack-invalid +all: test-zdiff test-zgrep-context test-zgrep-f test-zgrep-signal test-znew-k test-z-suffix + @echo Testing completed + +.PHONY: zstd +zstd: + $(MAKE) -C $(PRGDIR) zstd + ln -sf $(PRGDIR)/zstd gzip + @echo PATH=$(PATH) + gzip --version + +.PHONY: clean +clean: + @$(MAKE) -C $(PRGDIR) $@ > $(VOID) + @$(RM) *.trs *.log + @echo Cleaning completed + + +#------------------------------------------------------------------------------ +# validated only for Linux, macOS, Hurd and some BSD targets +#------------------------------------------------------------------------------ +ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU FreeBSD DragonFly NetBSD)) + +test-%: zstd + @./test-driver.sh --test-name $* --log-file $*.log --trs-file $*.trs --expect-failure "no" --color-tests "yes" --enable-hard-errors "yes" ./$*.sh + # || echo ignoring error + +endif diff --git a/src/zstd/tests/gzip/gzip-env.sh b/src/zstd/tests/gzip/gzip-env.sh new file mode 100755 index 000000000..120e52d78 --- /dev/null +++ b/src/zstd/tests/gzip/gzip-env.sh @@ -0,0 +1,46 @@ +#!/bin/sh +# Test the obsolescent GZIP environment variable. + +# Copyright 2015-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +#echo PATH=$PATH +#gzip --version + +echo a >exp || framework_failure_ +gzip <exp >in || framework_failure_ + +fail=0 +GZIP=-qv gzip -d <in >out 2>err || fail=1 +compare exp out || fail=1 + +for badopt in -- -c --stdout -d --decompress -f --force -h --help -k --keep \ + -l --list -L --license -r --recursive -Sxxx --suffix=xxx '--suffix xxx' \ + -t --test -V --version +do + GZIP=$badopt gzip -d <in >out 2>err && fail=1 +done + +for goodopt in -n --no-name -N --name -q --quiet -v --verbose \ + -1 --fast -2 -3 -4 -5 -6 -7 -8 -9 --best +do + GZIP=$goodopt gzip -d <in >out 2>err || fail=1 + compare exp out || fail=1 +done + +Exit $fail diff --git a/src/zstd/tests/gzip/helin-segv.sh b/src/zstd/tests/gzip/helin-segv.sh new file mode 100644 index 000000000..f182c8066 --- /dev/null +++ b/src/zstd/tests/gzip/helin-segv.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Before gzip-1.4, gzip -d would segfault on some inputs. + +# Copyright (C) 2010-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +# This test case was provided by Aki Helin. +printf '\037\235\220\0\0\0\304' > helin.gz || framework_failure_ +printf '\0\0' > exp || framework_failure_ + +fail=0 + +gzip -dc helin.gz > out || fail=1 +compare exp out || fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/help-version.sh b/src/zstd/tests/gzip/help-version.sh new file mode 100644 index 000000000..ee0c19f7d --- /dev/null +++ b/src/zstd/tests/gzip/help-version.sh @@ -0,0 +1,270 @@ +#! /bin/sh +# Make sure all these programs work properly +# when invoked with --help or --version. + +# Copyright (C) 2000-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Ensure that $SHELL is set to *some* value and exported. +# This is required for dircolors, which would fail e.g., when +# invoked via debuild (which removes SHELL from the environment). +test "x$SHELL" = x && SHELL=/bin/sh +export SHELL + +. "${srcdir=.}/init.sh"; path_prepend_ . + +expected_failure_status_chroot=125 +expected_failure_status_env=125 +expected_failure_status_nice=125 +expected_failure_status_nohup=125 +expected_failure_status_stdbuf=125 +expected_failure_status_su=125 +expected_failure_status_timeout=125 +expected_failure_status_printenv=2 +expected_failure_status_tty=3 +expected_failure_status_sort=2 +expected_failure_status_expr=3 +expected_failure_status_lbracket=2 +expected_failure_status_dir=2 +expected_failure_status_ls=2 +expected_failure_status_vdir=2 + +expected_failure_status_cmp=2 +expected_failure_status_zcmp=2 +expected_failure_status_sdiff=2 +expected_failure_status_diff3=2 +expected_failure_status_diff=2 +expected_failure_status_zdiff=2 +expected_failure_status_zgrep=2 +expected_failure_status_zegrep=2 +expected_failure_status_zfgrep=2 + +expected_failure_status_grep=2 +expected_failure_status_egrep=2 +expected_failure_status_fgrep=2 + +test "$built_programs" \ + || fail_ "built_programs not specified!?!" + +test "$VERSION" \ + || fail_ "set envvar VERSION; it is required for a PATH sanity-check" + +# Extract version from --version output of the first program +for i in $built_programs; do + v=$(env $i --version | sed -n '1s/.* //p;q') + break +done + +# Ensure that it matches $VERSION. +test "x$v" = "x$VERSION" \ + || fail_ "--version-\$VERSION mismatch" + +for lang in C fr da; do + for i in $built_programs; do + + # Skip `test'; it doesn't accept --help or --version. + test $i = test && continue; + + # false fails even when invoked with --help or --version. + if test $i = false; then + env LC_MESSAGES=$lang $i --help >/dev/null && fail=1 + env LC_MESSAGES=$lang $i --version >/dev/null && fail=1 + continue + fi + + args= + + # The just-built install executable is always named `ginstall'. + test $i = install && i=ginstall + + # Make sure they exit successfully, under normal conditions. + eval "env \$i $args --help > h-\$i " || fail=1 + eval "env \$i $args --version >/dev/null" || fail=1 + + # Make sure they mention the bug-reporting address in --help output. + grep "$PACKAGE_BUGREPORT" h-$i > /dev/null || fail=1 + rm -f h-$i + + # Make sure they fail upon `disk full' error. + if test -w /dev/full && test -c /dev/full; then + eval "env \$i $args --help >/dev/full 2>/dev/null" && fail=1 + eval "env \$i $args --version >/dev/full 2>/dev/null" && fail=1 + status=$? + test $i = [ && prog=lbracket || prog=$i + eval "expected=\$expected_failure_status_$prog" + test x$expected = x && expected=1 + if test $status = $expected; then + : # ok + else + fail=1 + echo "*** $i: bad exit status \`$status' (expected $expected)," 1>&2 + echo " with --help or --version output redirected to /dev/full" 1>&2 + fi + fi + done +done + +bigZ_in=bigZ-in.Z +zin=zin.gz +zin2=zin2.gz + +tmp=tmp-$$ +tmp_in=in-$$ +tmp_in2=in2-$$ +tmp_dir=dir-$$ +tmp_out=out-$$ +mkdir $tmp || fail=1 +cd $tmp || fail=1 + +comm_setup () { args="$tmp_in $tmp_in"; } +csplit_setup () { args="$tmp_in //"; } +cut_setup () { args='-f 1'; } +join_setup () { args="$tmp_in $tmp_in"; } +tr_setup () { args='a a'; } + +chmod_setup () { args="a+x $tmp_in"; } +# Punt on these. +chgrp_setup () { args=--version; } +chown_setup () { args=--version; } +mkfifo_setup () { args=--version; } +mknod_setup () { args=--version; } +# Punt on uptime, since it fails (e.g., failing to get boot time) +# on some systems, and we shouldn't let that stop `make check'. +uptime_setup () { args=--version; } + +# Create a file in the current directory, not in $TMPDIR. +mktemp_setup () { args=mktemp.XXXX; } + +cmp_setup () { args="$tmp_in $tmp_in2"; } + +# Tell dd not to print the line with transfer rate and total. +# The transfer rate would vary between runs. +dd_setup () { args=status=noxfer; } + +zdiff_setup () { args="$args $zin $zin2"; } +zcmp_setup () { zdiff_setup; } +zcat_setup () { args="$args $zin"; } +gunzip_setup () { zcat_setup; } +zmore_setup () { zcat_setup; } +zless_setup () { zcat_setup; } +znew_setup () { args="$args $bigZ_in"; } +zforce_setup () { zcat_setup; } +zgrep_setup () { args="$args z $zin"; } +zegrep_setup () { zgrep_setup; } +zfgrep_setup () { zgrep_setup; } +gzexe_setup () { args="$args $tmp_in"; } + +# We know that $tmp_in contains a "0" +grep_setup () { args="0 $tmp_in"; } +egrep_setup () { args="0 $tmp_in"; } +fgrep_setup () { args="0 $tmp_in"; } + +diff_setup () { args="$tmp_in $tmp_in2"; } +sdiff_setup () { args="$tmp_in $tmp_in2"; } +diff3_setup () { args="$tmp_in $tmp_in2 $tmp_in2"; } +cp_setup () { args="$tmp_in $tmp_in2"; } +ln_setup () { args="$tmp_in ln-target"; } +ginstall_setup () { args="$tmp_in $tmp_in2"; } +mv_setup () { args="$tmp_in $tmp_in2"; } +mkdir_setup () { args=$tmp_dir/subdir; } +rmdir_setup () { args=$tmp_dir; } +rm_setup () { args=$tmp_in; } +shred_setup () { args=$tmp_in; } +touch_setup () { args=$tmp_in2; } +truncate_setup () { args="--reference=$tmp_in $tmp_in2"; } + +basename_setup () { args=$tmp_in; } +dirname_setup () { args=$tmp_in; } +expr_setup () { args=foo; } + +# Punt, in case GNU `id' hasn't been installed yet. +groups_setup () { args=--version; } + +pathchk_setup () { args=$tmp_in; } +yes_setup () { args=--version; } +logname_setup () { args=--version; } +nohup_setup () { args=--version; } +printf_setup () { args=foo; } +seq_setup () { args=10; } +sleep_setup () { args=0; } +su_setup () { args=--version; } +stdbuf_setup () { args="-oL true"; } +timeout_setup () { args=--version; } + +# I'd rather not run sync, since it spins up disks that I've +# deliberately caused to spin down (but not unmounted). +sync_setup () { args=--version; } + +test_setup () { args=foo; } + +# This is necessary in the unusual event that there is +# no valid entry in /etc/mtab. +df_setup () { args=/; } + +# This is necessary in the unusual event that getpwuid (getuid ()) fails. +id_setup () { args=-u; } + +# Use env to avoid invoking built-in sleep of Solaris 11's /bin/sh. +kill_setup () { + env sleep 10m & + args=$! +} + +link_setup () { args="$tmp_in link-target"; } +unlink_setup () { args=$tmp_in; } + +readlink_setup () { + ln -s . slink + args=slink; +} + +stat_setup () { args=$tmp_in; } +unlink_setup () { args=$tmp_in; } +lbracket_setup () { args=": ]"; } + +# Ensure that each program "works" (exits successfully) when doing +# something more than --help or --version. +for i in $built_programs; do + # Skip these. + case $i in chroot|stty|tty|false|chcon|runcon) continue;; esac + + rm -rf $tmp_in $tmp_in2 $tmp_dir $tmp_out $bigZ_in $zin $zin2 + echo z |gzip > $zin + cp $zin $zin2 + cp $zin $bigZ_in + + # This is sort of kludgey: use numbers so this is valid input for factor, + # and two tokens so it's valid input for tsort. + echo 2147483647 0 > $tmp_in + # Make $tmp_in2 identical. Then, using $tmp_in and $tmp_in2 as arguments + # to the likes of cmp and diff makes them exit successfully. + cp $tmp_in $tmp_in2 + mkdir $tmp_dir + # echo ================== $i + test $i = [ && prog=lbracket || prog=$i + args= + if type ${prog}_setup > /dev/null 2>&1; then + ${prog}_setup + fi + if eval "env \$i $args < \$tmp_in > \$tmp_out"; then + : # ok + else + echo FAIL: $i + fail=1 + fi + rm -rf $tmp_in $tmp_in2 $tmp_out $tmp_dir +done + +Exit $fail diff --git a/src/zstd/tests/gzip/hufts-segv.gz b/src/zstd/tests/gzip/hufts-segv.gz Binary files differnew file mode 100644 index 000000000..32cb2a256 --- /dev/null +++ b/src/zstd/tests/gzip/hufts-segv.gz diff --git a/src/zstd/tests/gzip/hufts.sh b/src/zstd/tests/gzip/hufts.sh new file mode 100644 index 000000000..9b9576ce3 --- /dev/null +++ b/src/zstd/tests/gzip/hufts.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Exercise a bug whereby an invalid input could make gzip -d misbehave. + +# Copyright (C) 2009-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +printf '\n...: invalid compressed data--format violated\n' > exp \ + || framework_failure_ + +fail=0 +gzip -dc "$abs_srcdir/hufts-segv.gz" > out 2> err +test $? = 1 || fail=1 + +compare /dev/null out || fail=1 + +sed 's/.*hufts-segv.gz: /...: /' err > k; mv k err || fail=1 +compare exp err || fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/init.cfg b/src/zstd/tests/gzip/init.cfg new file mode 100644 index 000000000..901209cea --- /dev/null +++ b/src/zstd/tests/gzip/init.cfg @@ -0,0 +1,5 @@ +# This file is sourced by init.sh, *before* its initialization. + +# This goes hand in hand with the "exec 9>&2;" in Makefile.am's +# TESTS_ENVIRONMENT definition. +stderr_fileno_=9 diff --git a/src/zstd/tests/gzip/init.sh b/src/zstd/tests/gzip/init.sh new file mode 100644 index 000000000..97e4e4ba5 --- /dev/null +++ b/src/zstd/tests/gzip/init.sh @@ -0,0 +1,616 @@ +# source this file; set up for tests + +# Copyright (C) 2009-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# Using this file in a test +# ========================= +# +# The typical skeleton of a test looks like this: +# +# #!/bin/sh +# . "${srcdir=.}/init.sh"; path_prepend_ . +# Execute some commands. +# Note that these commands are executed in a subdirectory, therefore you +# need to prepend "../" to relative filenames in the build directory. +# Note that the "path_prepend_ ." is useful only if the body of your +# test invokes programs residing in the initial directory. +# For example, if the programs you want to test are in src/, and this test +# script is named tests/test-1, then you would use "path_prepend_ ../src", +# or perhaps export PATH='$(abs_top_builddir)/src$(PATH_SEPARATOR)'"$$PATH" +# to all tests via automake's TESTS_ENVIRONMENT. +# Set the exit code 0 for success, 77 for skipped, or 1 or other for failure. +# Use the skip_ and fail_ functions to print a diagnostic and then exit +# with the corresponding exit code. +# Exit $? + +# Executing a test that uses this file +# ==================================== +# +# Running a single test: +# $ make check TESTS=test-foo.sh +# +# Running a single test, with verbose output: +# $ make check TESTS=test-foo.sh VERBOSE=yes +# +# Running a single test, with single-stepping: +# 1. Go into a sub-shell: +# $ bash +# 2. Set relevant environment variables from TESTS_ENVIRONMENT in the +# Makefile: +# $ export srcdir=../../tests # this is an example +# 3. Execute the commands from the test, copy&pasting them one by one: +# $ . "$srcdir/init.sh"; path_prepend_ . +# ... +# 4. Finally +# $ exit + +ME_=`expr "./$0" : '.*/\(.*\)$'` + +# We use a trap below for cleanup. This requires us to go through +# hoops to get the right exit status transported through the handler. +# So use 'Exit STATUS' instead of 'exit STATUS' inside of the tests. +# Turn off errexit here so that we don't trip the bug with OSF1/Tru64 +# sh inside this function. +Exit () { set +e; (exit $1); exit $1; } + +# Print warnings (e.g., about skipped and failed tests) to this file number. +# Override by defining to say, 9, in init.cfg, and putting say, +# export ...ENVVAR_SETTINGS...; $(SHELL) 9>&2 +# in the definition of TESTS_ENVIRONMENT in your tests/Makefile.am file. +# This is useful when using automake's parallel tests mode, to print +# the reason for skip/failure to console, rather than to the .log files. +: ${stderr_fileno_=2} + +# Note that correct expansion of "$*" depends on IFS starting with ' '. +# Always write the full diagnostic to stderr. +# When stderr_fileno_ is not 2, also emit the first line of the +# diagnostic to that file descriptor. +warn_ () +{ + # If IFS does not start with ' ', set it and emit the warning in a subshell. + case $IFS in + ' '*) printf '%s\n' "$*" >&2 + test $stderr_fileno_ = 2 \ + || { printf '%s\n' "$*" | sed 1q >&$stderr_fileno_ ; } ;; + *) (IFS=' '; warn_ "$@");; + esac +} +fail_ () { warn_ "$ME_: failed test: $@"; Exit 1; } +skip_ () { warn_ "$ME_: skipped test: $@"; Exit 77; } +fatal_ () { warn_ "$ME_: hard error: $@"; Exit 99; } +framework_failure_ () { warn_ "$ME_: set-up failure: $@"; Exit 99; } + +# This is used to simplify checking of the return value +# which is useful when ensuring a command fails as desired. +# I.e., just doing `command ... &&fail=1` will not catch +# a segfault in command for example. With this helper you +# instead check an explicit exit code like +# returns_ 1 command ... || fail +returns_ () { + # Disable tracing so it doesn't interfere with stderr of the wrapped command + { set +x; } 2>/dev/null + + local exp_exit="$1" + shift + "$@" + test $? -eq $exp_exit && ret_=0 || ret_=1 + + if test "$VERBOSE" = yes && test "$gl_set_x_corrupts_stderr_" = false; then + set -x + fi + { return $ret_; } 2>/dev/null +} + +# Sanitize this shell to POSIX mode, if possible. +DUALCASE=1; export DUALCASE +if test -n "${ZSH_VERSION+set}" && (emulate sh) >/dev/null 2>&1; then + emulate sh + NULLCMD=: + alias -g '${1+"$@"}'='"$@"' + setopt NO_GLOB_SUBST +else + case `(set -o) 2>/dev/null` in + *posix*) set -o posix ;; + esac +fi + +# We require $(...) support unconditionally. +# We require a few additional shell features only when $EXEEXT is nonempty, +# in order to support automatic $EXEEXT emulation: +# - hyphen-containing alias names +# - we prefer to use ${var#...} substitution, rather than having +# to work around lack of support for that feature. +# The following code attempts to find a shell with support for these features. +# If the current shell passes the test, we're done. Otherwise, test other +# shells until we find one that passes. If one is found, re-exec it. +# If no acceptable shell is found, skip the current test. +# +# The "...set -x; P=1 true 2>err..." test is to disqualify any shell that +# emits "P=1" into err, as /bin/sh from SunOS 5.11 and OpenBSD 4.7 do. +# +# Use "9" to indicate success (rather than 0), in case some shell acts +# like Solaris 10's /bin/sh but exits successfully instead of with status 2. + +# Eval this code in a subshell to determine a shell's suitability. +# 10 - passes all tests; ok to use +# 9 - ok, but enabling "set -x" corrupts app stderr; prefer higher score +# ? - not ok +gl_shell_test_script_=' +test $(echo y) = y || exit 1 +f_local_() { local v=1; }; f_local_ || exit 1 +score_=10 +if test "$VERBOSE" = yes; then + test -n "$( (exec 3>&1; set -x; P=1 true 2>&3) 2> /dev/null)" && score_=9 +fi +test -z "$EXEEXT" && exit $score_ +shopt -s expand_aliases +alias a-b="echo zoo" +v=abx + test ${v%x} = ab \ + && test ${v#a} = bx \ + && test $(a-b) = zoo \ + && exit $score_ +' + +if test "x$1" = "x--no-reexec"; then + shift +else + # Assume a working shell. Export to subshells (setup_ needs this). + gl_set_x_corrupts_stderr_=false + export gl_set_x_corrupts_stderr_ + + # Record the first marginally acceptable shell. + marginal_= + + # Search for a shell that meets our requirements. + for re_shell_ in __current__ "${CONFIG_SHELL:-no_shell}" \ + /bin/sh bash dash zsh pdksh fail + do + test "$re_shell_" = no_shell && continue + + # If we've made it all the way to the sentinel, "fail" without + # finding even a marginal shell, skip this test. + if test "$re_shell_" = fail; then + test -z "$marginal_" && skip_ failed to find an adequate shell + re_shell_=$marginal_ + break + fi + + # When testing the current shell, simply "eval" the test code. + # Otherwise, run it via $re_shell_ -c ... + if test "$re_shell_" = __current__; then + # 'eval'ing this code makes Solaris 10's /bin/sh exit with + # $? set to 2. It does not evaluate any of the code after the + # "unexpected" first '('. Thus, we must run it in a subshell. + ( eval "$gl_shell_test_script_" ) > /dev/null 2>&1 + else + "$re_shell_" -c "$gl_shell_test_script_" 2>/dev/null + fi + + st_=$? + + # $re_shell_ works just fine. Use it. + if test $st_ = 10; then + gl_set_x_corrupts_stderr_=false + break + fi + + # If this is our first marginally acceptable shell, remember it. + if test "$st_:$marginal_" = 9: ; then + marginal_="$re_shell_" + gl_set_x_corrupts_stderr_=true + fi + done + + if test "$re_shell_" != __current__; then + # Found a usable shell. Preserve -v and -x. + case $- in + *v*x* | *x*v*) opts_=-vx ;; + *v*) opts_=-v ;; + *x*) opts_=-x ;; + *) opts_= ;; + esac + re_shell=$re_shell_ + export re_shell + exec "$re_shell_" $opts_ "$0" --no-reexec "$@" + echo "$ME_: exec failed" 1>&2 + exit 127 + fi +fi + +# If this is bash, turn off all aliases. +test -n "$BASH_VERSION" && unalias -a + +# Note that when supporting $EXEEXT (transparently mapping from PROG_NAME to +# PROG_NAME.exe), we want to support hyphen-containing names like test-acos. +# That is part of the shell-selection test above. Why use aliases rather +# than functions? Because support for hyphen-containing aliases is more +# widespread than that for hyphen-containing function names. +test -n "$EXEEXT" && shopt -s expand_aliases + +# Enable glibc's malloc-perturbing option. +# This is useful for exposing code that depends on the fact that +# malloc-related functions often return memory that is mostly zeroed. +# If you have the time and cycles, use valgrind to do an even better job. +: ${MALLOC_PERTURB_=87} +export MALLOC_PERTURB_ + +# This is a stub function that is run upon trap (upon regular exit and +# interrupt). Override it with a per-test function, e.g., to unmount +# a partition, or to undo any other global state changes. +cleanup_ () { :; } + +# Emit a header similar to that from diff -u; Print the simulated "diff" +# command so that the order of arguments is clear. Don't bother with @@ lines. +emit_diff_u_header_ () +{ + printf '%s\n' "diff -u $*" \ + "--- $1 1970-01-01" \ + "+++ $2 1970-01-01" +} + +# Arrange not to let diff or cmp operate on /dev/null, +# since on some systems (at least OSF/1 5.1), that doesn't work. +# When there are not two arguments, or no argument is /dev/null, return 2. +# When one argument is /dev/null and the other is not empty, +# cat the nonempty file to stderr and return 1. +# Otherwise, return 0. +compare_dev_null_ () +{ + test $# = 2 || return 2 + + if test "x$1" = x/dev/null; then + test -s "$2" || return 0 + emit_diff_u_header_ "$@"; sed 's/^/+/' "$2" + return 1 + fi + + if test "x$2" = x/dev/null; then + test -s "$1" || return 0 + emit_diff_u_header_ "$@"; sed 's/^/-/' "$1" + return 1 + fi + + return 2 +} + +if diff_out_=`exec 2>/dev/null; diff -u "$0" "$0" < /dev/null` \ + && diff -u Makefile "$0" 2>/dev/null | grep '^[+]#!' >/dev/null; then + # diff accepts the -u option and does not (like AIX 7 'diff') produce an + # extra space on column 1 of every content line. + if test -z "$diff_out_"; then + compare_ () { diff -u "$@"; } + else + compare_ () + { + if diff -u "$@" > diff.out; then + # No differences were found, but Solaris 'diff' produces output + # "No differences encountered". Hide this output. + rm -f diff.out + true + else + cat diff.out + rm -f diff.out + false + fi + } + fi +elif + for diff_opt_ in -U3 -c '' no; do + test "$diff_opt_" = no && break + diff_out_=`exec 2>/dev/null; diff $diff_opt_ "$0" "$0" </dev/null` && break + done + test "$diff_opt_" != no +then + if test -z "$diff_out_"; then + compare_ () { diff $diff_opt_ "$@"; } + else + compare_ () + { + if diff $diff_opt_ "$@" > diff.out; then + # No differences were found, but AIX and HP-UX 'diff' produce output + # "No differences encountered" or "There are no differences between the + # files.". Hide this output. + rm -f diff.out + true + else + cat diff.out + rm -f diff.out + false + fi + } + fi +elif cmp -s /dev/null /dev/null 2>/dev/null; then + compare_ () { cmp -s "$@"; } +else + compare_ () { cmp "$@"; } +fi + +# Usage: compare EXPECTED ACTUAL +# +# Given compare_dev_null_'s preprocessing, defer to compare_ if 2 or more. +# Otherwise, propagate $? to caller: any diffs have already been printed. +compare () +{ + # This looks like it can be factored to use a simple "case $?" + # after unchecked compare_dev_null_ invocation, but that would + # fail in a "set -e" environment. + if compare_dev_null_ "$@"; then + return 0 + else + case $? in + 1) return 1;; + *) compare_ "$@";; + esac + fi +} + +# An arbitrary prefix to help distinguish test directories. +testdir_prefix_ () { printf gt; } + +# Run the user-overridable cleanup_ function, remove the temporary +# directory and exit with the incoming value of $?. +remove_tmp_ () +{ + __st=$? + cleanup_ + # cd out of the directory we're about to remove + cd "$initial_cwd_" || cd / || cd /tmp + chmod -R u+rwx "$test_dir_" + # If removal fails and exit status was to be 0, then change it to 1. + rm -rf "$test_dir_" || { test $__st = 0 && __st=1; } + exit $__st +} + +# Given a directory name, DIR, if every entry in it that matches *.exe +# contains only the specified bytes (see the case stmt below), then print +# a space-separated list of those names and return 0. Otherwise, don't +# print anything and return 1. Naming constraints apply also to DIR. +find_exe_basenames_ () +{ + feb_dir_=$1 + feb_fail_=0 + feb_result_= + feb_sp_= + for feb_file_ in $feb_dir_/*.exe; do + # If there was no *.exe file, or there existed a file named "*.exe" that + # was deleted between the above glob expansion and the existence test + # below, just skip it. + test "x$feb_file_" = "x$feb_dir_/*.exe" && test ! -f "$feb_file_" \ + && continue + # Exempt [.exe, since we can't create a function by that name, yet + # we can't invoke [ by PATH search anyways due to shell builtins. + test "x$feb_file_" = "x$feb_dir_/[.exe" && continue + case $feb_file_ in + *[!-a-zA-Z/0-9_.+]*) feb_fail_=1; break;; + *) # Remove leading file name components as well as the .exe suffix. + feb_file_=${feb_file_##*/} + feb_file_=${feb_file_%.exe} + feb_result_="$feb_result_$feb_sp_$feb_file_";; + esac + feb_sp_=' ' + done + test $feb_fail_ = 0 && printf %s "$feb_result_" + return $feb_fail_ +} + +# Consider the files in directory, $1. +# For each file name of the form PROG.exe, create an alias named +# PROG that simply invokes PROG.exe, then return 0. If any selected +# file name or the directory name, $1, contains an unexpected character, +# define no alias and return 1. +create_exe_shims_ () +{ + case $EXEEXT in + '') return 0 ;; + .exe) ;; + *) echo "$0: unexpected \$EXEEXT value: $EXEEXT" 1>&2; return 1 ;; + esac + + base_names_=`find_exe_basenames_ $1` \ + || { echo "$0 (exe_shim): skipping directory: $1" 1>&2; return 0; } + + if test -n "$base_names_"; then + for base_ in $base_names_; do + alias "$base_"="$base_$EXEEXT" + done + fi + + return 0 +} + +# Use this function to prepend to PATH an absolute name for each +# specified, possibly-$initial_cwd_-relative, directory. +path_prepend_ () +{ + while test $# != 0; do + path_dir_=$1 + case $path_dir_ in + '') fail_ "invalid path dir: '$1'";; + /*) abs_path_dir_=$path_dir_;; + *) abs_path_dir_=$initial_cwd_/$path_dir_;; + esac + case $abs_path_dir_ in + *:*) fail_ "invalid path dir: '$abs_path_dir_'";; + esac + PATH="$abs_path_dir_:$PATH" + + # Create an alias, FOO, for each FOO.exe in this directory. + create_exe_shims_ "$abs_path_dir_" \ + || fail_ "something failed (above): $abs_path_dir_" + shift + done + export PATH +} + +setup_ () +{ + if test "$VERBOSE" = yes; then + # Test whether set -x may cause the selected shell to corrupt an + # application's stderr. Many do, including zsh-4.3.10 and the /bin/sh + # from SunOS 5.11, OpenBSD 4.7 and Irix 5.x and 6.5. + # If enabling verbose output this way would cause trouble, simply + # issue a warning and refrain. + if $gl_set_x_corrupts_stderr_; then + warn_ "using SHELL=$SHELL with 'set -x' corrupts stderr" + else + set -x + fi + fi + + initial_cwd_=$PWD + + pfx_=`testdir_prefix_` + test_dir_=`mktempd_ "$initial_cwd_" "$pfx_-$ME_.XXXX"` \ + || fail_ "failed to create temporary directory in $initial_cwd_" + cd "$test_dir_" || fail_ "failed to cd to temporary directory" + + # As autoconf-generated configure scripts do, ensure that IFS + # is defined initially, so that saving and restoring $IFS works. + gl_init_sh_nl_=' +' + IFS=" "" $gl_init_sh_nl_" + + # This trap statement, along with a trap on 0 below, ensure that the + # temporary directory, $test_dir_, is removed upon exit as well as + # upon receipt of any of the listed signals. + for sig_ in 1 2 3 13 15; do + eval "trap 'Exit $(expr $sig_ + 128)' $sig_" + done +} + +# Create a temporary directory, much like mktemp -d does. +# Written by Jim Meyering. +# +# Usage: mktempd_ /tmp phoey.XXXXXXXXXX +# +# First, try to use the mktemp program. +# Failing that, we'll roll our own mktemp-like function: +# - try to get random bytes from /dev/urandom +# - failing that, generate output from a combination of quickly-varying +# sources and gzip. Ignore non-varying gzip header, and extract +# "random" bits from there. +# - given those bits, map to file-name bytes using tr, and try to create +# the desired directory. +# - make only $MAX_TRIES_ attempts + +# Helper function. Print $N pseudo-random bytes from a-zA-Z0-9. +rand_bytes_ () +{ + n_=$1 + + # Maybe try openssl rand -base64 $n_prime_|tr '+/=\012' abcd first? + # But if they have openssl, they probably have mktemp, too. + + chars_=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 + dev_rand_=/dev/urandom + if test -r "$dev_rand_"; then + # Note: 256-length($chars_) == 194; 3 copies of $chars_ is 186 + 8 = 194. + dd ibs=$n_ count=1 if=$dev_rand_ 2>/dev/null \ + | LC_ALL=C tr -c $chars_ 01234567$chars_$chars_$chars_ + return + fi + + n_plus_50_=`expr $n_ + 50` + cmds_='date; date +%N; free; who -a; w; ps auxww; ps ef; netstat -n' + data_=` (eval "$cmds_") 2>&1 | gzip ` + + # Ensure that $data_ has length at least 50+$n_ + while :; do + len_=`echo "$data_"|wc -c` + test $n_plus_50_ -le $len_ && break; + data_=` (echo "$data_"; eval "$cmds_") 2>&1 | gzip ` + done + + echo "$data_" \ + | dd bs=1 skip=50 count=$n_ 2>/dev/null \ + | LC_ALL=C tr -c $chars_ 01234567$chars_$chars_$chars_ +} + +mktempd_ () +{ + case $# in + 2);; + *) fail_ "Usage: mktempd_ DIR TEMPLATE";; + esac + + destdir_=$1 + template_=$2 + + MAX_TRIES_=4 + + # Disallow any trailing slash on specified destdir: + # it would subvert the post-mktemp "case"-based destdir test. + case $destdir_ in + / | //) destdir_slash_=$destdir;; + */) fail_ "invalid destination dir: remove trailing slash(es)";; + *) destdir_slash_=$destdir_/;; + esac + + case $template_ in + *XXXX) ;; + *) fail_ \ + "invalid template: $template_ (must have a suffix of at least 4 X's)";; + esac + + # First, try to use mktemp. + d=`unset TMPDIR; { mktemp -d -t -p "$destdir_" "$template_"; } 2>/dev/null` && + + # The resulting name must be in the specified directory. + case $d in "$destdir_slash_"*) :;; *) false;; esac && + + # It must have created the directory. + test -d "$d" && + + # It must have 0700 permissions. Handle sticky "S" bits. + perms=`ls -dgo "$d" 2>/dev/null` && + case $perms in drwx--[-S]---*) :;; *) false;; esac && { + echo "$d" + return + } + + # If we reach this point, we'll have to create a directory manually. + + # Get a copy of the template without its suffix of X's. + base_template_=`echo "$template_"|sed 's/XX*$//'` + + # Calculate how many X's we've just removed. + template_length_=`echo "$template_" | wc -c` + nx_=`echo "$base_template_" | wc -c` + nx_=`expr $template_length_ - $nx_` + + err_= + i_=1 + while :; do + X_=`rand_bytes_ $nx_` + candidate_dir_="$destdir_slash_$base_template_$X_" + err_=`mkdir -m 0700 "$candidate_dir_" 2>&1` \ + && { echo "$candidate_dir_"; return; } + test $MAX_TRIES_ -le $i_ && break; + i_=`expr $i_ + 1` + done + fail_ "$err_" +} + +# If you want to override the testdir_prefix_ function, +# or to add more utility functions, use this file. +test -f "$srcdir/init.cfg" \ + && . "$srcdir/init.cfg" + +setup_ "$@" +# This trap is here, rather than in the setup_ function, because some +# shells run the exit trap at shell function exit, rather than script exit. +trap remove_tmp_ 0 diff --git a/src/zstd/tests/gzip/keep.sh b/src/zstd/tests/gzip/keep.sh new file mode 100644 index 000000000..ab9a21811 --- /dev/null +++ b/src/zstd/tests/gzip/keep.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# Exercise the --keep option. + +# Copyright (C) 2013-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +echo fooooooooo > in || framework_failure_ +cp in orig || framework_failure_ + +fail=0 + +# Compress and decompress both with and without --keep. +for k in --keep ''; do + # With --keep, the source must be retained, otherwise, it must be removed. + case $k in --keep) op='||' ;; *) op='&&' ;; esac + + gzip $k in || fail=1 + eval "test -f in $op fail=1" + test -f in.gz || fail=1 + rm -f in || fail=1 + + gzip -d $k in.gz || fail=1 + eval "test -f in.gz $op fail=1" + test -f in || fail=1 + compare in orig || fail=1 + rm -f in.gz || fail=1 +done + +cp orig in || framework_failure_ +log=$(gzip -kv in 2>&1) || fail=1 +case $log in + *'created in.gz'*) ;; + *) fail=1;; +esac + +Exit $fail diff --git a/src/zstd/tests/gzip/list.sh b/src/zstd/tests/gzip/list.sh new file mode 100644 index 000000000..75912e1e2 --- /dev/null +++ b/src/zstd/tests/gzip/list.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Exercise the --list option. + +# Copyright 2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +echo zoology zucchini > in || framework_failure_ +cp in orig || framework_failure_ + +gzip -l in && fail=1 +gzip -9 in || fail=1 +gzip -l in.gz >out1 || fail=1 +gzip -l in.gz | cat >out2 || fail=1 +compare out1 out2 || fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/memcpy-abuse.sh b/src/zstd/tests/gzip/memcpy-abuse.sh new file mode 100644 index 000000000..7d5c056de --- /dev/null +++ b/src/zstd/tests/gzip/memcpy-abuse.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Before gzip-1.4, this the use of memcpy in inflate_codes could +# mistakenly operate on overlapping regions. Exercise that code. + +# Copyright (C) 2010-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +# The input must be larger than 32KiB and slightly +# less uniform than e.g., all zeros. +printf wxy%032767d 0 | tee in | gzip > in.gz || framework_failure_ + +fail=0 + +# Before the fix, this would call memcpy with overlapping regions. +gzip -dc in.gz > out || fail=1 + +compare in out || fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/mixed.sh b/src/zstd/tests/gzip/mixed.sh new file mode 100644 index 000000000..383a54f5e --- /dev/null +++ b/src/zstd/tests/gzip/mixed.sh @@ -0,0 +1,68 @@ +#!/bin/sh +# Ensure that gzip -cdf handles mixed compressed/not-compressed data +# Before gzip-1.5, it would produce invalid output. + +# Copyright (C) 2010-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +printf 'xxx\nyyy\n' > exp2 || framework_failure_ +printf 'aaa\nbbb\nccc\n' > exp3 || framework_failure_ + +fail=0 + +(echo xxx; echo yyy) > in || fail=1 +gzip -cdf < in > out || fail=1 +compare exp2 out || fail=1 + +# Uncompressed input, followed by compressed data. +# Currently fails, so skip it. +# (echo xxx; echo yyy|gzip) > in || fail=1 +# gzip -cdf < in > out || fail=1 +# compare exp2 out || fail=1 + +# Compressed input, followed by regular (not-compressed) data. +(echo xxx|gzip; echo yyy) > in || fail=1 +gzip -cdf < in > out || fail=1 +compare exp2 out || fail=1 + +(echo xxx|gzip; echo yyy|gzip) > in || fail=1 +gzip -cdf < in > out || fail=1 +compare exp2 out || fail=1 + +in_str=0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-+=% +for i in 0 1 2 3 4 5 6 7 8 9 a; do in_str="$in_str$in_str" ;done + +# Start with some small sizes. $(seq 64) +sizes=$(i=0; while :; do echo $i; test $i = 64 && break; i=$(expr $i + 1); done) + +# gzip's internal buffer size is 32KiB + 64 bytes: +sizes="$sizes 32831 32832 32833" + +# 128KiB, +/- 1 +sizes="$sizes 131071 131072 131073" + +# Ensure that "gzip -cdf" acts like cat, for a range of small input files. +i=0 +for i in $sizes; do + echo $i + printf %$i.${i}s $in_str > in + gzip -cdf < in > out + compare in out || fail=1 +done + +Exit $fail diff --git a/src/zstd/tests/gzip/null-suffix-clobber.sh b/src/zstd/tests/gzip/null-suffix-clobber.sh new file mode 100644 index 000000000..0efd0e344 --- /dev/null +++ b/src/zstd/tests/gzip/null-suffix-clobber.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Before gzip-1.5, gzip -d -S '' k.gz would delete F.gz and not create "F" + +# Copyright (C) 2010-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +printf anything | gzip > F.gz || framework_failure_ +echo y > yes || framework_failure_ +echo "gzip: invalid suffix ''" > expected-err || framework_failure_ + +fail=0 + +gzip ---presume-input-tty -d -S '' F.gz < yes > out 2>err && fail=1 + +compare /dev/null out || fail=1 +compare expected-err err || fail=1 + +test -f F.gz || fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/stdin.sh b/src/zstd/tests/gzip/stdin.sh new file mode 100644 index 000000000..eef4cd8b1 --- /dev/null +++ b/src/zstd/tests/gzip/stdin.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Ensure that gzip interprets "-" as stdin. + +# Copyright (C) 2009-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +printf a | gzip > in || framework_failure_ +printf aaa > exp || framework_failure_ + +fail=0 +gzip -dc in - in < in > out 2>err || fail=1 + +compare exp out || fail=1 +compare /dev/null err || fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/test-driver.sh b/src/zstd/tests/gzip/test-driver.sh new file mode 100644 index 000000000..649c084e4 --- /dev/null +++ b/src/zstd/tests/gzip/test-driver.sh @@ -0,0 +1,150 @@ +#! /bin/sh +# test-driver - basic testsuite driver script. + +scriptversion=2016-01-11.22; # UTC + +# Copyright (C) 2011-2015 Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# This file is maintained in Automake, please report +# bugs to <bug-automake@gnu.org> or send patches to +# <automake-patches@gnu.org>. + +# Make unconditional expansion of undefined variables an error. This +# helps a lot in preventing typo-related bugs. +set -u + +usage_error () +{ + echo "$0: $*" >&2 + print_usage >&2 + exit 2 +} + +print_usage () +{ + cat <<END +Usage: + test-driver --test-name=NAME --log-file=PATH --trs-file=PATH + [--expect-failure={yes|no}] [--color-tests={yes|no}] + [--enable-hard-errors={yes|no}] [--] + TEST-SCRIPT [TEST-SCRIPT-ARGUMENTS] +The '--test-name', '--log-file' and '--trs-file' options are mandatory. +END +} + +test_name= # Used for reporting. +log_file= # Where to save the output of the test script. +trs_file= # Where to save the metadata of the test run. +expect_failure=no +color_tests=no +enable_hard_errors=yes +while test $# -gt 0; do + case $1 in + --help) print_usage; exit $?;; + --version) echo "test-driver $scriptversion"; exit $?;; + --test-name) test_name=$2; shift;; + --log-file) log_file=$2; shift;; + --trs-file) trs_file=$2; shift;; + --color-tests) color_tests=$2; shift;; + --expect-failure) expect_failure=$2; shift;; + --enable-hard-errors) enable_hard_errors=$2; shift;; + --) shift; break;; + -*) usage_error "invalid option: '$1'";; + *) break;; + esac + shift +done + +missing_opts= +test x"$test_name" = x && missing_opts="$missing_opts --test-name" +test x"$log_file" = x && missing_opts="$missing_opts --log-file" +test x"$trs_file" = x && missing_opts="$missing_opts --trs-file" +if test x"$missing_opts" != x; then + usage_error "the following mandatory options are missing:$missing_opts" +fi + +if test $# -eq 0; then + usage_error "missing argument" +fi + +if test $color_tests = yes; then + # Keep this in sync with 'lib/am/check.am:$(am__tty_colors)'. + red='[0;31m' # Red. + grn='[0;32m' # Green. + lgn='[1;32m' # Light green. + blu='[1;34m' # Blue. + mgn='[0;35m' # Magenta. + std='[m' # No color. +else + red= grn= lgn= blu= mgn= std= +fi + +do_exit='rm -f $log_file $trs_file; (exit $st); exit $st' +trap "st=129; $do_exit" 1 +trap "st=130; $do_exit" 2 +trap "st=141; $do_exit" 13 +trap "st=143; $do_exit" 15 + +# Test script is run here. +"$@" >$log_file 2>&1 +estatus=$? + +if test $enable_hard_errors = no && test $estatus -eq 99; then + tweaked_estatus=1 +else + tweaked_estatus=$estatus +fi + +case $tweaked_estatus:$expect_failure in + 0:yes) col=$red res=XPASS recheck=yes gcopy=yes;; + 0:*) col=$grn res=PASS recheck=no gcopy=no;; + 77:*) col=$blu res=SKIP recheck=no gcopy=yes;; + 99:*) col=$mgn res=ERROR recheck=yes gcopy=yes;; + *:yes) col=$lgn res=XFAIL recheck=no gcopy=yes;; + *:*) col=$red res=FAIL recheck=yes gcopy=yes;; +esac + +# Report the test outcome and exit status in the logs, so that one can +# know whether the test passed or failed simply by looking at the '.log' +# file, without the need of also peaking into the corresponding '.trs' +# file (automake bug#11814). +echo "$res $test_name (exit status: $estatus)" >>$log_file + +# Report outcome to console. +echo "${col}${res}${std}: $test_name" + +# Register the test result, and other relevant metadata. +echo ":test-result: $res" > $trs_file +echo ":global-test-result: $res" >> $trs_file +echo ":recheck: $recheck" >> $trs_file +echo ":copy-in-global-log: $gcopy" >> $trs_file + +# Local Variables: +# mode: shell-script +# sh-indentation: 2 +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC0" +# time-stamp-end: "; # UTC" +# End: + +exit $tweaked_estatus diff --git a/src/zstd/tests/gzip/trailing-nul.sh b/src/zstd/tests/gzip/trailing-nul.sh new file mode 100644 index 000000000..7b15d5e55 --- /dev/null +++ b/src/zstd/tests/gzip/trailing-nul.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# gzip accepts trailing NUL bytes; don't fail if there is exactly one. +# Before gzip-1.4, this would fail. + +# Copyright (C) 2009-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +(echo 0 | gzip; printf '\0') > 0.gz || framework_failure_ +(echo 00 | gzip; printf '\0\0') > 00.gz || framework_failure_ +(echo 1 | gzip; printf '\1') > 1.gz || framework_failure_ + +fail=0 + +for i in 0 00 1; do + gzip -d $i.gz; ret=$? + test $ret -eq $i || fail=1 + test $ret = 1 && continue + echo $i > exp || fail=1 + compare exp $i || fail=1 +done + +Exit $fail diff --git a/src/zstd/tests/gzip/unpack-invalid.sh b/src/zstd/tests/gzip/unpack-invalid.sh new file mode 100644 index 000000000..fe8384d73 --- /dev/null +++ b/src/zstd/tests/gzip/unpack-invalid.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# gzip should report invalid 'unpack' input when uncompressing. +# With gzip-1.5, it would output invalid data instead. + +# Copyright (C) 2012-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +for input in \ + '\037\036\000\000\037\213\010\000\000\000\000\000\002\003\036\000\000\000\002\003\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\037\000\302\240\037\213\010\000\000\000\000\000\002\003\355\301' \ + '\037\213\010\000\000\000\000\000\002\003\355\301\001\015\000\000\000\302\240\076\366\017\370\036\016\030\000\000\000\000\000\000\000\000\000\034\010\105\140\104\025\020\047\000\000\037\036\016\030\000\000\000'; do + + printf "$input" >in || framework_failure_ + + if gzip -d <in >out 2>err; then + fail=1 + else + fail=0 + fi +done + +Exit $fail diff --git a/src/zstd/tests/gzip/z-suffix.sh b/src/zstd/tests/gzip/z-suffix.sh new file mode 100644 index 000000000..a870a5408 --- /dev/null +++ b/src/zstd/tests/gzip/z-suffix.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Check that -Sz works. + +# Copyright 2014-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +printf anything > F && cp F G || framework_failure_ +gzip -Sz F || fail=1 +test ! -f F || fail=1 +test -f Fz || fail=1 +gzip -dSz F || fail=1 +test ! -f Fz || fail=1 +compare F G || fail\1 + +Exit $fail diff --git a/src/zstd/tests/gzip/zdiff.sh b/src/zstd/tests/gzip/zdiff.sh new file mode 100644 index 000000000..d62a84606 --- /dev/null +++ b/src/zstd/tests/gzip/zdiff.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# Exercise zdiff with two compressed inputs. +# Before gzip-1.4, this would fail. + +# Copyright (C) 2009-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +echo a > a || framework_failure_ +echo b > b || framework_failure_ +gzip a b || framework_failure_ + +cat <<EOF > exp +1c1 +< a +--- +> b +EOF + +fail=0 +zdiff a.gz b.gz > out 2>&1 +test $? = 1 || fail=1 + +compare exp out || fail=1 + +rm -f out +# expect success, for equal files +zdiff a.gz a.gz > out 2> err || fail=1 +# expect no output +test -s out && fail=1 +# expect no stderr +test -s err && fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/zgrep-context.sh b/src/zstd/tests/gzip/zgrep-context.sh new file mode 100644 index 000000000..c8648b7e4 --- /dev/null +++ b/src/zstd/tests/gzip/zgrep-context.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Ensure that zgrep -15 works. Before gzip-1.5, it would fail. + +# Copyright (C) 2012-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +# A limited replacement for seq: handle 1 or 2 args; increment must be 1 +seq() +{ + case $# in + 1) start=1 final=$1;; + 2) start=$1 final=$2;; + *) echo you lose 1>&2; exit 1;; + esac + awk 'BEGIN{for(i='$start';i<='$final';i++) print i}' < /dev/null +} + +seq 40 > in || framework_failure_ +gzip < in > in.gz || framework_failure_ +seq 2 32 > exp || framework_failure_ + +: ${GREP=grep} +$GREP -15 17 - < in > out && compare exp out || { + echo >&2 "$0: $GREP does not support context options; skipping this test" + exit 77 +} + +fail=0 +zgrep -15 17 - < in.gz > out || fail=1 +compare exp out || fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/zgrep-f.sh b/src/zstd/tests/gzip/zgrep-f.sh new file mode 100644 index 000000000..d0cf27f7e --- /dev/null +++ b/src/zstd/tests/gzip/zgrep-f.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Ensure that zgrep -f - works like grep -f - +# Before gzip-1.4, it would fail. + +# Copyright (C) 2009-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +printf 'needle\nn2\n' > n || framework_failure_ +cp n haystack || framework_failure_ +gzip haystack || framework_failure_ + +fail=0 +zgrep -f - haystack.gz < n > out 2>&1 || fail=1 + +compare out n || fail=1 + +if ${BASH_VERSION+:} false; then + set +o posix + # This failed with gzip 1.6. + cat n n >nn || framework_failure_ + eval 'zgrep -h -f <(cat n) haystack.gz haystack.gz' >out || fail=1 + compare out nn || fail=1 +fi + +# This failed with gzip 1.4. +echo a-b | zgrep -e - > /dev/null || fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/zgrep-signal.sh b/src/zstd/tests/gzip/zgrep-signal.sh new file mode 100644 index 000000000..a8c53881a --- /dev/null +++ b/src/zstd/tests/gzip/zgrep-signal.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# Check that zgrep is terminated gracefully by signal when +# its grep/sed pipeline is terminated by a signal. + +# Copyright (C) 2010-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +echo a | gzip -c > f.gz || framework_failure_ + +test "x$PERL" = x && PERL=perl +("$PERL" -e 'use POSIX qw(dup2)') >/dev/null 2>&1 || + skip_ "no suitable perl found" + +# Run the arguments as a command, in a process where stdout is a +# dangling pipe and SIGPIPE has the default signal-handling action. +# This can't be done portably in the shell, because if SIGPIPE is +# ignored when the shell is entered, the shell might refuse to trap +# it. Fall back on Perl+POSIX, if available. Take care to close the +# pipe's read end before running the program; the equivalent of the +# shell's "command | :" has a race condition in that COMMAND could +# write before ":" exits. +write_to_dangling_pipe () { + program=${1?} + shift + args= + for arg; do + args="$args, '$arg'" + done + "$PERL" -e ' + use POSIX qw(dup2); + $SIG{PIPE} = "DEFAULT"; + pipe my ($read_end, $write_end) or die "pipe: $!\n"; + dup2 fileno $write_end, 1 or die "dup2: $!\n"; + close $read_end or die "close: $!\n"; + exec '"'$program'$args"'; + ' +} + +write_to_dangling_pipe cat f.gz f.gz +signal_status=$? +test 128 -lt $signal_status || + framework_failure_ 'signal handling busted on this host' + +fail=0 + +write_to_dangling_pipe zgrep a f.gz f.gz +test $? -eq $signal_status || fail=1 + +Exit $fail diff --git a/src/zstd/tests/gzip/znew-k.sh b/src/zstd/tests/gzip/znew-k.sh new file mode 100644 index 000000000..6c239e28e --- /dev/null +++ b/src/zstd/tests/gzip/znew-k.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Check that znew -K works without compress(1). + +# Copyright (C) 2010-2016 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +# limit so don't run it by default. + +. "${srcdir=.}/init.sh"; path_prepend_ . + +cat <<'EOF' >compress || framework_failure_ +#!/bin/sh +echo >&2 'compress has been invoked' +exit 1 +EOF +chmod +x compress || framework_failure_ + +# Note that the basename must have a length of 6 or greater. +# Otherwise, "test -f $name" below would fail. +name=123456.Z + +printf '%1012977s' ' ' | gzip -c > $name || framework_failure_ + +fail=0 + +znew -K $name || fail=1 +test -f $name || fail=1 + +Exit $fail diff --git a/src/zstd/tests/invalidDictionaries.c b/src/zstd/tests/invalidDictionaries.c new file mode 100644 index 000000000..23e93fd54 --- /dev/null +++ b/src/zstd/tests/invalidDictionaries.c @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include <stddef.h> +#include "zstd.h" + +static const char invalidRepCode[] = { + 0x37, 0xa4, 0x30, 0xec, 0x2a, 0x00, 0x00, 0x00, 0x39, 0x10, 0xc0, 0xc2, + 0xa6, 0x00, 0x0c, 0x30, 0xc0, 0x00, 0x03, 0x0c, 0x30, 0x20, 0x72, 0xf8, + 0xb4, 0x6d, 0x4b, 0x9f, 0xfc, 0x97, 0x29, 0x49, 0xb2, 0xdf, 0x4b, 0x29, + 0x7d, 0x4a, 0xfc, 0x83, 0x18, 0x22, 0x75, 0x23, 0x24, 0x44, 0x4d, 0x02, + 0xb7, 0x97, 0x96, 0xf6, 0xcb, 0xd1, 0xcf, 0xe8, 0x22, 0xea, 0x27, 0x36, + 0xb7, 0x2c, 0x40, 0x46, 0x01, 0x08, 0x23, 0x01, 0x00, 0x00, 0x06, 0x1e, + 0x3c, 0x83, 0x81, 0xd6, 0x18, 0xd4, 0x12, 0x3a, 0x04, 0x00, 0x80, 0x03, + 0x08, 0x0e, 0x12, 0x1c, 0x12, 0x11, 0x0d, 0x0e, 0x0a, 0x0b, 0x0a, 0x09, + 0x10, 0x0c, 0x09, 0x05, 0x04, 0x03, 0x06, 0x06, 0x06, 0x02, 0x00, 0x03, + 0x00, 0x00, 0x02, 0x02, 0x00, 0x04, 0x06, 0x03, 0x06, 0x08, 0x24, 0x6b, + 0x0d, 0x01, 0x10, 0x04, 0x81, 0x07, 0x00, 0x00, 0x04, 0xb9, 0x58, 0x18, + 0x06, 0x59, 0x92, 0x43, 0xce, 0x28, 0xa5, 0x08, 0x88, 0xc0, 0x80, 0x88, + 0x8c, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x08, 0x00, 0x00, 0x00 +}; + +typedef struct dictionary_s { + const char *data; + size_t size; +} dictionary; + +static const dictionary dictionaries[] = { + {invalidRepCode, sizeof(invalidRepCode)}, + {NULL, 0}, +}; + +int main(int argc, const char** argv) { + const dictionary *dict; + for (dict = dictionaries; dict->data != NULL; ++dict) { + ZSTD_CDict *cdict; + ZSTD_DDict *ddict; + cdict = ZSTD_createCDict(dict->data, dict->size, 1); + if (cdict) { + ZSTD_freeCDict(cdict); + return 1; + } + ddict = ZSTD_createDDict(dict->data, dict->size); + if (ddict) { + ZSTD_freeDDict(ddict); + return 2; + } + } + + (void)argc; + (void)argv; + return 0; +} diff --git a/src/zstd/tests/legacy.c b/src/zstd/tests/legacy.c new file mode 100644 index 000000000..3d3ec4358 --- /dev/null +++ b/src/zstd/tests/legacy.c @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + This program uses hard-coded data compressed with Zstd legacy versions + and tests that the API decompresses them correctly +*/ + +/*=========================================== +* Dependencies +*==========================================*/ +#include <stddef.h> /* size_t */ +#include <stdlib.h> /* malloc, free */ +#include <stdio.h> /* fprintf */ +#include <string.h> /* strlen */ +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_decompressBound */ +#include "zstd.h" +#include "zstd_errors.h" + +/*=========================================== +* Macros +*==========================================*/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + +/*=========================================== +* Precompressed frames +*==========================================*/ +const char* const COMPRESSED; /* content is at end of file */ +size_t const COMPRESSED_SIZE = 917; +const char* const EXPECTED; /* content is at end of file */ + + +static int testSimpleAPI(void) +{ + size_t const size = strlen(EXPECTED); + char* const output = malloc(size); + + if (!output) { + DISPLAY("ERROR: Not enough memory!\n"); + return 1; + } + + { + size_t const ret = ZSTD_decompress(output, size, COMPRESSED, COMPRESSED_SIZE); + if (ZSTD_isError(ret)) { + if (ret == ZSTD_error_prefix_unknown) { + DISPLAY("ERROR: Invalid frame magic number, was this compiled " + "without legacy support?\n"); + } else { + DISPLAY("ERROR: %s\n", ZSTD_getErrorName(ret)); + } + return 1; + } + if (ret != size) { + DISPLAY("ERROR: Wrong decoded size\n"); + } + } + if (memcmp(EXPECTED, output, size) != 0) { + DISPLAY("ERROR: Wrong decoded output produced\n"); + return 1; + } + + free(output); + DISPLAY("Simple API OK\n"); + return 0; +} + + +static int testStreamingAPI(void) +{ + int error_code = 0; + size_t const outBuffSize = ZSTD_DStreamOutSize(); + char* const outBuff = malloc(outBuffSize); + ZSTD_DStream* const stream = ZSTD_createDStream(); + ZSTD_inBuffer input = { COMPRESSED, COMPRESSED_SIZE, 0 }; + size_t outputPos = 0; + int needsInit = 1; + + if (outBuff == NULL) { + DISPLAY("ERROR: Could not allocate memory\n"); + return 1; + } + if (stream == NULL) { + DISPLAY("ERROR: Could not create dstream\n"); + free(outBuff); + return 1; + } + + while (1) { + ZSTD_outBuffer output = {outBuff, outBuffSize, 0}; + if (needsInit) { + size_t const ret = ZSTD_initDStream(stream); + if (ZSTD_isError(ret)) { + DISPLAY("ERROR: ZSTD_initDStream: %s\n", ZSTD_getErrorName(ret)); + error_code = 1; + break; + } } + + { size_t const ret = ZSTD_decompressStream(stream, &output, &input); + if (ZSTD_isError(ret)) { + DISPLAY("ERROR: ZSTD_decompressStream: %s\n", ZSTD_getErrorName(ret)); + error_code = 1; + break; + } + + if (ret == 0) { + needsInit = 1; + } } + + if (memcmp(outBuff, EXPECTED + outputPos, output.pos) != 0) { + DISPLAY("ERROR: Wrong decoded output produced\n"); + error_code = 1; + break; + } + outputPos += output.pos; + if (input.pos == input.size && output.pos < output.size) { + break; + } + } + + free(outBuff); + ZSTD_freeDStream(stream); + if (error_code == 0) DISPLAY("Streaming API OK\n"); + return error_code; +} + +static int testFrameDecoding(void) +{ + if (strlen(EXPECTED) > ZSTD_decompressBound(COMPRESSED, COMPRESSED_SIZE)) { + DISPLAY("ERROR: ZSTD_decompressBound: decompressed bound too small\n"); + return 1; + } + { const char* ip = COMPRESSED; + size_t remainingSize = COMPRESSED_SIZE; + while (1) { + size_t frameSize = ZSTD_findFrameCompressedSize(ip, remainingSize); + if (ZSTD_isError(frameSize)) { + DISPLAY("ERROR: ZSTD_findFrameCompressedSize: %s\n", ZSTD_getErrorName(frameSize)); + return 1; + } + if (frameSize > remainingSize) { + DISPLAY("ERROR: ZSTD_findFrameCompressedSize: expected frameSize to align with src buffer"); + return 1; + } + ip += frameSize; + remainingSize -= frameSize; + if (remainingSize == 0) break; + } + } + DISPLAY("Frame Decoding OK\n"); + return 0; +} + +int main(void) +{ + { int const ret = testSimpleAPI(); + if (ret) return ret; } + { int const ret = testStreamingAPI(); + if (ret) return ret; } + { int const ret = testFrameDecoding(); + if (ret) return ret; } + + DISPLAY("OK\n"); + return 0; +} + +/* Consists of the "EXPECTED" string compressed with default settings on + - v0.4.3 + - v0.5.0 + - v0.6.0 + - v0.7.0 + - v0.8.0 +*/ +const char* const COMPRESSED = + "\x24\xB5\x2F\xFD\x00\x00\x00\xBB\xB0\x02\xC0\x10\x00\x1E\xB0\x01" + "\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF\xE9\xF3\xEF\x53" + "\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89\x17\x00\x18\x00" + "\x18\x00\x3F\xE6\xE2\xE3\x74\xD6\xEC\xC9\x4A\xE0\x71\x71\x42\x3E" + "\x64\x4F\x6A\x45\x4E\x78\xEC\x49\x03\x3F\xC6\x80\xAB\x8F\x75\x5E" + "\x6F\x2E\x3E\x7E\xC6\xDC\x45\x69\x6C\xC5\xFD\xC7\x40\xB8\x84\x8A" + "\x01\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B\x67" + "\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\x19\x03\x01\x50\x67\x56\xF5\x9F" + "\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF" + "\xC6\xBA\x01\x0E\x00\x54\x00\x00\x19\x00\x00\x54\x14\x00\x24\x24" + "\x04\xFE\x04\x84\x4E\x41\x00\x27\xE2\x02\xC4\xB1\x00\xD2\x51\x00" + "\x79\x58\x41\x28\x00\xE0\x0C\x01\x68\x65\x00\x04\x13\x0C\xDA\x0C" + "\x80\x22\x06\xC0\x00\x00\x25\xB5\x2F\xFD\x00\x00\x00\xAD\x12\xB0" + "\x7D\x1E\xB0\x01\x02\x00\x00\x80\x00\xE8\x92\x34\x12\x97\xC8\xDF" + "\xE9\xF3\xEF\x53\xEA\x1D\x27\x4F\x0C\x44\x90\x0C\x8D\xF1\xB4\x89" + "\x03\x01\x50\x67\x56\xF5\x9F\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC" + "\xAB\xAB\xE0\xE2\x81\xFA\xCF\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C" + "\x64\xF8\xEB\x53\xE6\x18\x0B\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A" + "\xF9\x63\x0C\xB8\xFA\x58\xE7\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6" + "\x56\xDC\x7F\x0C\x84\x4B\xA8\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC" + "\x04\x1E\x17\x27\xE4\x43\xF6\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00" + "\x00\x32\x40\x80\xA8\x00\x01\x49\x81\xE0\x3C\x01\x29\x1D\x00\x87" + "\xCE\x80\x75\x08\x80\x72\x24\x00\x7B\x52\x00\x94\x00\x20\xCC\x01" + "\x86\xD2\x00\x81\x09\x83\xC1\x34\xA0\x88\x01\xC0\x00\x00\x26\xB5" + "\x2F\xFD\x42\xEF\x00\x00\xA6\x12\xB0\x7D\x1E\xB0\x01\x02\x00\x00" + "\x54\xA0\xBA\x24\x8D\xC4\x25\xF2\x77\xFA\xFC\xFB\x94\x7A\xC7\xC9" + "\x13\x03\x11\x24\x43\x63\x3C\x6D\x22\x03\x01\x50\x67\x56\xF5\x9F" + "\x35\x84\x60\xA0\x60\x91\xC9\x0A\xDC\xAB\xAB\xE0\xE2\x81\xFA\xCF" + "\xC6\xBA\xEB\xA8\xD1\x40\x39\x90\x4C\x64\xF8\xEB\x53\xE6\x18\x0B" + "\x67\x12\xAD\xB8\x99\xB3\x5A\x6F\x8A\xF9\x63\x0C\xB8\xFA\x58\xE7" + "\xF5\xE6\xE2\xE3\x67\xCC\x5D\x94\xC6\x56\xDC\x7F\x0C\x84\x4B\xA8" + "\xF8\x63\x2E\x3E\x4E\x67\xCD\x9E\xAC\x04\x1E\x17\x27\xE4\x43\xF6" + "\xA4\x56\xE4\x84\xC7\x9E\x34\x0E\x00\x35\x0B\x71\xB5\xC0\x2A\x5C" + "\x26\x94\x22\x20\x8B\x4C\x8D\x13\x47\x58\x67\x15\x6C\xF1\x1C\x4B" + "\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0\x00\x00" + "\x27\xB5\x2F\xFD\x20\xEF\x00\x00\xA6\x12\xE4\x84\x1F\xB0\x01\x10" + "\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF\x9F\xEF" + "\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21\x80\x32" + "\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20\x5F\x45" + "\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB\x44\x14" + "\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB\x89\x51" + "\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F\x59\xDB" + "\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E\xDF\x78" + "\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55\x4A\xD4" + "\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39\x43\xF1" + "\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D\x01\xC0" + "\x00\x00\x28\xB5\x2F\xFD\x24\xEF\x35\x05\x00\x92\x0B\x21\x1F\xB0" + "\x01\x10\x00\x00\x00\x35\x59\xA6\xE7\xA1\xEF\x7C\xFC\xBD\x3F\xFF" + "\x9F\xEF\xEE\xEF\x61\xC3\xAA\x31\x1D\x34\x38\x22\x22\x04\x44\x21" + "\x80\x32\xAD\x28\xF3\xD6\x28\x0C\x0A\x0E\xD6\x5C\xAC\x19\x8D\x20" + "\x5F\x45\x02\x2E\x17\x50\x66\x6D\xAC\x8B\x9C\x6E\x07\x73\x46\xBB" + "\x44\x14\xE7\x98\xC3\xB9\x17\x32\x6E\x33\x7C\x0E\x21\xB1\xDB\xCB" + "\x89\x51\x23\x34\xAB\x9D\xBC\x6D\x20\xF5\x03\xA9\x91\x4C\x2E\x1F" + "\x59\xDB\xD9\x35\x67\x4B\x0C\x95\x79\x10\x00\x85\xA6\x96\x95\x2E" + "\xDF\x78\x7B\x4A\x5C\x09\x76\x97\xD1\x5C\x96\x12\x75\x35\xA3\x55" + "\x4A\xD4\x0B\x00\x35\x0B\x71\xB5\xC0\x2A\x5C\xE6\x08\x45\xF1\x39" + "\x43\xF1\x1C\x4B\x54\x10\x9D\x31\x50\x85\x4B\x54\x0E\x01\x4B\x3D" + "\x01\xD2\x2F\x21\x80"; + +const char* const EXPECTED = + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" + + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" + + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" + + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n" + + "snowden is snowed in / he's now then in his snow den / when does the snow end?\n" + "goodbye little dog / you dug some holes in your day / they'll be hard to fill.\n" + "when life shuts a door, / just open it. it’s a door. / that is how doors work.\n"; diff --git a/src/zstd/tests/libzstd_partial_builds.sh b/src/zstd/tests/libzstd_partial_builds.sh new file mode 100755 index 000000000..b1c1e3b1a --- /dev/null +++ b/src/zstd/tests/libzstd_partial_builds.sh @@ -0,0 +1,89 @@ +#!/bin/sh -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +ECHO=echo +RM="rm -f" +GREP="grep" +INTOVOID="/dev/null" + +die() { + $ECHO "$@" 1>&2 + exit 1 +} + +isPresent() { + $GREP $@ tmplog || die "$@" "should be present" +} + +mustBeAbsent() { + $GREP $@ tmplog && die "$@ should not be there !!" + $ECHO "$@ correctly not present" # for some reason, this $ECHO must exist, otherwise mustBeAbsent() always fails (??) +} + +# default compilation : all features enabled +make clean > /dev/null +$ECHO "testing default library compilation" +CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +isPresent "zstd_decompress.o" +isPresent "zdict.o" +isPresent "zstd_v07.o" +isPresent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog + +# compression disabled => also disable zdict and zbuff +$ECHO "testing with compression disabled" +ZSTD_LIB_COMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +mustBeAbsent "zstd_compress.o" +isPresent "zstd_decompress.o" +mustBeAbsent "zdict.o" +isPresent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog + +# decompression disabled => also disable legacy and zbuff +$ECHO "testing with decompression disabled" +ZSTD_LIB_DECOMPRESSION=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +mustBeAbsent "zstd_decompress.o" +isPresent "zdict.o" +mustBeAbsent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog + +# deprecated function disabled => only remove zbuff +$ECHO "testing with deprecated functions disabled" +ZSTD_LIB_DEPRECATED=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +isPresent "zstd_decompress.o" +isPresent "zdict.o" +isPresent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog + +# dictionary builder disabled => only remove zdict +$ECHO "testing with dictionary builder disabled" +ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +isPresent "zstd_decompress.o" +mustBeAbsent "zdict.o" +isPresent "zstd_v07.o" +isPresent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog + +# both decompression and dictionary builder disabled => only compression remains +$ECHO "testing with both decompression and dictionary builder disabled (only compression remains)" +ZSTD_LIB_DECOMPRESSION=0 ZSTD_LIB_DICTBUILDER=0 CFLAGS= make -C $DIR/../lib libzstd.a > $INTOVOID +nm $DIR/../lib/libzstd.a | $GREP "\.o" > tmplog +isPresent "zstd_compress.o" +mustBeAbsent "zstd_decompress.o" +mustBeAbsent "zdict.o" +mustBeAbsent "zstd_v07.o" +mustBeAbsent "zbuff_compress.o" +$RM $DIR/../lib/libzstd.a tmplog diff --git a/src/zstd/tests/longmatch.c b/src/zstd/tests/longmatch.c new file mode 100644 index 000000000..93e78dd1f --- /dev/null +++ b/src/zstd/tests/longmatch.c @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2017-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#include <stdio.h> +#include <stddef.h> +#include <stdlib.h> +#include <stdint.h> +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +static int +compress(ZSTD_CStream *ctx, ZSTD_outBuffer out, const void *data, size_t size) +{ + ZSTD_inBuffer in = { data, size, 0 }; + while (in.pos < in.size) { + ZSTD_outBuffer tmp = out; + const size_t rc = ZSTD_compressStream(ctx, &tmp, &in); + if (ZSTD_isError(rc)) return 1; + } + { ZSTD_outBuffer tmp = out; + const size_t rc = ZSTD_flushStream(ctx, &tmp); + if (rc != 0) { return 1; } + } + return 0; +} + +int main(int argc, const char** argv) +{ + ZSTD_CStream* ctx; + ZSTD_parameters params; + size_t rc; + unsigned windowLog; + (void)argc; + (void)argv; + /* Create stream */ + ctx = ZSTD_createCStream(); + if (!ctx) { return 1; } + /* Set parameters */ + memset(¶ms, 0, sizeof(params)); + params.cParams.windowLog = 18; + params.cParams.chainLog = 13; + params.cParams.hashLog = 14; + params.cParams.searchLog = 1; + params.cParams.minMatch = 7; + params.cParams.targetLength = 16; + params.cParams.strategy = ZSTD_fast; + windowLog = params.cParams.windowLog; + /* Initialize stream */ + rc = ZSTD_initCStream_advanced(ctx, NULL, 0, params, 0); + if (ZSTD_isError(rc)) { return 2; } + { + U64 compressed = 0; + const U64 toCompress = ((U64)1) << 33; + const size_t size = 1 << windowLog; + size_t pos = 0; + char *srcBuffer = (char*) malloc(1 << windowLog); + char *dstBuffer = (char*) malloc(ZSTD_compressBound(1 << windowLog)); + ZSTD_outBuffer out = { dstBuffer, ZSTD_compressBound(1 << windowLog), 0 }; + const char match[] = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const size_t randomData = (1 << windowLog) - 2*sizeof(match); + size_t i; + printf("\n === Long Match Test === \n"); + printf("Creating random data to produce long matches \n"); + for (i = 0; i < sizeof(match); ++i) { + srcBuffer[i] = match[i]; + } + for (i = 0; i < randomData; ++i) { + srcBuffer[sizeof(match) + i] = (char)(rand() & 0xFF); + } + for (i = 0; i < sizeof(match); ++i) { + srcBuffer[sizeof(match) + randomData + i] = match[i]; + } + printf("Compressing, trying to generate a segfault \n"); + if (compress(ctx, out, srcBuffer, size)) { + return 1; + } + compressed += size; + while (compressed < toCompress) { + const size_t block = rand() % (size - pos + 1); + if (pos == size) { pos = 0; } + if (compress(ctx, out, srcBuffer + pos, block)) { + return 1; + } + pos += block; + compressed += block; + } + printf("Compression completed successfully (no error triggered)\n"); + free(srcBuffer); + free(dstBuffer); + } + return 0; +} diff --git a/src/zstd/tests/paramgrill.c b/src/zstd/tests/paramgrill.c new file mode 100644 index 000000000..e9cc2a94e --- /dev/null +++ b/src/zstd/tests/paramgrill.c @@ -0,0 +1,2966 @@ +/* + * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*-************************************ +* Dependencies +**************************************/ +#include "util.h" /* Ensure platform.h is compiled first; also : compiler options, UTIL_GetFileSize */ +#include <stdlib.h> /* malloc */ +#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <string.h> /* strcmp */ +#include <math.h> /* log */ +#include <assert.h> + +#include "timefn.h" /* SEC_TO_MICRO, UTIL_time_t, UTIL_clockSpanMicro, UTIL_clockSpanNano, UTIL_getTime */ +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_parameters, ZSTD_estimateCCtxSize */ +#include "zstd.h" +#include "datagen.h" +#include "xxhash.h" +#include "benchfn.h" +#include "benchzstd.h" +#include "zstd_errors.h" +#include "zstd_internal.h" /* should not be needed */ + + +/*-************************************ +* Constants +**************************************/ +#define PROGRAM_DESCRIPTION "ZSTD parameters tester" +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %s %i-bits, by %s ***\n", PROGRAM_DESCRIPTION, ZSTD_VERSION_STRING, (int)(sizeof(void*)*8), AUTHOR + +#define TIMELOOP_NANOSEC (1*1000000000ULL) /* 1 second */ +#define NB_LEVELS_TRACKED 22 /* ensured being >= ZSTD_maxCLevel() in BMK_init_level_constraints() */ + +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); + +#define COMPRESSIBILITY_DEFAULT 0.50 + +static const U64 g_maxVariationTime = 60 * SEC_TO_MICRO; +static const int g_maxNbVariations = 64; + + +/*-************************************ +* Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(n, ...) if(g_displayLevel >= n) { fprintf(stderr, __VA_ARGS__); } +#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } + +#define TIMED 0 +#ifndef DEBUG +# define DEBUG 0 +#endif + +#undef MIN +#undef MAX +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) +#define MAX(a,b) ( (a) > (b) ? (a) : (b) ) +#define CUSTOM_LEVEL 99 +#define BASE_CLEVEL 1 + +#define FADT_MIN 0 +#define FADT_MAX ((U32)-1) + +#define WLOG_RANGE (ZSTD_WINDOWLOG_MAX - ZSTD_WINDOWLOG_MIN + 1) +#define CLOG_RANGE (ZSTD_CHAINLOG_MAX - ZSTD_CHAINLOG_MIN + 1) +#define HLOG_RANGE (ZSTD_HASHLOG_MAX - ZSTD_HASHLOG_MIN + 1) +#define SLOG_RANGE (ZSTD_SEARCHLOG_MAX - ZSTD_SEARCHLOG_MIN + 1) +#define MML_RANGE (ZSTD_MINMATCH_MAX - ZSTD_MINMATCH_MIN + 1) +#define TLEN_RANGE 17 +#define STRT_RANGE (ZSTD_STRATEGY_MAX - ZSTD_STRATEGY_MIN + 1) +#define FADT_RANGE 3 + +#define CHECKTIME(r) { if(BMK_timeSpan_s(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); return r; } } +#define CHECKTIMEGT(ret, val, _gototag) { if(BMK_timeSpan_s(g_time) > g_timeLimit_s) { DEBUGOUTPUT("Time Limit Reached\n"); ret = val; goto _gototag; } } + +#define PARAM_UNSET ((U32)-2) /* can't be -1 b/c fadt uses -1 */ + +static const char* g_stratName[ZSTD_STRATEGY_MAX+1] = { + "(none) ", "ZSTD_fast ", "ZSTD_dfast ", + "ZSTD_greedy ", "ZSTD_lazy ", "ZSTD_lazy2 ", + "ZSTD_btlazy2 ", "ZSTD_btopt ", "ZSTD_btultra ", + "ZSTD_btultra2"}; + +static const U32 tlen_table[TLEN_RANGE] = { 0, 1, 2, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96, 128, 256, 512, 999 }; + + +/*-************************************ +* Setup for Adding new params +**************************************/ + +/* indices for each of the variables */ +typedef enum { + wlog_ind = 0, + clog_ind = 1, + hlog_ind = 2, + slog_ind = 3, + mml_ind = 4, + tlen_ind = 5, + strt_ind = 6, + fadt_ind = 7, /* forceAttachDict */ + NUM_PARAMS = 8 +} varInds_t; + +typedef struct { + U32 vals[NUM_PARAMS]; +} paramValues_t; + +/* minimum value of parameters */ +static const U32 mintable[NUM_PARAMS] = + { ZSTD_WINDOWLOG_MIN, ZSTD_CHAINLOG_MIN, ZSTD_HASHLOG_MIN, ZSTD_SEARCHLOG_MIN, ZSTD_MINMATCH_MIN, ZSTD_TARGETLENGTH_MIN, ZSTD_STRATEGY_MIN, FADT_MIN }; + +/* maximum value of parameters */ +static const U32 maxtable[NUM_PARAMS] = + { ZSTD_WINDOWLOG_MAX, ZSTD_CHAINLOG_MAX, ZSTD_HASHLOG_MAX, ZSTD_SEARCHLOG_MAX, ZSTD_MINMATCH_MAX, ZSTD_TARGETLENGTH_MAX, ZSTD_STRATEGY_MAX, FADT_MAX }; + +/* # of values parameters can take on */ +static const U32 rangetable[NUM_PARAMS] = + { WLOG_RANGE, CLOG_RANGE, HLOG_RANGE, SLOG_RANGE, MML_RANGE, TLEN_RANGE, STRT_RANGE, FADT_RANGE }; + +/* ZSTD_cctxSetParameter() index to set */ +static const ZSTD_cParameter cctxSetParamTable[NUM_PARAMS] = + { ZSTD_c_windowLog, ZSTD_c_chainLog, ZSTD_c_hashLog, ZSTD_c_searchLog, ZSTD_c_minMatch, ZSTD_c_targetLength, ZSTD_c_strategy, ZSTD_c_forceAttachDict }; + +/* names of parameters */ +static const char* g_paramNames[NUM_PARAMS] = + { "windowLog", "chainLog", "hashLog","searchLog", "minMatch", "targetLength", "strategy", "forceAttachDict" }; + +/* shortened names of parameters */ +static const char* g_shortParamNames[NUM_PARAMS] = + { "wlog", "clog", "hlog", "slog", "mml", "tlen", "strat", "fadt" }; + +/* maps value from { 0 to rangetable[param] - 1 } to valid paramvalues */ +static U32 rangeMap(varInds_t param, int ind) +{ + U32 const uind = (U32)MAX(MIN(ind, (int)rangetable[param] - 1), 0); + switch(param) { + case wlog_ind: /* using default: triggers -Wswitch-enum */ + case clog_ind: + case hlog_ind: + case slog_ind: + case mml_ind: + case strt_ind: + return mintable[param] + uind; + case tlen_ind: + return tlen_table[uind]; + case fadt_ind: /* 0, 1, 2 -> -1, 0, 1 */ + return uind - 1; + case NUM_PARAMS: + default:; + } + DISPLAY("Error, not a valid param\n "); + assert(0); + return (U32)-1; +} + +/* inverse of rangeMap */ +static int invRangeMap(varInds_t param, U32 value) +{ + value = MIN(MAX(mintable[param], value), maxtable[param]); + switch(param) { + case wlog_ind: + case clog_ind: + case hlog_ind: + case slog_ind: + case mml_ind: + case strt_ind: + return (int)(value - mintable[param]); + case tlen_ind: /* bin search */ + { + int lo = 0; + int hi = TLEN_RANGE; + while(lo < hi) { + int mid = (lo + hi) / 2; + if(tlen_table[mid] < value) { + lo = mid + 1; + } if(tlen_table[mid] == value) { + return mid; + } else { + hi = mid; + } + } + return lo; + } + case fadt_ind: + return (int)value + 1; + case NUM_PARAMS: + default:; + } + DISPLAY("Error, not a valid param\n "); + assert(0); + return -2; +} + +/* display of params */ +static void displayParamVal(FILE* f, varInds_t param, unsigned value, int width) +{ + switch(param) { + case wlog_ind: + case clog_ind: + case hlog_ind: + case slog_ind: + case mml_ind: + case tlen_ind: + if(width) { + fprintf(f, "%*u", width, value); + } else { + fprintf(f, "%u", value); + } + break; + case strt_ind: + if(width) { + fprintf(f, "%*s", width, g_stratName[value]); + } else { + fprintf(f, "%s", g_stratName[value]); + } + break; + case fadt_ind: /* force attach dict */ + if(width) { + fprintf(f, "%*d", width, (int)value); + } else { + fprintf(f, "%d", (int)value); + } + break; + case NUM_PARAMS: + default: + DISPLAY("Error, not a valid param\n "); + assert(0); + break; + } +} + + +/*-************************************ +* Benchmark Parameters/Global Variables +**************************************/ + +/* General Utility */ +static U32 g_timeLimit_s = 99999; /* about 27 hours */ +static UTIL_time_t g_time; /* to be used to compare solution finding speeds to compare to original */ +static U32 g_blockSize = 0; +static U32 g_rand = 1; + +/* Display */ +static int g_displayLevel = 3; +static BYTE g_silenceParams[NUM_PARAMS]; /* can selectively silence some params when displaying them */ + +/* Mode Selection */ +static U32 g_singleRun = 0; +static U32 g_optimizer = 0; +static int g_optmode = 0; + +/* For cLevel Table generation */ +static U32 g_target = 0; +static U32 g_noSeed = 0; + +/* For optimizer */ +static paramValues_t g_params; /* Initialized at the beginning of main w/ emptyParams() function */ +static double g_ratioMultiplier = 5.; +static U32 g_strictness = PARAM_UNSET; /* range 1 - 100, measure of how strict */ +static BMK_benchResult_t g_lvltarget; + +typedef enum { + directMap, + xxhashMap, + noMemo +} memoTableType_t; + +typedef struct { + memoTableType_t tableType; + BYTE* table; + size_t tableLen; + varInds_t varArray[NUM_PARAMS]; + size_t varLen; +} memoTable_t; + +typedef struct { + BMK_benchResult_t result; + paramValues_t params; +} winnerInfo_t; + +typedef struct { + U32 cSpeed; /* bytes / sec */ + U32 dSpeed; + U32 cMem; /* bytes */ +} constraint_t; + +typedef struct winner_ll_node winner_ll_node; +struct winner_ll_node { + winnerInfo_t res; + winner_ll_node* next; +}; + +static winner_ll_node* g_winners; /* linked list sorted ascending by cSize & cSpeed */ + +/* + * Additional Global Variables (Defined Above Use) + * g_level_constraint + * g_alreadyTested + * g_maxTries + * g_clockGranularity + */ + + +/*-******************************************************* +* General Util Functions +*********************************************************/ + +/* nullified useless params, to ensure count stats */ +/* cleans up params for memoizing / display */ +static paramValues_t sanitizeParams(paramValues_t params) +{ + if (params.vals[strt_ind] == ZSTD_fast) + params.vals[clog_ind] = 0, params.vals[slog_ind] = 0; + if (params.vals[strt_ind] == ZSTD_dfast) + params.vals[slog_ind] = 0; + if ( (params.vals[strt_ind] < ZSTD_btopt) && (params.vals[strt_ind] != ZSTD_fast) ) + params.vals[tlen_ind] = 0; + + return params; +} + +static ZSTD_compressionParameters pvalsToCParams(paramValues_t p) +{ + ZSTD_compressionParameters c; + memset(&c, 0, sizeof(ZSTD_compressionParameters)); + c.windowLog = p.vals[wlog_ind]; + c.chainLog = p.vals[clog_ind]; + c.hashLog = p.vals[hlog_ind]; + c.searchLog = p.vals[slog_ind]; + c.minMatch = p.vals[mml_ind]; + c.targetLength = p.vals[tlen_ind]; + c.strategy = p.vals[strt_ind]; + /* no forceAttachDict */ + return c; +} + +static paramValues_t cParamsToPVals(ZSTD_compressionParameters c) +{ + paramValues_t p; + varInds_t i; + p.vals[wlog_ind] = c.windowLog; + p.vals[clog_ind] = c.chainLog; + p.vals[hlog_ind] = c.hashLog; + p.vals[slog_ind] = c.searchLog; + p.vals[mml_ind] = c.minMatch; + p.vals[tlen_ind] = c.targetLength; + p.vals[strt_ind] = c.strategy; + + /* set all other params to their minimum value */ + for (i = strt_ind + 1; i < NUM_PARAMS; i++) { + p.vals[i] = mintable[i]; + } + return p; +} + +/* equivalent of ZSTD_adjustCParams for paramValues_t */ +static paramValues_t +adjustParams(paramValues_t p, const size_t maxBlockSize, const size_t dictSize) +{ + paramValues_t ot = p; + varInds_t i; + p = cParamsToPVals(ZSTD_adjustCParams(pvalsToCParams(p), maxBlockSize, dictSize)); + if (!dictSize) { p.vals[fadt_ind] = 0; } + /* retain value of all other parameters */ + for(i = strt_ind + 1; i < NUM_PARAMS; i++) { + p.vals[i] = ot.vals[i]; + } + return p; +} + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t const step = 64 MB; + void* testmem = NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + if (requiredMem > maxMemory) requiredMem = maxMemory; + + requiredMem += 2 * step; + while (!testmem && requiredMem > 0) { + testmem = malloc ((size_t)requiredMem); + requiredMem -= step; + } + + free (testmem); + return (size_t) requiredMem; +} + +/* accuracy in seconds only, span can be multiple years */ +static U32 BMK_timeSpan_s(const UTIL_time_t tStart) +{ + return (U32)(UTIL_clockSpanMicro(tStart) / 1000000ULL); +} + +static U32 FUZ_rotl32(U32 x, U32 r) +{ + return ((x << r) | (x >> (32 - r))); +} + +static U32 FUZ_rand(U32* src) +{ + const U32 prime1 = 2654435761U; + const U32 prime2 = 2246822519U; + U32 rand32 = *src; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *src = rand32; + return rand32 >> 5; +} + +#define BOUNDCHECK(val,min,max) { \ + if (((val)<(min)) | ((val)>(max))) { \ + DISPLAY("INVALID PARAMETER CONSTRAINTS\n"); \ + return 0; \ +} } + +static int paramValid(const paramValues_t paramTarget) +{ + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + BOUNDCHECK(paramTarget.vals[i], mintable[i], maxtable[i]); + } + return 1; +} + +/* cParamUnsetMin() : + * if any parameter in paramTarget is not yet set, + * it will receive its corresponding minimal value. + * This function never fails */ +static paramValues_t cParamUnsetMin(paramValues_t paramTarget) +{ + varInds_t vi; + for (vi = 0; vi < NUM_PARAMS; vi++) { + if (paramTarget.vals[vi] == PARAM_UNSET) { + paramTarget.vals[vi] = mintable[vi]; + } + } + return paramTarget; +} + +static paramValues_t emptyParams(void) +{ + U32 i; + paramValues_t p; + for(i = 0; i < NUM_PARAMS; i++) { + p.vals[i] = PARAM_UNSET; + } + return p; +} + +static winnerInfo_t initWinnerInfo(const paramValues_t p) +{ + winnerInfo_t w1; + w1.result.cSpeed = 0.; + w1.result.dSpeed = 0.; + w1.result.cMem = (size_t)-1; + w1.result.cSize = (size_t)-1; + w1.params = p; + return w1; +} + +static paramValues_t +overwriteParams(paramValues_t base, const paramValues_t mask) +{ + U32 i; + for(i = 0; i < NUM_PARAMS; i++) { + if(mask.vals[i] != PARAM_UNSET) { + base.vals[i] = mask.vals[i]; + } + } + return base; +} + +static void +paramVaryOnce(const varInds_t paramIndex, const int amt, paramValues_t* ptr) +{ + ptr->vals[paramIndex] = rangeMap(paramIndex, + invRangeMap(paramIndex, ptr->vals[paramIndex]) + amt); +} + +/* varies ptr by nbChanges respecting varyParams*/ +static void +paramVariation(paramValues_t* ptr, memoTable_t* mtAll, const U32 nbChanges) +{ + paramValues_t p; + int validated = 0; + while (!validated) { + U32 i; + p = *ptr; + for (i = 0 ; i < nbChanges ; i++) { + const U32 changeID = (U32)FUZ_rand(&g_rand) % (mtAll[p.vals[strt_ind]].varLen << 1); + paramVaryOnce(mtAll[p.vals[strt_ind]].varArray[changeID >> 1], + (int)((changeID & 1) << 1) - 1, + &p); + } + validated = paramValid(p); + } + *ptr = p; +} + +/* Completely random parameter selection */ +static paramValues_t randomParams(void) +{ + varInds_t v; paramValues_t p; + for(v = 0; v < NUM_PARAMS; v++) { + p.vals[v] = rangeMap(v, (int)(FUZ_rand(&g_rand) % rangetable[v])); + } + return p; +} + +static U64 g_clockGranularity = 100000000ULL; + +static void init_clockGranularity(void) +{ + UTIL_time_t const clockStart = UTIL_getTime(); + U64 el1 = 0, el2 = 0; + int i = 0; + do { + el1 = el2; + el2 = UTIL_clockSpanNano(clockStart); + if(el1 < el2) { + U64 iv = el2 - el1; + if(g_clockGranularity > iv) { + g_clockGranularity = iv; + i = 0; + } else { + i++; + } + } + } while(i < 10); + DEBUGOUTPUT("Granularity: %llu\n", (unsigned long long)g_clockGranularity); +} + +/*-************************************ +* Optimizer Util Functions +**************************************/ + +/* checks results are feasible */ +static int feasible(const BMK_benchResult_t results, const constraint_t target) { + return (results.cSpeed >= target.cSpeed) + && (results.dSpeed >= target.dSpeed) + && (results.cMem <= target.cMem) + && (!g_optmode || results.cSize <= g_lvltarget.cSize); +} + +/* hill climbing value for part 1 */ +/* Scoring here is a linear reward for all set constraints normalized between 0 to 1 + * (with 0 at 0 and 1 being fully fulfilling the constraint), summed with a logarithmic + * bonus to exceeding the constraint value. We also give linear ratio for compression ratio. + * The constant factors are experimental. + */ +static double +resultScore(const BMK_benchResult_t res, const size_t srcSize, const constraint_t target) +{ + double cs = 0., ds = 0., rt, cm = 0.; + const double r1 = 1, r2 = 0.1, rtr = 0.5; + double ret; + if(target.cSpeed) { cs = res.cSpeed / (double)target.cSpeed; } + if(target.dSpeed) { ds = res.dSpeed / (double)target.dSpeed; } + if(target.cMem != (U32)-1) { cm = (double)target.cMem / res.cMem; } + rt = ((double)srcSize / res.cSize); + + ret = (MIN(1, cs) + MIN(1, ds) + MIN(1, cm))*r1 + rt * rtr + + (MAX(0, log(cs))+ MAX(0, log(ds))+ MAX(0, log(cm))) * r2; + + return ret; +} + +/* calculates normalized squared euclidean distance of result1 if it is in the first quadrant relative to lvlRes */ +static double +resultDistLvl(const BMK_benchResult_t result1, const BMK_benchResult_t lvlRes) +{ + double normalizedCSpeedGain1 = (result1.cSpeed / lvlRes.cSpeed) - 1; + double normalizedRatioGain1 = ((double)lvlRes.cSize / result1.cSize) - 1; + if(normalizedRatioGain1 < 0 || normalizedCSpeedGain1 < 0) { + return 0.0; + } + return normalizedRatioGain1 * g_ratioMultiplier + normalizedCSpeedGain1; +} + +/* return true if r2 strictly better than r1 */ +static int +compareResultLT(const BMK_benchResult_t result1, const BMK_benchResult_t result2, const constraint_t target, size_t srcSize) +{ + if(feasible(result1, target) && feasible(result2, target)) { + if(g_optmode) { + return resultDistLvl(result1, g_lvltarget) < resultDistLvl(result2, g_lvltarget); + } else { + return (result1.cSize > result2.cSize) + || (result1.cSize == result2.cSize && result2.cSpeed > result1.cSpeed) + || (result1.cSize == result2.cSize && result2.cSpeed == result1.cSpeed && result2.dSpeed > result1.dSpeed); + } + } + return feasible(result2, target) + || (!feasible(result1, target) + && (resultScore(result1, srcSize, target) < resultScore(result2, srcSize, target))); +} + +static constraint_t relaxTarget(constraint_t target) { + target.cMem = (U32)-1; + target.cSpeed = (target.cSpeed * g_strictness) / 100; + target.dSpeed = (target.dSpeed * g_strictness) / 100; + return target; +} + +static void optimizerAdjustInput(paramValues_t* pc, const size_t maxBlockSize) +{ + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(pc->vals[v] != PARAM_UNSET) { + U32 newval = MIN(MAX(pc->vals[v], mintable[v]), maxtable[v]); + if(newval != pc->vals[v]) { + pc->vals[v] = newval; + DISPLAY("Warning: parameter %s not in valid range, adjusting to ", + g_paramNames[v]); + displayParamVal(stderr, v, newval, 0); DISPLAY("\n"); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET) { + + U32 sshb = maxBlockSize > 1 ? ZSTD_highbit32((U32)(maxBlockSize-1)) + 1 : 1; + /* edge case of highBit not working for 0 */ + + if(maxBlockSize < (1ULL << 31) && sshb + 1 < pc->vals[wlog_ind]) { + U32 adjust = MAX(mintable[wlog_ind], sshb); + if(adjust != pc->vals[wlog_ind]) { + pc->vals[wlog_ind] = adjust; + DISPLAY("Warning: windowLog larger than src/block size, adjusted to %u\n", + (unsigned)pc->vals[wlog_ind]); + } + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + U32 maxclog; + if(pc->vals[strt_ind] == PARAM_UNSET || pc->vals[strt_ind] >= (U32)ZSTD_btlazy2) { + maxclog = pc->vals[wlog_ind] + 1; + } else { + maxclog = pc->vals[wlog_ind]; + } + + if(pc->vals[clog_ind] > maxclog) { + pc->vals[clog_ind] = maxclog; + DISPLAY("Warning: chainlog too much larger than windowLog size, adjusted to %u\n", + (unsigned)pc->vals[clog_ind]); + } + } + + if(pc->vals[wlog_ind] != PARAM_UNSET && pc->vals[hlog_ind] != PARAM_UNSET) { + if(pc->vals[wlog_ind] + 1 < pc->vals[hlog_ind]) { + pc->vals[hlog_ind] = pc->vals[wlog_ind] + 1; + DISPLAY("Warning: hashlog too much larger than windowLog size, adjusted to %u\n", + (unsigned)pc->vals[hlog_ind]); + } + } + + if(pc->vals[slog_ind] != PARAM_UNSET && pc->vals[clog_ind] != PARAM_UNSET) { + if(pc->vals[slog_ind] > pc->vals[clog_ind]) { + pc->vals[clog_ind] = pc->vals[slog_ind]; + DISPLAY("Warning: searchLog larger than chainLog, adjusted to %u\n", + (unsigned)pc->vals[slog_ind]); + } + } +} + +static int +redundantParams(const paramValues_t paramValues, const constraint_t target, const size_t maxBlockSize) +{ + return + (ZSTD_estimateCStreamSize_usingCParams(pvalsToCParams(paramValues)) > (size_t)target.cMem) /* Uses too much memory */ + || ((1ULL << (paramValues.vals[wlog_ind] - 1)) >= maxBlockSize && paramValues.vals[wlog_ind] != mintable[wlog_ind]) /* wlog too much bigger than src size */ + || (paramValues.vals[clog_ind] > (paramValues.vals[wlog_ind] + (paramValues.vals[strt_ind] > ZSTD_btlazy2))) /* chainLog larger than windowLog*/ + || (paramValues.vals[slog_ind] > paramValues.vals[clog_ind]) /* searchLog larger than chainLog */ + || (paramValues.vals[hlog_ind] > paramValues.vals[wlog_ind] + 1); /* hashLog larger than windowLog + 1 */ +} + + +/*-************************************ +* Display Functions +**************************************/ + +/* BMK_paramValues_into_commandLine() : + * transform a set of parameters paramValues_t + * into a command line compatible with `zstd` syntax + * and writes it into FILE* f. + * f must be already opened and writable */ +static void +BMK_paramValues_into_commandLine(FILE* f, const paramValues_t params) +{ + varInds_t v; + int first = 1; + fprintf(f,"--zstd="); + for (v = 0; v < NUM_PARAMS; v++) { + if (g_silenceParams[v]) { continue; } + if (!first) { fprintf(f, ","); } + fprintf(f,"%s=", g_paramNames[v]); + + if (v == strt_ind) { fprintf(f,"%u", (unsigned)params.vals[v]); } + else { displayParamVal(f, v, params.vals[v], 0); } + first = 0; + } + fprintf(f, "\n"); +} + + +/* comparison function: */ +/* strictly better, strictly worse, equal, speed-side adv, size-side adv */ +#define WORSE_RESULT 0 +#define BETTER_RESULT 1 +#define ERROR_RESULT 2 + +#define SPEED_RESULT 4 +#define SIZE_RESULT 5 +/* maybe have epsilon-eq to limit table size? */ +static int +speedSizeCompare(const BMK_benchResult_t r1, const BMK_benchResult_t r2) +{ + if(r1.cSpeed < r2.cSpeed) { + if(r1.cSize >= r2.cSize) { + return BETTER_RESULT; + } + return SPEED_RESULT; /* r2 is smaller but not faster. */ + } else { + if(r1.cSize <= r2.cSize) { + return WORSE_RESULT; + } + return SIZE_RESULT; /* r2 is faster but not smaller */ + } +} + +/* 0 for insertion, 1 for no insert */ +/* maintain invariant speedSizeCompare(n, n->next) = SPEED_RESULT */ +static int +insertWinner(const winnerInfo_t w, const constraint_t targetConstraints) +{ + BMK_benchResult_t r = w.result; + winner_ll_node* cur_node = g_winners; + /* first node to insert */ + if(!feasible(r, targetConstraints)) { + return 1; + } + + if(g_winners == NULL) { + winner_ll_node* first_node = malloc(sizeof(winner_ll_node)); + if(first_node == NULL) { + return 1; + } + first_node->next = NULL; + first_node->res = w; + g_winners = first_node; + return 0; + } + + while(cur_node->next != NULL) { + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: + { + return 1; /* never insert if better */ + } + case BETTER_RESULT: + { + winner_ll_node* tmp; + cur_node->res = cur_node->next->res; + tmp = cur_node->next; + cur_node->next = cur_node->next->next; + free(tmp); + break; + } + case SIZE_RESULT: + { + cur_node = cur_node->next; + break; + } + case SPEED_RESULT: /* insert after first size result, then return */ + { + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + } + + } + + assert(cur_node->next == NULL); + switch(speedSizeCompare(cur_node->res.result, r)) { + case WORSE_RESULT: + { + return 1; /* never insert if better */ + } + case BETTER_RESULT: + { + cur_node->res = w; + return 0; + } + case SIZE_RESULT: + { + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = w; + newnode->next = NULL; + cur_node->next = newnode; + return 0; + } + case SPEED_RESULT: /* insert before first size result, then return */ + { + winner_ll_node* newnode = malloc(sizeof(winner_ll_node)); + if(newnode == NULL) { + return 1; + } + newnode->res = cur_node->res; + cur_node->res = w; + newnode->next = cur_node->next; + cur_node->next = newnode; + return 0; + } + default: + return 1; + } +} + +static void +BMK_displayOneResult(FILE* f, winnerInfo_t res, const size_t srcSize) +{ + varInds_t v; + int first = 1; + res.params = cParamUnsetMin(res.params); + fprintf(f, " {"); + for (v = 0; v < NUM_PARAMS; v++) { + if (g_silenceParams[v]) { continue; } + if (!first) { fprintf(f, ","); } + displayParamVal(f, v, res.params.vals[v], 3); + first = 0; + } + + { double const ratio = res.result.cSize ? + (double)srcSize / res.result.cSize : 0; + double const cSpeedMBps = (double)res.result.cSpeed / MB_UNIT; + double const dSpeedMBps = (double)res.result.dSpeed / MB_UNIT; + + fprintf(f, " }, /* R:%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + ratio, cSpeedMBps, dSpeedMBps); + } +} + +/* Writes to f the results of a parameter benchmark */ +/* when used with --optimize, will only print results better than previously discovered */ +static void +BMK_printWinner(FILE* f, const int cLevel, const BMK_benchResult_t result, const paramValues_t params, const size_t srcSize) +{ + char lvlstr[15] = "Custom Level"; + winnerInfo_t w; + w.params = params; + w.result = result; + + fprintf(f, "\r%79s\r", ""); + + if(cLevel != CUSTOM_LEVEL) { + snprintf(lvlstr, 15, " Level %2d ", cLevel); + } + + if(TIMED) { + const U64 mn_in_ns = 60ULL * TIMELOOP_NANOSEC; + const U64 time_ns = UTIL_clockSpanNano(g_time); + const U64 minutes = time_ns / mn_in_ns; + fprintf(f, "%1lu:%2lu:%05.2f - ", + (unsigned long) minutes / 60, + (unsigned long) minutes % 60, + (double)(time_ns - (minutes * mn_in_ns)) / TIMELOOP_NANOSEC ); + } + + fprintf(f, "/* %s */ ", lvlstr); + BMK_displayOneResult(f, w, srcSize); +} + +static void +BMK_printWinnerOpt(FILE* f, const U32 cLevel, const BMK_benchResult_t result, const paramValues_t params, const constraint_t targetConstraints, const size_t srcSize) +{ + /* global winner used for constraints */ + /* cSize, cSpeed, dSpeed, cMem */ + static winnerInfo_t g_winner = { { (size_t)-1LL, 0, 0, (size_t)-1LL }, + { { PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET, PARAM_UNSET } } + }; + if ( DEBUG + || compareResultLT(g_winner.result, result, targetConstraints, srcSize) + || g_displayLevel >= 4) { + if ( DEBUG + && compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + DISPLAY("New Winner: \n"); + } + + if(g_displayLevel >= 2) { + BMK_printWinner(f, cLevel, result, params, srcSize); + } + + if(compareResultLT(g_winner.result, result, targetConstraints, srcSize)) { + if(g_displayLevel >= 1) { BMK_paramValues_into_commandLine(f, params); } + g_winner.result = result; + g_winner.params = params; + } + } + + if(g_optmode && g_optimizer && (DEBUG || g_displayLevel == 3)) { + winnerInfo_t w; + winner_ll_node* n; + w.result = result; + w.params = params; + insertWinner(w, targetConstraints); + + if(!DEBUG) { fprintf(f, "\033c"); } + fprintf(f, "\n"); + + /* the table */ + fprintf(f, "================================\n"); + for(n = g_winners; n != NULL; n = n->next) { + BMK_displayOneResult(f, n->res, srcSize); + } + fprintf(f, "================================\n"); + fprintf(f, "Level Bounds: R: > %.3f AND C: < %.1f MB/s \n\n", + (double)srcSize / g_lvltarget.cSize, (double)g_lvltarget.cSpeed / MB_UNIT); + + + fprintf(f, "Overall Winner: \n"); + BMK_displayOneResult(f, g_winner, srcSize); + BMK_paramValues_into_commandLine(f, g_winner.params); + + fprintf(f, "Latest BMK: \n");\ + BMK_displayOneResult(f, w, srcSize); + } +} + + +/* BMK_print_cLevelEntry() : + * Writes one cLevelTable entry, for one level. + * f must exist, be already opened, and be seekable. + * this function cannot error. + */ +static void +BMK_print_cLevelEntry(FILE* f, const int cLevel, + paramValues_t params, + const BMK_benchResult_t result, const size_t srcSize) +{ + varInds_t v; + int first = 1; + + assert(cLevel >= 0); + assert(cLevel <= NB_LEVELS_TRACKED); + params = cParamUnsetMin(params); + + fprintf(f, " {"); + /* print cParams. + * assumption : all cParams are present and in order in the following range */ + for (v = 0; v <= strt_ind; v++) { + if (!first) { fprintf(f, ","); } + displayParamVal(f, v, params.vals[v], 3); + first = 0; + } + /* print comment */ + { double const ratio = result.cSize ? + (double)srcSize / result.cSize : 0; + double const cSpeedMBps = (double)result.cSpeed / MB_UNIT; + double const dSpeedMBps = (double)result.dSpeed / MB_UNIT; + + fprintf(f, " }, /* level %2i: R=%5.3f at %5.1f MB/s - %5.1f MB/s */\n", + cLevel, ratio, cSpeedMBps, dSpeedMBps); + } +} + + +/* BMK_print_cLevelTable() : + * print candidate compression table into proposed FILE* f. + * f must exist, be already opened, and be seekable. + * winners must be a table of NB_LEVELS_TRACKED+1 elements winnerInfo_t, all entries presumed initialized + * this function cannot error. + */ +static void +BMK_print_cLevelTable(FILE* f, const winnerInfo_t* winners, const size_t srcSize) +{ + int cLevel; + + fprintf(f, "\n /* Proposed configurations : */ \n"); + fprintf(f, " /* W, C, H, S, L, T, strat */ \n"); + + for (cLevel=0; cLevel <= NB_LEVELS_TRACKED; cLevel++) + BMK_print_cLevelEntry(f, + cLevel, winners[cLevel].params, + winners[cLevel].result, srcSize); +} + + +/* BMK_saveAndPrint_cLevelTable() : + * save candidate compression table into FILE* f, + * and then to stdout. + * f must exist, be already opened, and be seekable. + * winners must be a table of NB_LEVELS_TRACKED+1 elements winnerInfo_t, all entries presumed initialized + * this function cannot error. + */ +static void +BMK_saveAndPrint_cLevelTable(FILE* const f, + const winnerInfo_t* winners, + const size_t srcSize) +{ + fseek(f, 0, SEEK_SET); + BMK_print_cLevelTable(f, winners, srcSize); + fflush(f); + BMK_print_cLevelTable(stdout, winners, srcSize); +} + + +/*-******************************************************* +* Functions to Benchmark +*********************************************************/ + +typedef struct { + ZSTD_CCtx* cctx; + const void* dictBuffer; + size_t dictBufferSize; + int cLevel; + const paramValues_t* comprParams; +} BMK_initCCtxArgs; + +static size_t local_initCCtx(void* payload) { + const BMK_initCCtxArgs* ag = (const BMK_initCCtxArgs*)payload; + varInds_t i; + ZSTD_CCtx_reset(ag->cctx, ZSTD_reset_session_and_parameters); + ZSTD_CCtx_setParameter(ag->cctx, ZSTD_c_compressionLevel, ag->cLevel); + + for(i = 0; i < NUM_PARAMS; i++) { + if(ag->comprParams->vals[i] != PARAM_UNSET) + ZSTD_CCtx_setParameter(ag->cctx, cctxSetParamTable[i], ag->comprParams->vals[i]); + } + ZSTD_CCtx_loadDictionary(ag->cctx, ag->dictBuffer, ag->dictBufferSize); + + return 0; +} + +typedef struct { + ZSTD_DCtx* dctx; + const void* dictBuffer; + size_t dictBufferSize; +} BMK_initDCtxArgs; + +static size_t local_initDCtx(void* payload) { + const BMK_initDCtxArgs* ag = (const BMK_initDCtxArgs*)payload; + ZSTD_DCtx_reset(ag->dctx, ZSTD_reset_session_and_parameters); + ZSTD_DCtx_loadDictionary(ag->dctx, ag->dictBuffer, ag->dictBufferSize); + return 0; +} + +/* additional argument is just the context */ +static size_t local_defaultCompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) +{ + ZSTD_CCtx* cctx = (ZSTD_CCtx*)addArgs; + assert(dstSize == ZSTD_compressBound(srcSize)); /* specific to this version, which is only used in paramgrill */ + return ZSTD_compress2(cctx, dstBuffer, dstSize, srcBuffer, srcSize); +} + +/* additional argument is just the context */ +static size_t local_defaultDecompress( + const void* srcBuffer, size_t srcSize, + void* dstBuffer, size_t dstSize, + void* addArgs) { + size_t moreToFlush = 1; + ZSTD_DCtx* dctx = (ZSTD_DCtx*)addArgs; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + in.src = srcBuffer; + in.size = srcSize; + in.pos = 0; + out.dst = dstBuffer; + out.size = dstSize; + out.pos = 0; + while (moreToFlush) { + if(out.pos == out.size) { + return (size_t)-ZSTD_error_dstSize_tooSmall; + } + moreToFlush = ZSTD_decompressStream(dctx, + &out, &in); + if (ZSTD_isError(moreToFlush)) { + return moreToFlush; + } + } + return out.pos; + +} + +/*-************************************ +* Data Initialization Functions +**************************************/ + +typedef struct { + void* srcBuffer; + size_t srcSize; + const void** srcPtrs; + size_t* srcSizes; + void** dstPtrs; + size_t* dstCapacities; + size_t* dstSizes; + void** resPtrs; + size_t* resSizes; + size_t nbBlocks; + size_t maxBlockSize; +} buffers_t; + +typedef struct { + size_t dictSize; + void* dictBuffer; + ZSTD_CCtx* cctx; + ZSTD_DCtx* dctx; +} contexts_t; + +static void freeNonSrcBuffers(const buffers_t b) { + free(b.srcPtrs); + free(b.srcSizes); + + if(b.dstPtrs != NULL) { + free(b.dstPtrs[0]); + } + free(b.dstPtrs); + free(b.dstCapacities); + free(b.dstSizes); + + if(b.resPtrs != NULL) { + free(b.resPtrs[0]); + } + free(b.resPtrs); + free(b.resSizes); +} + +static void freeBuffers(const buffers_t b) { + if(b.srcPtrs != NULL) { + free(b.srcBuffer); + } + freeNonSrcBuffers(b); +} + +/* srcBuffer will be freed by freeBuffers now */ +static int createBuffersFromMemory(buffers_t* buff, void * srcBuffer, const size_t nbFiles, + const size_t* fileSizes) +{ + size_t pos = 0, n, blockSize; + U32 maxNbBlocks, blockNb = 0; + buff->srcSize = 0; + for(n = 0; n < nbFiles; n++) { + buff->srcSize += fileSizes[n]; + } + + if(buff->srcSize == 0) { + DISPLAY("No data to bench\n"); + return 1; + } + + blockSize = g_blockSize ? g_blockSize : buff->srcSize; + maxNbBlocks = (U32) ((buff->srcSize + (blockSize-1)) / blockSize) + (U32)nbFiles; + + buff->srcPtrs = (const void**)calloc(maxNbBlocks, sizeof(void*)); + buff->srcSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->dstPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->dstCapacities = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + buff->dstSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + buff->resPtrs = (void**)calloc(maxNbBlocks, sizeof(void*)); + buff->resSizes = (size_t*)malloc(maxNbBlocks * sizeof(size_t)); + + if(!buff->srcPtrs || !buff->srcSizes || !buff->dstPtrs || !buff->dstCapacities || !buff->dstSizes || !buff->resPtrs || !buff->resSizes) { + DISPLAY("alloc error\n"); + freeNonSrcBuffers(*buff); + return 1; + } + + buff->srcBuffer = srcBuffer; + buff->srcPtrs[0] = (const void*)buff->srcBuffer; + buff->dstPtrs[0] = malloc(ZSTD_compressBound(buff->srcSize) + (maxNbBlocks * 1024)); + buff->resPtrs[0] = malloc(buff->srcSize); + + if(!buff->dstPtrs[0] || !buff->resPtrs[0]) { + DISPLAY("alloc error\n"); + freeNonSrcBuffers(*buff); + return 1; + } + + for(n = 0; n < nbFiles; n++) { + size_t pos_end = pos + fileSizes[n]; + for(; pos < pos_end; blockNb++) { + buff->srcPtrs[blockNb] = (const void*)((char*)srcBuffer + pos); + buff->srcSizes[blockNb] = blockSize; + pos += blockSize; + } + + if(fileSizes[n] > 0) { buff->srcSizes[blockNb - 1] = ((fileSizes[n] - 1) % blockSize) + 1; } + pos = pos_end; + } + + buff->dstCapacities[0] = ZSTD_compressBound(buff->srcSizes[0]); + buff->dstSizes[0] = buff->dstCapacities[0]; + buff->resSizes[0] = buff->srcSizes[0]; + buff->maxBlockSize = buff->srcSizes[0]; + + for(n = 1; n < blockNb; n++) { + buff->dstPtrs[n] = ((char*)buff->dstPtrs[n-1]) + buff->dstCapacities[n-1]; + buff->resPtrs[n] = ((char*)buff->resPtrs[n-1]) + buff->resSizes[n-1]; + buff->dstCapacities[n] = ZSTD_compressBound(buff->srcSizes[n]); + buff->dstSizes[n] = buff->dstCapacities[n]; + buff->resSizes[n] = buff->srcSizes[n]; + + buff->maxBlockSize = MAX(buff->maxBlockSize, buff->srcSizes[n]); + } + + buff->nbBlocks = blockNb; + + return 0; +} + +/* allocates buffer's arguments. returns success / failure */ +static int createBuffers(buffers_t* buff, const char* const * const fileNamesTable, + size_t nbFiles) { + size_t pos = 0; + size_t n; + size_t totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, (U32)nbFiles); + size_t benchedSize = MIN(BMK_findMaxMem(totalSizeToLoad * 3) / 3, totalSizeToLoad); + size_t* fileSizes = calloc(sizeof(size_t), nbFiles); + void* srcBuffer = NULL; + int ret = 0; + + if(!totalSizeToLoad || !benchedSize) { + ret = 1; + DISPLAY("Nothing to Bench\n"); + goto _cleanUp; + } + + srcBuffer = malloc(benchedSize); + + if(!fileSizes || !srcBuffer) { + ret = 1; + goto _cleanUp; + } + + for(n = 0; n < nbFiles; n++) { + FILE* f; + U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); + if (UTIL_isDirectory(fileNamesTable[n])) { + DISPLAY("Ignoring %s directory... \n", fileNamesTable[n]); + continue; + } + if (fileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAY("Cannot evaluate size of %s, ignoring ... \n", fileNamesTable[n]); + continue; + } + f = fopen(fileNamesTable[n], "rb"); + if (f==NULL) { + DISPLAY("impossible to open file %s\n", fileNamesTable[n]); + fclose(f); + ret = 10; + goto _cleanUp; + } + + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[n]); + + if (fileSize + pos > benchedSize) fileSize = benchedSize - pos, nbFiles=n; /* buffer too small - stop after this file */ + { + char* buffer = (char*)(srcBuffer); + size_t const readSize = fread((buffer)+pos, 1, (size_t)fileSize, f); + fclose(f); + if (readSize != (size_t)fileSize) { + DISPLAY("could not read %s", fileNamesTable[n]); + ret = 1; + goto _cleanUp; + } + + fileSizes[n] = readSize; + pos += readSize; + } + } + + ret = createBuffersFromMemory(buff, srcBuffer, nbFiles, fileSizes); + +_cleanUp: + if(ret) { free(srcBuffer); } + free(fileSizes); + return ret; +} + +static void freeContexts(const contexts_t ctx) { + free(ctx.dictBuffer); + ZSTD_freeCCtx(ctx.cctx); + ZSTD_freeDCtx(ctx.dctx); +} + +static int createContexts(contexts_t* ctx, const char* dictFileName) { + FILE* f; + size_t readSize; + ctx->cctx = ZSTD_createCCtx(); + ctx->dctx = ZSTD_createDCtx(); + assert(ctx->cctx != NULL); + assert(ctx->dctx != NULL); + + if(dictFileName == NULL) { + ctx->dictSize = 0; + ctx->dictBuffer = NULL; + return 0; + } + { U64 const dictFileSize = UTIL_getFileSize(dictFileName); + assert(dictFileSize != UTIL_FILESIZE_UNKNOWN); + ctx->dictSize = dictFileSize; + assert((U64)ctx->dictSize == dictFileSize); /* check overflow */ + } + ctx->dictBuffer = malloc(ctx->dictSize); + + f = fopen(dictFileName, "rb"); + + if (f==NULL) { + DISPLAY("unable to open file\n"); + freeContexts(*ctx); + return 1; + } + + if (ctx->dictSize > 64 MB || !(ctx->dictBuffer)) { + DISPLAY("dictionary too large\n"); + fclose(f); + freeContexts(*ctx); + return 1; + } + readSize = fread(ctx->dictBuffer, 1, ctx->dictSize, f); + fclose(f); + if (readSize != ctx->dictSize) { + DISPLAY("unable to read file\n"); + freeContexts(*ctx); + return 1; + } + return 0; +} + +/*-************************************ +* Optimizer Memoization Functions +**************************************/ + +/* return: new length */ +/* keep old array, will need if iter over strategy. */ +/* prunes useless params */ +static size_t sanitizeVarArray(varInds_t* varNew, const size_t varLength, const varInds_t* varArray, const ZSTD_strategy strat) { + size_t i, j = 0; + for(i = 0; i < varLength; i++) { + if( !((varArray[i] == clog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_fast) + || (varArray[i] == slog_ind && strat == ZSTD_dfast) + || (varArray[i] == tlen_ind && strat < ZSTD_btopt && strat != ZSTD_fast))) { + varNew[j] = varArray[i]; + j++; + } + } + return j; +} + +/* res should be NUM_PARAMS size */ +/* constructs varArray from paramValues_t style parameter */ +/* pass in using dict. */ +static size_t variableParams(const paramValues_t paramConstraints, varInds_t* res, const int usingDictionary) { + varInds_t i; + size_t j = 0; + for(i = 0; i < NUM_PARAMS; i++) { + if(paramConstraints.vals[i] == PARAM_UNSET) { + if(i == fadt_ind && !usingDictionary) continue; /* don't use fadt if no dictionary */ + res[j] = i; j++; + } + } + return j; +} + +/* length of memo table given free variables */ +static size_t memoTableLen(const varInds_t* varyParams, const size_t varyLen) { + size_t arrayLen = 1; + size_t i; + for(i = 0; i < varyLen; i++) { + if(varyParams[i] == strt_ind) continue; /* strategy separated by table */ + arrayLen *= rangetable[varyParams[i]]; + } + return arrayLen; +} + +/* returns unique index in memotable of compression parameters */ +static unsigned memoTableIndDirect(const paramValues_t* ptr, const varInds_t* varyParams, const size_t varyLen) { + size_t i; + unsigned ind = 0; + for(i = 0; i < varyLen; i++) { + varInds_t v = varyParams[i]; + if(v == strt_ind) continue; /* exclude strategy from memotable */ + ind *= rangetable[v]; ind += (unsigned)invRangeMap(v, ptr->vals[v]); + } + return ind; +} + +static size_t memoTableGet(const memoTable_t* memoTableArray, const paramValues_t p) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + return mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)]; + case xxhashMap: + return mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen]; + case noMemo: + return 0; + } + return 0; /* should never happen, stop compiler warnings */ +} + +static void memoTableSet(const memoTable_t* memoTableArray, const paramValues_t p, const BYTE value) { + const memoTable_t mt = memoTableArray[p.vals[strt_ind]]; + switch(mt.tableType) { + case directMap: + mt.table[memoTableIndDirect(&p, mt.varArray, mt.varLen)] = value; break; + case xxhashMap: + mt.table[(XXH64(&p.vals, sizeof(U32) * NUM_PARAMS, 0) >> 3) % mt.tableLen] = value; break; + case noMemo: + break; + } +} + +/* frees all allocated memotables */ +/* secret contract : + * mtAll is a table of (ZSTD_STRATEGY_MAX+1) memoTable_t */ +static void freeMemoTableArray(memoTable_t* const mtAll) { + int i; + if(mtAll == NULL) { return; } + for(i = 1; i <= (int)ZSTD_STRATEGY_MAX; i++) { + free(mtAll[i].table); + } + free(mtAll); +} + +/* inits memotables for all (including mallocs), all strategies */ +/* takes unsanitized varyParams */ +static memoTable_t* +createMemoTableArray(const paramValues_t p, + const varInds_t* const varyParams, + const size_t varyLen, + const U32 memoTableLog) +{ + memoTable_t* const mtAll = (memoTable_t*)calloc(sizeof(memoTable_t),(ZSTD_STRATEGY_MAX + 1)); + ZSTD_strategy i, stratMin = ZSTD_STRATEGY_MIN, stratMax = ZSTD_STRATEGY_MAX; + + if(mtAll == NULL) { + return NULL; + } + + for(i = 1; i <= (int)ZSTD_STRATEGY_MAX; i++) { + mtAll[i].varLen = sanitizeVarArray(mtAll[i].varArray, varyLen, varyParams, i); + } + + /* no memoization */ + if(memoTableLog == 0) { + for(i = 1; i <= (int)ZSTD_STRATEGY_MAX; i++) { + mtAll[i].tableType = noMemo; + mtAll[i].table = NULL; + mtAll[i].tableLen = 0; + } + return mtAll; + } + + + if(p.vals[strt_ind] != PARAM_UNSET) { + stratMin = p.vals[strt_ind]; + stratMax = p.vals[strt_ind]; + } + + + for(i = stratMin; i <= stratMax; i++) { + size_t mtl = memoTableLen(mtAll[i].varArray, mtAll[i].varLen); + mtAll[i].tableType = directMap; + + if(memoTableLog != PARAM_UNSET && mtl > (1ULL << memoTableLog)) { /* use hash table */ /* provide some option to only use hash tables? */ + mtAll[i].tableType = xxhashMap; + mtl = (1ULL << memoTableLog); + } + + mtAll[i].table = (BYTE*)calloc(sizeof(BYTE), mtl); + mtAll[i].tableLen = mtl; + + if(mtAll[i].table == NULL) { + freeMemoTableArray(mtAll); + return NULL; + } + } + + return mtAll; +} + +/* Sets pc to random unmeasured set of parameters */ +/* specify strategy */ +static void randomConstrainedParams(paramValues_t* pc, const memoTable_t* memoTableArray, const ZSTD_strategy st) +{ + size_t j; + const memoTable_t mt = memoTableArray[st]; + pc->vals[strt_ind] = st; + for(j = 0; j < mt.tableLen; j++) { + int i; + for(i = 0; i < NUM_PARAMS; i++) { + varInds_t v = mt.varArray[i]; + if(v == strt_ind) continue; + pc->vals[v] = rangeMap(v, FUZ_rand(&g_rand) % rangetable[v]); + } + + if(!(memoTableGet(memoTableArray, *pc))) break; /* only pick unpicked params. */ + } +} + +/*-************************************ +* Benchmarking Functions +**************************************/ + +static void display_params_tested(paramValues_t cParams) +{ + varInds_t vi; + DISPLAYLEVEL(3, "\r testing :"); + for (vi=0; vi < NUM_PARAMS; vi++) { + DISPLAYLEVEL(3, "%3u,", (unsigned)cParams.vals[vi]); + } + DISPLAYLEVEL(3, "\b \r"); +} + +/* Replicate functionality of benchMemAdvanced, but with pre-split src / dst buffers */ +/* The purpose is so that sufficient information is returned so that a decompression call to benchMemInvertible is possible */ +/* BMK_benchMemAdvanced(srcBuffer,srcSize, dstBuffer, dstSize, fileSizes, nbFiles, 0, &cParams, dictBuffer, dictSize, ctx, dctx, 0, "File", &adv); */ +/* nbSeconds used in same way as in BMK_advancedParams_t */ +/* if in decodeOnly, then srcPtr's will be compressed blocks, and uncompressedBlocks will be written to dstPtrs */ +/* dictionary nullable, nothing else though. */ +/* note : it would be a lot better if this function was present in benchzstd.c, + * sharing code with benchMemAdvanced(), since it's technically a part of it */ +static BMK_benchOutcome_t +BMK_benchMemInvertible( buffers_t buf, contexts_t ctx, + int cLevel, const paramValues_t* comprParams, + BMK_mode_t mode, unsigned nbSeconds) +{ + U32 i; + BMK_benchResult_t bResult; + const void *const *const srcPtrs = (const void *const *const)buf.srcPtrs; + size_t const *const srcSizes = buf.srcSizes; + void** const dstPtrs = buf.dstPtrs; + size_t const *const dstCapacities = buf.dstCapacities; + size_t* const dstSizes = buf.dstSizes; + void** const resPtrs = buf.resPtrs; + size_t const *const resSizes = buf.resSizes; + const void* dictBuffer = ctx.dictBuffer; + const size_t dictBufferSize = ctx.dictSize; + const size_t nbBlocks = buf.nbBlocks; + const size_t srcSize = buf.srcSize; + ZSTD_CCtx* cctx = ctx.cctx; + ZSTD_DCtx* dctx = ctx.dctx; + + /* init */ + display_params_tested(*comprParams); + memset(&bResult, 0, sizeof(bResult)); + + /* warming up memory */ + for (i = 0; i < buf.nbBlocks; i++) { + if (mode != BMK_decodeOnly) { + RDG_genBuffer(dstPtrs[i], dstCapacities[i], 0.10, 0.50, 1); + } else { + RDG_genBuffer(resPtrs[i], resSizes[i], 0.10, 0.50, 1); + } + } + + /* Bench */ + { + /* init args */ + int compressionCompleted = (mode == BMK_decodeOnly); + int decompressionCompleted = (mode == BMK_compressOnly); + BMK_timedFnState_t* timeStateCompress = BMK_createTimedFnState(nbSeconds * 1000, 1000); + BMK_timedFnState_t* timeStateDecompress = BMK_createTimedFnState(nbSeconds * 1000, 1000); + BMK_benchParams_t cbp, dbp; + BMK_initCCtxArgs cctxprep; + BMK_initDCtxArgs dctxprep; + + cbp.benchFn = local_defaultCompress; + cbp.benchPayload = cctx; + cbp.initFn = local_initCCtx; + cbp.initPayload = &cctxprep; + cbp.errorFn = ZSTD_isError; + cbp.blockCount = nbBlocks; + cbp.srcBuffers = srcPtrs; + cbp.srcSizes = srcSizes; + cbp.dstBuffers = dstPtrs; + cbp.dstCapacities = dstCapacities; + cbp.blockResults = dstSizes; + + cctxprep.cctx = cctx; + cctxprep.dictBuffer = dictBuffer; + cctxprep.dictBufferSize = dictBufferSize; + cctxprep.cLevel = cLevel; + cctxprep.comprParams = comprParams; + + dbp.benchFn = local_defaultDecompress; + dbp.benchPayload = dctx; + dbp.initFn = local_initDCtx; + dbp.initPayload = &dctxprep; + dbp.errorFn = ZSTD_isError; + dbp.blockCount = nbBlocks; + dbp.srcBuffers = (const void* const *) dstPtrs; + dbp.srcSizes = dstCapacities; + dbp.dstBuffers = resPtrs; + dbp.dstCapacities = resSizes; + dbp.blockResults = NULL; + + dctxprep.dctx = dctx; + dctxprep.dictBuffer = dictBuffer; + dctxprep.dictBufferSize = dictBufferSize; + + assert(timeStateCompress != NULL); + assert(timeStateDecompress != NULL); + while(!compressionCompleted) { + BMK_runOutcome_t const cOutcome = BMK_benchTimedFn(timeStateCompress, cbp); + + if (!BMK_isSuccessful_runOutcome(cOutcome)) { + BMK_benchOutcome_t bOut; + memset(&bOut, 0, sizeof(bOut)); + bOut.tag = 1; /* should rather be a function or a constant */ + BMK_freeTimedFnState(timeStateCompress); + BMK_freeTimedFnState(timeStateDecompress); + return bOut; + } + { BMK_runTime_t const rResult = BMK_extract_runTime(cOutcome); + bResult.cSpeed = (unsigned long long)((double)srcSize * TIMELOOP_NANOSEC / rResult.nanoSecPerRun); + bResult.cSize = rResult.sumOfReturn; + } + compressionCompleted = BMK_isCompleted_TimedFn(timeStateCompress); + } + + while (!decompressionCompleted) { + BMK_runOutcome_t const dOutcome = BMK_benchTimedFn(timeStateDecompress, dbp); + + if (!BMK_isSuccessful_runOutcome(dOutcome)) { + BMK_benchOutcome_t bOut; + memset(&bOut, 0, sizeof(bOut)); + bOut.tag = 1; /* should rather be a function or a constant */ + BMK_freeTimedFnState(timeStateCompress); + BMK_freeTimedFnState(timeStateDecompress); + return bOut; + } + { BMK_runTime_t const rResult = BMK_extract_runTime(dOutcome); + bResult.dSpeed = (unsigned long long)((double)srcSize * TIMELOOP_NANOSEC / rResult.nanoSecPerRun); + } + decompressionCompleted = BMK_isCompleted_TimedFn(timeStateDecompress); + } + + BMK_freeTimedFnState(timeStateCompress); + BMK_freeTimedFnState(timeStateDecompress); + } + + /* Bench */ + bResult.cMem = (1 << (comprParams->vals[wlog_ind])) + ZSTD_sizeof_CCtx(cctx); + + { BMK_benchOutcome_t bOut; + bOut.tag = 0; + bOut.internal_never_use_directly = bResult; /* should be a function */ + return bOut; + } +} + +/* BMK_benchParam() : + * benchmark a set of `cParams` over sample `buf`, + * store the result in `resultPtr`. + * @return : 0 if success, 1 if error */ +static int BMK_benchParam ( BMK_benchResult_t* resultPtr, + buffers_t buf, contexts_t ctx, + paramValues_t cParams) +{ + BMK_benchOutcome_t const outcome = BMK_benchMemInvertible(buf, ctx, + BASE_CLEVEL, &cParams, + BMK_both, 3); + if (!BMK_isSuccessful_benchOutcome(outcome)) return 1; + *resultPtr = BMK_extract_benchResult(outcome); + return 0; +} + + +/* Benchmarking which stops when we are sufficiently sure the solution is infeasible / worse than the winner */ +#define VARIANCE 1.2 +static int allBench(BMK_benchResult_t* resultPtr, + const buffers_t buf, const contexts_t ctx, + const paramValues_t cParams, + const constraint_t target, + BMK_benchResult_t* winnerResult, int feas) +{ + BMK_benchResult_t benchres; + double uncertaintyConstantC = 3., uncertaintyConstantD = 3.; + double winnerRS; + + BMK_benchOutcome_t const outcome = BMK_benchMemInvertible(buf, ctx, BASE_CLEVEL, &cParams, BMK_both, 2); + if (!BMK_isSuccessful_benchOutcome(outcome)) { + DEBUGOUTPUT("Benchmarking failed \n"); + return ERROR_RESULT; + } + benchres = BMK_extract_benchResult(outcome); + + winnerRS = resultScore(*winnerResult, buf.srcSize, target); + DEBUGOUTPUT("WinnerScore: %f \n ", winnerRS); + + *resultPtr = benchres; + + /* anything with worse ratio in feas is definitely worse, discard */ + if(feas && benchres.cSize < winnerResult->cSize && !g_optmode) { + return WORSE_RESULT; + } + + /* calculate uncertainty in compression / decompression runs */ + if (benchres.cSpeed) { + U64 const loopDurationC = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.cSpeed); + uncertaintyConstantC = ((loopDurationC + (double)(2 * g_clockGranularity))/loopDurationC); + } + + if (benchres.dSpeed) { + U64 const loopDurationD = (((U64)buf.srcSize * TIMELOOP_NANOSEC) / benchres.dSpeed); + uncertaintyConstantD = ((loopDurationD + (double)(2 * g_clockGranularity))/loopDurationD); + } + + /* optimistic assumption of benchres */ + { BMK_benchResult_t resultMax = benchres; + resultMax.cSpeed = (unsigned long long)(resultMax.cSpeed * uncertaintyConstantC * VARIANCE); + resultMax.dSpeed = (unsigned long long)(resultMax.dSpeed * uncertaintyConstantD * VARIANCE); + + /* disregard infeasible results in feas mode */ + /* disregard if resultMax < winner in infeas mode */ + if((feas && !feasible(resultMax, target)) || + (!feas && (winnerRS > resultScore(resultMax, buf.srcSize, target)))) { + return WORSE_RESULT; + } + } + + /* compare by resultScore when in infeas */ + /* compare by compareResultLT when in feas */ + if((!feas && (resultScore(benchres, buf.srcSize, target) > resultScore(*winnerResult, buf.srcSize, target))) || + (feas && (compareResultLT(*winnerResult, benchres, target, buf.srcSize))) ) { + return BETTER_RESULT; + } else { + return WORSE_RESULT; + } +} + + +#define INFEASIBLE_THRESHOLD 200 +/* Memoized benchmarking, won't benchmark anything which has already been benchmarked before. */ +static int benchMemo(BMK_benchResult_t* resultPtr, + const buffers_t buf, const contexts_t ctx, + const paramValues_t cParams, + const constraint_t target, + BMK_benchResult_t* winnerResult, memoTable_t* const memoTableArray, + const int feas) { + static int bmcount = 0; + int res; + + if ( memoTableGet(memoTableArray, cParams) >= INFEASIBLE_THRESHOLD + || redundantParams(cParams, target, buf.maxBlockSize) ) { + return WORSE_RESULT; + } + + res = allBench(resultPtr, buf, ctx, cParams, target, winnerResult, feas); + + if(DEBUG && !(bmcount % 250)) { + DISPLAY("Count: %d\n", bmcount); + bmcount++; + } + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, *resultPtr, cParams, target, buf.srcSize); + + if(res == BETTER_RESULT || feas) { + memoTableSet(memoTableArray, cParams, 255); /* what happens if collisions are frequent */ + } + return res; +} + + +typedef struct { + U64 cSpeed_min; + U64 dSpeed_min; + U32 windowLog_max; + ZSTD_strategy strategy_max; +} level_constraints_t; + +static level_constraints_t g_level_constraint[NB_LEVELS_TRACKED+1]; + +static void BMK_init_level_constraints(int bytePerSec_level1) +{ + assert(NB_LEVELS_TRACKED >= ZSTD_maxCLevel()); + memset(g_level_constraint, 0, sizeof(g_level_constraint)); + g_level_constraint[1].cSpeed_min = bytePerSec_level1; + g_level_constraint[1].dSpeed_min = 0.; + g_level_constraint[1].windowLog_max = 19; + g_level_constraint[1].strategy_max = ZSTD_fast; + + /* establish speed objectives (relative to level 1) */ + { int l; + for (l=2; l<=NB_LEVELS_TRACKED; l++) { + g_level_constraint[l].cSpeed_min = (g_level_constraint[l-1].cSpeed_min * 49) / 64; + g_level_constraint[l].dSpeed_min = 0.; + g_level_constraint[l].windowLog_max = (l<20) ? 23 : l+5; /* only --ultra levels >= 20 can use windowlog > 23 */ + g_level_constraint[l].strategy_max = ZSTD_STRATEGY_MAX; + } } +} + +static int BMK_seed(winnerInfo_t* winners, + const paramValues_t params, + const buffers_t buf, + const contexts_t ctx) +{ + BMK_benchResult_t testResult; + int better = 0; + int cLevel; + + BMK_benchParam(&testResult, buf, ctx, params); + + for (cLevel = 1; cLevel <= NB_LEVELS_TRACKED; cLevel++) { + + if (testResult.cSpeed < g_level_constraint[cLevel].cSpeed_min) + continue; /* not fast enough for this level */ + if (testResult.dSpeed < g_level_constraint[cLevel].dSpeed_min) + continue; /* not fast enough for this level */ + if (params.vals[wlog_ind] > g_level_constraint[cLevel].windowLog_max) + continue; /* too much memory for this level */ + if (params.vals[strt_ind] > g_level_constraint[cLevel].strategy_max) + continue; /* forbidden strategy for this level */ + if (winners[cLevel].result.cSize==0) { + /* first solution for this cLevel */ + winners[cLevel].result = testResult; + winners[cLevel].params = params; + BMK_print_cLevelEntry(stdout, cLevel, params, testResult, buf.srcSize); + better = 1; + continue; + } + + if ((double)testResult.cSize <= ((double)winners[cLevel].result.cSize * (1. + (0.02 / cLevel))) ) { + /* Validate solution is "good enough" */ + double W_ratio = (double)buf.srcSize / testResult.cSize; + double O_ratio = (double)buf.srcSize / winners[cLevel].result.cSize; + double W_ratioNote = log (W_ratio); + double O_ratioNote = log (O_ratio); + size_t W_DMemUsed = (1 << params.vals[wlog_ind]) + (16 KB); + size_t O_DMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + (16 KB); + double W_DMemUsed_note = W_ratioNote * ( 40 + 9*cLevel) - log((double)W_DMemUsed); + double O_DMemUsed_note = O_ratioNote * ( 40 + 9*cLevel) - log((double)O_DMemUsed); + + size_t W_CMemUsed = (1 << params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(params)); + size_t O_CMemUsed = (1 << winners[cLevel].params.vals[wlog_ind]) + ZSTD_estimateCCtxSize_usingCParams(pvalsToCParams(winners[cLevel].params)); + double W_CMemUsed_note = W_ratioNote * ( 50 + 13*cLevel) - log((double)W_CMemUsed); + double O_CMemUsed_note = O_ratioNote * ( 50 + 13*cLevel) - log((double)O_CMemUsed); + + double W_CSpeed_note = W_ratioNote * ( 30 + 10*cLevel) + log(testResult.cSpeed); + double O_CSpeed_note = O_ratioNote * ( 30 + 10*cLevel) + log(winners[cLevel].result.cSpeed); + + double W_DSpeed_note = W_ratioNote * ( 20 + 2*cLevel) + log(testResult.dSpeed); + double O_DSpeed_note = O_ratioNote * ( 20 + 2*cLevel) + log(winners[cLevel].result.dSpeed); + + if (W_DMemUsed_note < O_DMemUsed_note) { + /* uses too much Decompression memory for too little benefit */ + if (W_ratio > O_ratio) + DISPLAYLEVEL(3, "Decompression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", + W_ratio, (double)(W_DMemUsed) / 1024 / 1024, + O_ratio, (double)(O_DMemUsed) / 1024 / 1024, cLevel); + continue; + } + if (W_CMemUsed_note < O_CMemUsed_note) { + /* uses too much memory for compression for too little benefit */ + if (W_ratio > O_ratio) + DISPLAYLEVEL(3, "Compression Memory : %5.3f @ %4.1f MB vs %5.3f @ %4.1f MB : not enough for level %i\n", + W_ratio, (double)(W_CMemUsed) / 1024 / 1024, + O_ratio, (double)(O_CMemUsed) / 1024 / 1024, + cLevel); + continue; + } + if (W_CSpeed_note < O_CSpeed_note ) { + /* too large compression speed difference for the compression benefit */ + if (W_ratio > O_ratio) + DISPLAYLEVEL(3, "Compression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", + W_ratio, (double)testResult.cSpeed / MB_UNIT, + O_ratio, (double)winners[cLevel].result.cSpeed / MB_UNIT, + cLevel); + continue; + } + if (W_DSpeed_note < O_DSpeed_note ) { + /* too large decompression speed difference for the compression benefit */ + if (W_ratio > O_ratio) + DISPLAYLEVEL(3, "Decompression Speed : %5.3f @ %4.1f MB/s vs %5.3f @ %4.1f MB/s : not enough for level %i\n", + W_ratio, (double)testResult.dSpeed / MB_UNIT, + O_ratio, (double)winners[cLevel].result.dSpeed / MB_UNIT, + cLevel); + continue; + } + + if (W_ratio < O_ratio) + DISPLAYLEVEL(3, "Solution %4.3f selected over %4.3f at level %i, due to better secondary statistics \n", + W_ratio, O_ratio, cLevel); + + winners[cLevel].result = testResult; + winners[cLevel].params = params; + BMK_print_cLevelEntry(stdout, cLevel, params, testResult, buf.srcSize); + + better = 1; + } } + + return better; +} + +/*-************************************ +* Compression Level Table Generation Functions +**************************************/ + +#define PARAMTABLELOG 25 +#define PARAMTABLESIZE (1<<PARAMTABLELOG) +#define PARAMTABLEMASK (PARAMTABLESIZE-1) +static BYTE g_alreadyTested[PARAMTABLESIZE] = {0}; /* init to zero */ + +static BYTE* NB_TESTS_PLAYED(paramValues_t p) +{ + ZSTD_compressionParameters const cParams = pvalsToCParams(sanitizeParams(p)); + unsigned long long const h64 = XXH64(&cParams, sizeof(cParams), 0); + return &g_alreadyTested[(h64 >> 3) & PARAMTABLEMASK]; +} + +static void playAround(FILE* f, + winnerInfo_t* winners, + paramValues_t p, + const buffers_t buf, const contexts_t ctx) +{ + int nbVariations = 0; + UTIL_time_t const clockStart = UTIL_getTime(); + + while (UTIL_clockSpanMicro(clockStart) < g_maxVariationTime) { + if (nbVariations++ > g_maxNbVariations) break; + + do { + int i; + for(i = 0; i < 4; i++) { + paramVaryOnce(FUZ_rand(&g_rand) % (strt_ind + 1), + ((FUZ_rand(&g_rand) & 1) << 1) - 1, + &p); + } + } while (!paramValid(p)); + + /* exclude faster if already played params */ + if (FUZ_rand(&g_rand) & ((1 << *NB_TESTS_PLAYED(p))-1)) + continue; + + /* test */ + { BYTE* const b = NB_TESTS_PLAYED(p); + (*b)++; + } + if (!BMK_seed(winners, p, buf, ctx)) continue; + + /* improvement found => search more */ + BMK_saveAndPrint_cLevelTable(f, winners, buf.srcSize); + playAround(f, winners, p, buf, ctx); + } + +} + +static void +BMK_selectRandomStart( FILE* f, + winnerInfo_t* winners, + const buffers_t buf, const contexts_t ctx) +{ + U32 const id = FUZ_rand(&g_rand) % (NB_LEVELS_TRACKED+1); + if ((id==0) || (winners[id].params.vals[wlog_ind]==0)) { + /* use some random entry */ + paramValues_t const p = adjustParams(cParamsToPVals(pvalsToCParams(randomParams())), /* defaults nonCompression parameters */ + buf.srcSize, 0); + playAround(f, winners, p, buf, ctx); + } else { + playAround(f, winners, winners[id].params, buf, ctx); + } +} + + +/* BMK_generate_cLevelTable() : + * test a large number of configurations + * and distribute them across compression levels according to speed conditions. + * display and save all intermediate results into rfName = "grillResults.txt". + * the function automatically stops after g_timeLimit_s. + * this function cannot error, it directly exit() in case of problem. + */ +static void BMK_generate_cLevelTable(const buffers_t buf, const contexts_t ctx) +{ + paramValues_t params; + winnerInfo_t winners[NB_LEVELS_TRACKED+1]; + const char* const rfName = "grillResults.txt"; + FILE* const f = fopen(rfName, "w"); + + /* init */ + assert(g_singleRun==0); + memset(winners, 0, sizeof(winners)); + if (f==NULL) { DISPLAY("error opening %s \n", rfName); exit(1); } + + if (g_target) { + BMK_init_level_constraints(g_target * MB_UNIT); + } else { + /* baseline config for level 1 */ + paramValues_t const l1params = cParamsToPVals(ZSTD_getCParams(1, buf.maxBlockSize, ctx.dictSize)); + BMK_benchResult_t testResult; + BMK_benchParam(&testResult, buf, ctx, l1params); + BMK_init_level_constraints((int)((testResult.cSpeed * 31) / 32)); + } + + /* populate initial solution */ + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + int i; + for (i=0; i<=maxSeeds; i++) { + params = cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, 0)); + BMK_seed(winners, params, buf, ctx); + } } + BMK_saveAndPrint_cLevelTable(f, winners, buf.srcSize); + + /* start tests */ + { const UTIL_time_t grillStart = UTIL_getTime(); + do { + BMK_selectRandomStart(f, winners, buf, ctx); + } while (BMK_timeSpan_s(grillStart) < g_timeLimit_s); + } + + /* end summary */ + BMK_saveAndPrint_cLevelTable(f, winners, buf.srcSize); + DISPLAY("grillParams operations completed \n"); + + /* clean up*/ + fclose(f); +} + + +/*-************************************ +* Single Benchmark Functions +**************************************/ + +static int +benchOnce(const buffers_t buf, const contexts_t ctx, const int cLevel) +{ + BMK_benchResult_t testResult; + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevel, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); + + if (BMK_benchParam(&testResult, buf, ctx, g_params)) { + DISPLAY("Error during benchmarking\n"); + return 1; + } + + BMK_printWinner(stdout, CUSTOM_LEVEL, testResult, g_params, buf.srcSize); + + return 0; +} + +static int benchSample(double compressibility, int cLevel) +{ + const char* const name = "Sample 10MB"; + size_t const benchedSize = 10 MB; + void* const srcBuffer = malloc(benchedSize); + int ret = 0; + + buffers_t buf; + contexts_t ctx; + + if(srcBuffer == NULL) { + DISPLAY("Out of Memory\n"); + return 2; + } + + RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); + + if(createBuffersFromMemory(&buf, srcBuffer, 1, &benchedSize)) { + DISPLAY("Buffer Creation Error\n"); + free(srcBuffer); + return 3; + } + + if(createContexts(&ctx, NULL)) { + DISPLAY("Context Creation Error\n"); + freeBuffers(buf); + return 1; + } + + /* bench */ + DISPLAY("\r%79s\r", ""); + DISPLAY("using %s %i%%: \n", name, (int)(compressibility*100)); + + if(g_singleRun) { + ret = benchOnce(buf, ctx, cLevel); + } else { + BMK_generate_cLevelTable(buf, ctx); + } + + freeBuffers(buf); + freeContexts(ctx); + + return ret; +} + +/* benchFiles() : + * note: while this function takes a table of filenames, + * in practice, only the first filename will be used */ +static int benchFiles(const char** fileNamesTable, int nbFiles, + const char* dictFileName, int cLevel) +{ + buffers_t buf; + contexts_t ctx; + int ret = 0; + + if (createBuffers(&buf, fileNamesTable, nbFiles)) { + DISPLAY("unable to load files\n"); + return 1; + } + + if (createContexts(&ctx, dictFileName)) { + DISPLAY("unable to load dictionary\n"); + freeBuffers(buf); + return 2; + } + + DISPLAY("\r%79s\r", ""); + if (nbFiles == 1) { + DISPLAY("using %s : \n", fileNamesTable[0]); + } else { + DISPLAY("using %d Files : \n", nbFiles); + } + + if (g_singleRun) { + ret = benchOnce(buf, ctx, cLevel); + } else { + BMK_generate_cLevelTable(buf, ctx); + } + + freeBuffers(buf); + freeContexts(ctx); + return ret; +} + + +/*-************************************ +* Local Optimization Functions +**************************************/ + +/* One iteration of hill climbing. Specifically, it first tries all + * valid parameter configurations w/ manhattan distance 1 and picks the best one + * failing that, it progressively tries candidates further and further away (up to #dim + 2) + * if it finds a candidate exceeding winnerInfo, it will repeat. Otherwise, it will stop the + * current stage of hill climbing. + * Each iteration of hill climbing proceeds in 2 'phases'. Phase 1 climbs according to + * the resultScore function, which is effectively a linear increase in reward until it reaches + * the constraint-satisfying value, it which point any excess results in only logarithmic reward. + * This aims to find some constraint-satisfying point. + * Phase 2 optimizes in accordance with what the original function sets out to maximize, with + * all feasible solutions valued over all infeasible solutions. + */ + +/* sanitize all params here. + * all generation after random should be sanitized. (maybe sanitize random) + */ +static winnerInfo_t climbOnce(const constraint_t target, + memoTable_t* mtAll, + const buffers_t buf, const contexts_t ctx, + const paramValues_t init) +{ + /* + * cparam - currently considered 'center' + * candidate - params to benchmark/results + * winner - best option found so far. + */ + paramValues_t cparam = init; + winnerInfo_t candidateInfo, winnerInfo; + int better = 1; + int feas = 0; + + winnerInfo = initWinnerInfo(init); + candidateInfo = winnerInfo; + + { winnerInfo_t bestFeasible1 = initWinnerInfo(cparam); + DEBUGOUTPUT("Climb Part 1\n"); + while(better) { + int offset; + size_t i, dist; + const size_t varLen = mtAll[cparam.vals[strt_ind]].varLen; + better = 0; + DEBUGOUTPUT("Start\n"); + cparam = winnerInfo.params; + candidateInfo.params = cparam; + /* all dist-1 candidates */ + for (i = 0; i < varLen; i++) { + for (offset = -1; offset <= 1; offset += 2) { + CHECKTIME(winnerInfo); + candidateInfo.params = cparam; + paramVaryOnce(mtAll[cparam.vals[strt_ind]].varArray[i], + offset, + &candidateInfo.params); + + if(paramValid(candidateInfo.params)) { + int res; + res = benchMemo(&candidateInfo.result, buf, ctx, + sanitizeParams(candidateInfo.params), target, &winnerInfo.result, mtAll, feas); + DEBUGOUTPUT("Res: %d\n", res); + if(res == BETTER_RESULT) { /* synonymous with better when called w/ infeasibleBM */ + winnerInfo = candidateInfo; + better = 1; + if(compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { + bestFeasible1 = winnerInfo; + } + } + } + } /* for (offset = -1; offset <= 1; offset += 2) */ + } /* for (i = 0; i < varLen; i++) */ + + if(better) { + continue; + } + + for (dist = 2; dist < varLen + 2; dist++) { /* varLen is # dimensions */ + for (i = 0; i < (1 << varLen) / varLen + 2; i++) { + int res; + CHECKTIME(winnerInfo); + candidateInfo.params = cparam; + /* param error checking already done here */ + paramVariation(&candidateInfo.params, mtAll, (U32)dist); + + res = benchMemo(&candidateInfo.result, + buf, ctx, + sanitizeParams(candidateInfo.params), target, + &winnerInfo.result, mtAll, feas); + DEBUGOUTPUT("Res: %d\n", res); + if (res == BETTER_RESULT) { /* synonymous with better in this case*/ + winnerInfo = candidateInfo; + better = 1; + if (compareResultLT(bestFeasible1.result, winnerInfo.result, target, buf.srcSize)) { + bestFeasible1 = winnerInfo; + } + break; + } + } + + if (better) { + break; + } + } /* for(dist = 2; dist < varLen + 2; dist++) */ + + if (!better) { /* infeas -> feas -> stop */ + if (feas) return winnerInfo; + feas = 1; + better = 1; + winnerInfo = bestFeasible1; /* note with change, bestFeasible may not necessarily be feasible, but if one has been benchmarked, it will be. */ + DEBUGOUTPUT("Climb Part 2\n"); + } + } + winnerInfo = bestFeasible1; + } + + return winnerInfo; +} + +/* Optimizes for a fixed strategy */ + +/* flexible parameters: iterations of failed climbing (or if we do non-random, maybe this is when everything is close to visited) + weight more on visit for bad results, less on good results/more on later results / ones with more failures. + allocate memoTable here. + */ +static winnerInfo_t +optimizeFixedStrategy(const buffers_t buf, const contexts_t ctx, + const constraint_t target, paramValues_t paramTarget, + const ZSTD_strategy strat, + memoTable_t* memoTableArray, const int tries) +{ + int i = 0; + + paramValues_t init; + winnerInfo_t winnerInfo, candidateInfo; + winnerInfo = initWinnerInfo(emptyParams()); + /* so climb is given the right fixed strategy */ + paramTarget.vals[strt_ind] = strat; + /* to pass ZSTD_checkCParams */ + paramTarget = cParamUnsetMin(paramTarget); + + init = paramTarget; + + for(i = 0; i < tries; i++) { + DEBUGOUTPUT("Restart\n"); + do { + randomConstrainedParams(&init, memoTableArray, strat); + } while(redundantParams(init, target, buf.maxBlockSize)); + candidateInfo = climbOnce(target, memoTableArray, buf, ctx, init); + if (compareResultLT(winnerInfo.result, candidateInfo.result, target, buf.srcSize)) { + winnerInfo = candidateInfo; + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winnerInfo.result, winnerInfo.params, target, buf.srcSize); + i = 0; + continue; + } + CHECKTIME(winnerInfo); + i++; + } + return winnerInfo; +} + +/* goes best, best-1, best+1, best-2, ... */ +/* return 0 if nothing remaining */ +static int nextStrategy(const int currentStrategy, const int bestStrategy) +{ + if(bestStrategy <= currentStrategy) { + int candidate = 2 * bestStrategy - currentStrategy - 1; + if(candidate < 1) { + candidate = currentStrategy + 1; + if(candidate > (int)ZSTD_STRATEGY_MAX) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } else { /* bestStrategy >= currentStrategy */ + int candidate = 2 * bestStrategy - currentStrategy; + if(candidate > (int)ZSTD_STRATEGY_MAX) { + candidate = currentStrategy - 1; + if(candidate < 1) { + return 0; + } else { + return candidate; + } + } else { + return candidate; + } + } +} + +/* experiment with playing with this and decay value */ + +/* main fn called when using --optimize */ +/* Does strategy selection by benchmarking default compression levels + * then optimizes by strategy, starting with the best one and moving + * progressively moving further away by number + * args: + * fileNamesTable - list of files to benchmark + * nbFiles - length of fileNamesTable + * dictFileName - name of dictionary file if one, else NULL + * target - performance constraints (cSpeed, dSpeed, cMem) + * paramTarget - parameter constraints (i.e. restriction search space to where strategy = ZSTD_fast) + * cLevel - compression level to exceed (all solutions must be > lvl in cSpeed + ratio) + */ + +static unsigned g_maxTries = 5; +#define TRY_DECAY 1 + +static int +optimizeForSize(const char* const * const fileNamesTable, const size_t nbFiles, + const char* dictFileName, + constraint_t target, paramValues_t paramTarget, + const int cLevelOpt, const int cLevelRun, + const U32 memoTableLog) +{ + varInds_t varArray [NUM_PARAMS]; + int ret = 0; + const size_t varLen = variableParams(paramTarget, varArray, dictFileName != NULL); + winnerInfo_t winner = initWinnerInfo(emptyParams()); + memoTable_t* allMT = NULL; + paramValues_t paramBase; + contexts_t ctx; + buffers_t buf; + g_time = UTIL_getTime(); + + if (createBuffers(&buf, fileNamesTable, nbFiles)) { + DISPLAY("unable to load files\n"); + return 1; + } + + if (createContexts(&ctx, dictFileName)) { + DISPLAY("unable to load dictionary\n"); + freeBuffers(buf); + return 2; + } + + if (nbFiles == 1) { + DISPLAYLEVEL(2, "Loading %s... \r", fileNamesTable[0]); + } else { + DISPLAYLEVEL(2, "Loading %lu Files... \r", (unsigned long)nbFiles); + } + + /* sanitize paramTarget */ + optimizerAdjustInput(¶mTarget, buf.maxBlockSize); + paramBase = cParamUnsetMin(paramTarget); + + allMT = createMemoTableArray(paramTarget, varArray, varLen, memoTableLog); + + if (!allMT) { + DISPLAY("MemoTable Init Error\n"); + ret = 2; + goto _cleanUp; + } + + /* default strictnesses */ + if (g_strictness == PARAM_UNSET) { + if(g_optmode) { + g_strictness = 100; + } else { + g_strictness = 90; + } + } else { + if(0 >= g_strictness || g_strictness > 100) { + DISPLAY("Strictness Outside of Bounds\n"); + ret = 4; + goto _cleanUp; + } + } + + /* use level'ing mode instead of normal target mode */ + if (g_optmode) { + winner.params = cParamsToPVals(ZSTD_getCParams(cLevelOpt, buf.maxBlockSize, ctx.dictSize)); + if(BMK_benchParam(&winner.result, buf, ctx, winner.params)) { + ret = 3; + goto _cleanUp; + } + + g_lvltarget = winner.result; + g_lvltarget.cSpeed = (g_lvltarget.cSpeed * g_strictness) / 100; + g_lvltarget.dSpeed = (g_lvltarget.dSpeed * g_strictness) / 100; + g_lvltarget.cSize = (g_lvltarget.cSize * 100) / g_strictness; + + target.cSpeed = (U32)g_lvltarget.cSpeed; + target.dSpeed = (U32)g_lvltarget.dSpeed; + + BMK_printWinnerOpt(stdout, cLevelOpt, winner.result, winner.params, target, buf.srcSize); + } + + /* Don't want it to return anything worse than the best known result */ + if (g_singleRun) { + BMK_benchResult_t res; + g_params = adjustParams(overwriteParams(cParamsToPVals(ZSTD_getCParams(cLevelRun, buf.maxBlockSize, ctx.dictSize)), g_params), buf.maxBlockSize, ctx.dictSize); + if (BMK_benchParam(&res, buf, ctx, g_params)) { + ret = 45; + goto _cleanUp; + } + if(compareResultLT(winner.result, res, relaxTarget(target), buf.srcSize)) { + winner.result = res; + winner.params = g_params; + } + } + + /* bench */ + DISPLAYLEVEL(2, "\r%79s\r", ""); + if(nbFiles == 1) { + DISPLAYLEVEL(2, "optimizing for %s", fileNamesTable[0]); + } else { + DISPLAYLEVEL(2, "optimizing for %lu Files", (unsigned long)nbFiles); + } + + if(target.cSpeed != 0) { DISPLAYLEVEL(2," - limit compression speed %u MB/s", (unsigned)(target.cSpeed >> 20)); } + if(target.dSpeed != 0) { DISPLAYLEVEL(2, " - limit decompression speed %u MB/s", (unsigned)(target.dSpeed >> 20)); } + if(target.cMem != (U32)-1) { DISPLAYLEVEL(2, " - limit memory %u MB", (unsigned)(target.cMem >> 20)); } + + DISPLAYLEVEL(2, "\n"); + init_clockGranularity(); + + { paramValues_t CParams; + + /* find best solution from default params */ + { const int maxSeeds = g_noSeed ? 1 : ZSTD_maxCLevel(); + DEBUGOUTPUT("Strategy Selection\n"); + if (paramTarget.vals[strt_ind] == PARAM_UNSET) { + BMK_benchResult_t candidate; + int i; + for (i=1; i<=maxSeeds; i++) { + int ec; + CParams = overwriteParams(cParamsToPVals(ZSTD_getCParams(i, buf.maxBlockSize, ctx.dictSize)), paramTarget); + ec = BMK_benchParam(&candidate, buf, ctx, CParams); + BMK_printWinnerOpt(stdout, i, candidate, CParams, target, buf.srcSize); + + if(!ec && compareResultLT(winner.result, candidate, relaxTarget(target), buf.srcSize)) { + winner.result = candidate; + winner.params = CParams; + } + + CHECKTIMEGT(ret, 0, _displayCleanUp); /* if pass time limit, stop */ + /* if the current params are too slow, just stop. */ + if(target.cSpeed > candidate.cSpeed * 3 / 2) { break; } + } + + BMK_printWinnerOpt(stdout, CUSTOM_LEVEL, winner.result, winner.params, target, buf.srcSize); + } + } + + DEBUGOUTPUT("Real Opt\n"); + /* start 'real' optimization */ + { int bestStrategy = (int)winner.params.vals[strt_ind]; + if (paramTarget.vals[strt_ind] == PARAM_UNSET) { + int st = bestStrategy; + int tries = g_maxTries; + + /* one iterations of hill climbing with the level-defined parameters. */ + { winnerInfo_t const w1 = climbOnce(target, allMT, buf, ctx, winner.params); + if (compareResultLT(winner.result, w1.result, target, buf.srcSize)) { + winner = w1; + } + CHECKTIMEGT(ret, 0, _displayCleanUp); + } + + while(st && tries > 0) { + winnerInfo_t wc; + DEBUGOUTPUT("StrategySwitch: %s\n", g_stratName[st]); + + wc = optimizeFixedStrategy(buf, ctx, target, paramBase, st, allMT, tries); + + if(compareResultLT(winner.result, wc.result, target, buf.srcSize)) { + winner = wc; + tries = g_maxTries; + bestStrategy = st; + } else { + st = nextStrategy(st, bestStrategy); + tries -= TRY_DECAY; + } + CHECKTIMEGT(ret, 0, _displayCleanUp); + } + } else { + winner = optimizeFixedStrategy(buf, ctx, target, paramBase, paramTarget.vals[strt_ind], allMT, g_maxTries); + } + + } + + /* no solution found */ + if(winner.result.cSize == (size_t)-1) { + ret = 1; + DISPLAY("No feasible solution found\n"); + goto _cleanUp; + } + + /* end summary */ +_displayCleanUp: + if (g_displayLevel >= 0) { + BMK_displayOneResult(stdout, winner, buf.srcSize); + } + BMK_paramValues_into_commandLine(stdout, winner.params); + DISPLAYLEVEL(1, "grillParams size - optimizer completed \n"); + } + +_cleanUp: + freeContexts(ctx); + freeBuffers(buf); + freeMemoTableArray(allMT); + return ret; +} + +/*-************************************ +* CLI parsing functions +**************************************/ + +/** longCommandWArg() : + * check if *stringPtr is the same as longCommand. + * If yes, @return 1 and advances *stringPtr to the position which immediately follows longCommand. + * @return 0 and doesn't modify *stringPtr otherwise. + * from zstdcli.c + */ +static int longCommandWArg(const char** stringPtr, const char* longCommand) +{ + size_t const comSize = strlen(longCommand); + int const result = !strncmp(*stringPtr, longCommand, comSize); + if (result) *stringPtr += comSize; + return result; +} + +static void errorOut(const char* msg) +{ + DISPLAY("%s \n", msg); exit(1); +} + +/*! readU32FromChar() : + * @return : unsigned integer value read from input in `char` format. + * allows and interprets K, KB, KiB, M, MB and MiB suffix. + * Will also modify `*stringPtr`, advancing it to position where it stopped reading. + * Note : function will exit() program if digit sequence overflows */ +static unsigned readU32FromChar(const char** stringPtr) +{ + const char errorMsg[] = "error: numeric value too large"; + unsigned sign = 1; + unsigned result = 0; + if(**stringPtr == '-') { sign = (unsigned)-1; (*stringPtr)++; } + while ((**stringPtr >='0') && (**stringPtr <='9')) { + unsigned const max = (((unsigned)(-1)) / 10) - 1; + if (result > max) errorOut(errorMsg); + result *= 10; + assert(**stringPtr >= '0'); + result += (unsigned)(**stringPtr - '0'); + (*stringPtr)++ ; + } + if ((**stringPtr=='K') || (**stringPtr=='M')) { + unsigned const maxK = ((unsigned)(-1)) >> 10; + if (result > maxK) errorOut(errorMsg); + result <<= 10; + if (**stringPtr=='M') { + if (result > maxK) errorOut(errorMsg); + result <<= 10; + } + (*stringPtr)++; /* skip `K` or `M` */ + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result * sign; +} + +static double readDoubleFromChar(const char** stringPtr) +{ + double result = 0, divide = 10; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + } + if(**stringPtr!='.') { + return result; + } + (*stringPtr)++; + while ((**stringPtr >='0') && (**stringPtr <='9')) { + result += (double)(**stringPtr - '0') / divide, divide *= 10, (*stringPtr)++ ; + } + return result; +} + +static int usage(const char* exename) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] file\n", exename); + DISPLAY( "Arguments :\n"); + DISPLAY( " file : path to the file used as reference (if none, generates a compressible sample)\n"); + DISPLAY( " -H/-h : Help (this text + advanced options)\n"); + return 0; +} + +static int usage_advanced(void) +{ + DISPLAY( "\nAdvanced options :\n"); + DISPLAY( " -T# : set level 1 speed objective \n"); + DISPLAY( " -B# : cut input into blocks of size # (default : single block) \n"); + DISPLAY( " --optimize= : same as -O with more verbose syntax (see README.md)\n"); + DISPLAY( " -S : Single run \n"); + DISPLAY( " --zstd : Single run, parameter selection same as zstdcli \n"); + DISPLAY( " -P# : generated sample compressibility (default : %.1f%%) \n", COMPRESSIBILITY_DEFAULT * 100); + DISPLAY( " -t# : Caps runtime of operation in seconds (default : %u seconds (%.1f hours)) \n", + (unsigned)g_timeLimit_s, (double)g_timeLimit_s / 3600); + DISPLAY( " -v : Prints Benchmarking output\n"); + DISPLAY( " -D : Next argument dictionary file\n"); + DISPLAY( " -s : Seperate Files\n"); + return 0; +} + +static int badusage(const char* exename) +{ + DISPLAY("Wrong parameters\n"); + usage(exename); + return 1; +} + +#define PARSE_SUB_ARGS(stringLong, stringShort, variable) { \ + if ( longCommandWArg(&argument, stringLong) \ + || longCommandWArg(&argument, stringShort) ) { \ + variable = readU32FromChar(&argument); \ + if (argument[0]==',') { \ + argument++; continue; \ + } else break; \ +} } + +/* 1 if successful parse, 0 otherwise */ +static int parse_params(const char** argptr, paramValues_t* pv) { + int matched = 0; + const char* argOrig = *argptr; + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if ( longCommandWArg(argptr,g_shortParamNames[v]) + || longCommandWArg(argptr, g_paramNames[v]) ) { + if(**argptr == '=') { + (*argptr)++; + pv->vals[v] = readU32FromChar(argptr); + matched = 1; + break; + } + } + /* reset and try again */ + *argptr = argOrig; + } + return matched; +} + +/*-************************************ +* Main +**************************************/ + +int main(int argc, const char** argv) +{ + int i, + filenamesStart=0, + result; + const char* exename=argv[0]; + const char* input_filename = NULL; + const char* dictFileName = NULL; + U32 main_pause = 0; + int cLevelOpt = 0, cLevelRun = 0; + int seperateFiles = 0; + double compressibility = COMPRESSIBILITY_DEFAULT; + U32 memoTableLog = PARAM_UNSET; + constraint_t target = { 0, 0, (U32)-1 }; + + paramValues_t paramTarget = emptyParams(); + g_params = emptyParams(); + + assert(argc>=1); /* for exename */ + + for(i=1; i<argc; i++) { + const char* argument = argv[i]; + DEBUGOUTPUT("%d: %s\n", i, argument); + assert(argument != NULL); + + if(!strcmp(argument,"--no-seed")) { g_noSeed = 1; continue; } + + if (longCommandWArg(&argument, "--optimize=")) { + g_optimizer = 1; + for ( ; ;) { + if(parse_params(&argument, ¶mTarget)) { if(argument[0] == ',') { argument++; continue; } else break; } + PARSE_SUB_ARGS("compressionSpeed=" , "cSpeed=", target.cSpeed); + PARSE_SUB_ARGS("decompressionSpeed=", "dSpeed=", target.dSpeed); + PARSE_SUB_ARGS("compressionMemory=" , "cMem=", target.cMem); + PARSE_SUB_ARGS("strict=", "stc=", g_strictness); + PARSE_SUB_ARGS("maxTries=", "tries=", g_maxTries); + PARSE_SUB_ARGS("memoLimitLog=", "memLog=", memoTableLog); + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelOpt = (int)readU32FromChar(&argument); g_optmode = 1; if (argument[0]==',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "speedForRatio=") || longCommandWArg(&argument, "speedRatio=")) { g_ratioMultiplier = readDoubleFromChar(&argument); if (argument[0]==',') { argument++; continue; } else break; } + + DISPLAY("invalid optimization parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --optimize= format\n"); + return 1; /* check the end of string */ + } + continue; + } else if (longCommandWArg(&argument, "--zstd=")) { + /* Decode command (note : aggregated commands are allowed) */ + g_singleRun = 1; + for ( ; ;) { + if(parse_params(&argument, &g_params)) { if(argument[0] == ',') { argument++; continue; } else break; } + if (longCommandWArg(&argument, "level=") || longCommandWArg(&argument, "lvl=")) { cLevelRun = (int)readU32FromChar(&argument); g_params = emptyParams(); if (argument[0]==',') { argument++; continue; } else break; } + + DISPLAY("invalid compression parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --zstd= format\n"); + return 1; /* check the end of string */ + } + continue; + /* if not return, success */ + + } else if (longCommandWArg(&argument, "--display=")) { + /* Decode command (note : aggregated commands are allowed) */ + memset(g_silenceParams, 1, sizeof(g_silenceParams)); + for ( ; ;) { + int found = 0; + varInds_t v; + for(v = 0; v < NUM_PARAMS; v++) { + if(longCommandWArg(&argument, g_shortParamNames[v]) || longCommandWArg(&argument, g_paramNames[v])) { + g_silenceParams[v] = 0; + found = 1; + } + } + if(longCommandWArg(&argument, "compressionParameters") || longCommandWArg(&argument, "cParams")) { + for(v = 0; v <= strt_ind; v++) { + g_silenceParams[v] = 0; + } + found = 1; + } + + + if(found) { + if(argument[0]==',') { + continue; + } else { + break; + } + } + DISPLAY("invalid parameter name parameter \n"); + return 1; + } + + if (argument[0] != 0) { + DISPLAY("invalid --display format\n"); + return 1; /* check the end of string */ + } + continue; + } else if (argument[0]=='-') { + argument++; + + while (argument[0]!=0) { + + switch(argument[0]) + { + /* Display help on usage */ + case 'h' : + case 'H': usage(exename); usage_advanced(); return 0; + + /* Pause at the end (hidden option) */ + case 'p': main_pause = 1; argument++; break; + + /* Sample compressibility (when no file provided) */ + case 'P': + argument++; + { U32 const proba32 = readU32FromChar(&argument); + compressibility = (double)proba32 / 100.; + } + break; + + /* Run Single conf */ + case 'S': + g_singleRun = 1; + argument++; + for ( ; ; ) { + switch(*argument) + { + case 'w': + argument++; + g_params.vals[wlog_ind] = readU32FromChar(&argument); + continue; + case 'c': + argument++; + g_params.vals[clog_ind] = readU32FromChar(&argument); + continue; + case 'h': + argument++; + g_params.vals[hlog_ind] = readU32FromChar(&argument); + continue; + case 's': + argument++; + g_params.vals[slog_ind] = readU32FromChar(&argument); + continue; + case 'l': /* search length */ + argument++; + g_params.vals[mml_ind] = readU32FromChar(&argument); + continue; + case 't': /* target length */ + argument++; + g_params.vals[tlen_ind] = readU32FromChar(&argument); + continue; + case 'S': /* strategy */ + argument++; + g_params.vals[strt_ind] = readU32FromChar(&argument); + continue; + case 'f': /* forceAttachDict */ + argument++; + g_params.vals[fadt_ind] = readU32FromChar(&argument); + continue; + case 'L': + { argument++; + cLevelRun = (int)readU32FromChar(&argument); + g_params = emptyParams(); + continue; + } + default : ; + } + break; + } + + break; + + /* target level1 speed objective, in MB/s */ + case 'T': + argument++; + g_target = readU32FromChar(&argument); + break; + + /* cut input into blocks */ + case 'B': + argument++; + g_blockSize = readU32FromChar(&argument); + DISPLAY("using %u KB block size \n", (unsigned)(g_blockSize>>10)); + break; + + /* caps runtime (in seconds) */ + case 't': + argument++; + g_timeLimit_s = readU32FromChar(&argument); + break; + + case 's': + argument++; + seperateFiles = 1; + break; + + case 'q': + while (argument[0] == 'q') { argument++; g_displayLevel--; } + break; + + case 'v': + while (argument[0] == 'v') { argument++; g_displayLevel++; } + break; + + /* load dictionary file (only applicable for optimizer rn) */ + case 'D': + if(i == argc - 1) { /* last argument, return error. */ + DISPLAY("Dictionary file expected but not given : %d\n", i); + return 1; + } else { + i++; + dictFileName = argv[i]; + argument += strlen(argument); + } + break; + + /* Unknown command */ + default : return badusage(exename); + } + } + continue; + } /* if (argument[0]=='-') */ + + /* first provided filename is input */ + if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } + } + + /* Welcome message */ + DISPLAYLEVEL(2, WELCOME_MESSAGE); + + if (filenamesStart==0) { + if (g_optimizer) { + DISPLAY("Optimizer Expects File\n"); + return 1; + } else { + result = benchSample(compressibility, cLevelRun); + } + } else { + if(seperateFiles) { + for(i = 0; i < argc - filenamesStart; i++) { + if (g_optimizer) { + result = optimizeForSize(argv+filenamesStart + i, 1, dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog); + if(result) { DISPLAY("Error on File %d", i); return result; } + } else { + result = benchFiles(argv+filenamesStart + i, 1, dictFileName, cLevelRun); + if(result) { DISPLAY("Error on File %d", i); return result; } + } + } + } else { + if (g_optimizer) { + assert(filenamesStart < argc); + result = optimizeForSize(argv+filenamesStart, (size_t)(argc-filenamesStart), dictFileName, target, paramTarget, cLevelOpt, cLevelRun, memoTableLog); + } else { + result = benchFiles(argv+filenamesStart, argc-filenamesStart, dictFileName, cLevelRun); + } + } + } + + if (main_pause) { int unused; printf("press enter...\n"); unused = getchar(); (void)unused; } + + return result; +} diff --git a/src/zstd/tests/playTests.sh b/src/zstd/tests/playTests.sh new file mode 100755 index 000000000..f353229be --- /dev/null +++ b/src/zstd/tests/playTests.sh @@ -0,0 +1,1381 @@ +#!/bin/sh + +set -e + +die() { + println "$@" 1>&2 + exit 1 +} + +datagen() { + "$DATAGEN_BIN" $@ +} + +zstd() { + if [ -z "$EXEC_PREFIX" ]; then + "$ZSTD_BIN" $@ + else + "$EXEC_PREFIX" "$ZSTD_BIN" $@ + fi +} + +sudoZstd() { + if [ -z "$EXEC_PREFIX" ]; then + sudo "$ZSTD_BIN" $@ + else + sudo "$EXEC_PREFIX" "$ZSTD_BIN" $@ + fi +} + +roundTripTest() { + if [ -n "$3" ]; then + cLevel="$3" + proba="$2" + else + cLevel="$2" + proba="" + fi + if [ -n "$4" ]; then + dLevel="$4" + else + dLevel="$cLevel" + fi + + rm -f tmp1 tmp2 + println "roundTripTest: datagen $1 $proba | zstd -v$cLevel | zstd -d$dLevel" + datagen $1 $proba | $MD5SUM > tmp1 + datagen $1 $proba | zstd --ultra -v$cLevel | zstd -d$dLevel | $MD5SUM > tmp2 + $DIFF -q tmp1 tmp2 +} + +fileRoundTripTest() { + if [ -n "$3" ]; then + local_c="$3" + local_p="$2" + else + local_c="$2" + local_p="" + fi + if [ -n "$4" ]; then + local_d="$4" + else + local_d="$local_c" + fi + + rm -f tmp.zst tmp.md5.1 tmp.md5.2 + println "fileRoundTripTest: datagen $1 $local_p > tmp && zstd -v$local_c -c tmp | zstd -d$local_d" + datagen $1 $local_p > tmp + < tmp $MD5SUM > tmp.md5.1 + zstd --ultra -v$local_c -c tmp | zstd -d$local_d | $MD5SUM > tmp.md5.2 + $DIFF -q tmp.md5.1 tmp.md5.2 +} + +truncateLastByte() { + dd bs=1 count=$(($(wc -c < "$1") - 1)) if="$1" +} + +println() { + printf '%b\n' "${*}" +} + + +SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) +PRGDIR="$SCRIPT_DIR/../programs" +TESTDIR="$SCRIPT_DIR/../tests" +UNAME=$(uname) +ZSTDGREP="$PRGDIR/zstdgrep" + +detectedTerminal=false +if [ -t 0 ] && [ -t 1 ] +then + detectedTerminal=true +fi +isTerminal=${isTerminal:-$detectedTerminal} + +isWindows=false +INTOVOID="/dev/null" +case "$UNAME" in + GNU) DEVDEVICE="/dev/random" ;; + *) DEVDEVICE="/dev/zero" ;; +esac +case "$OS" in + Windows*) + isWindows=true + INTOVOID="NUL" + DEVDEVICE="NUL" + ;; +esac + +case "$UNAME" in + Darwin) MD5SUM="md5 -r" ;; + FreeBSD) MD5SUM="gmd5sum" ;; + OpenBSD) MD5SUM="md5" ;; + *) MD5SUM="md5sum" ;; +esac + +MTIME="stat -c %Y" +case "$UNAME" in + Darwin | FreeBSD | OpenBSD) MTIME="stat -f %m" ;; +esac + +DIFF="diff" +case "$UNAME" in + SunOS) DIFF="gdiff" ;; +esac + +println "\nStarting playTests.sh isWindows=$isWindows EXE_PREFIX='$EXE_PREFIX' ZSTD_BIN='$ZSTD_BIN' DATAGEN_BIN='$DATAGEN_BIN'" + +[ -n "$ZSTD_BIN" ] || die "\$ZSTD_BIN variable must be defined!" +[ -n "$DATAGEN_BIN" ] || die "\$DATAGEN_BIN variable must be defined!" + +if echo hello | zstd -v -T2 2>&1 > $INTOVOID | grep -q 'multi-threading is disabled' +then + hasMT="" +else + hasMT="true" +fi + + + +println "\n===> simple tests " + +datagen > tmp +println "test : basic compression " +zstd -f tmp # trivial compression case, creates tmp.zst +println "test : basic decompression" +zstd -df tmp.zst # trivial decompression case (overwrites tmp) +println "test : too large compression level => auto-fix" +zstd -99 -f tmp # too large compression level, automatic sized down +zstd -5000000000 -f tmp && die "too large numeric value : must fail" +println "test : --fast aka negative compression levels" +zstd --fast -f tmp # == -1 +zstd --fast=3 -f tmp # == -3 +zstd --fast=200000 -f tmp # too low compression level, automatic fixed +zstd --fast=5000000000 -f tmp && die "too large numeric value : must fail" +zstd -c --fast=0 tmp > $INTOVOID && die "--fast must not accept value 0" +println "test : too large numeric argument" +zstd --fast=9999999999 -f tmp && die "should have refused numeric value" +println "test : set compression level with environment variable ZSTD_CLEVEL" +ZSTD_CLEVEL=12 zstd -f tmp # positive compression level +ZSTD_CLEVEL=-12 zstd -f tmp # negative compression level +ZSTD_CLEVEL=+12 zstd -f tmp # valid: verbose '+' sign +ZSTD_CLEVEL='' zstd -f tmp # empty env var, warn and revert to default setting +ZSTD_CLEVEL=- zstd -f tmp # malformed env var, warn and revert to default setting +ZSTD_CLEVEL=a zstd -f tmp # malformed env var, warn and revert to default setting +ZSTD_CLEVEL=+a zstd -f tmp # malformed env var, warn and revert to default setting +ZSTD_CLEVEL=3a7 zstd -f tmp # malformed env var, warn and revert to default setting +ZSTD_CLEVEL=50000000000 zstd -f tmp # numeric value too large, warn and revert to default setting +println "test : override ZSTD_CLEVEL with command line option" +ZSTD_CLEVEL=12 zstd --fast=3 -f tmp # overridden by command line option +println "test : compress to stdout" +zstd tmp -c > tmpCompressed +zstd tmp --stdout > tmpCompressed # long command format +println "test : compress to named file" +rm tmpCompressed +zstd tmp -o tmpCompressed +test -f tmpCompressed # file must be created +println "test : -o must be followed by filename (must fail)" +zstd tmp -of tmpCompressed && die "-o must be followed by filename " +println "test : force write, correct order" +zstd tmp -fo tmpCompressed +println "test : forgotten argument" +cp tmp tmp2 +zstd tmp2 -fo && die "-o must be followed by filename " +println "test : implied stdout when input is stdin" +println bob | zstd | zstd -d +if [ "$isTerminal" = true ]; then +println "test : compressed data to terminal" +println bob | zstd && die "should have refused : compressed data to terminal" +println "test : compressed data from terminal (a hang here is a test fail, zstd is wrongly waiting on data from terminal)" +zstd -d > $INTOVOID && die "should have refused : compressed data from terminal" +fi +println "test : null-length file roundtrip" +println -n '' | zstd - --stdout | zstd -d --stdout +println "test : ensure small file doesn't add 3-bytes null block" +datagen -g1 > tmp1 +zstd tmp1 -c | wc -c | grep "14" +zstd < tmp1 | wc -c | grep "14" +println "test : decompress file with wrong suffix (must fail)" +zstd -d tmpCompressed && die "wrong suffix error not detected!" +zstd -df tmp && die "should have refused : wrong extension" +println "test : decompress into stdout" +zstd -d tmpCompressed -c > tmpResult # decompression using stdout +zstd --decompress tmpCompressed -c > tmpResult +zstd --decompress tmpCompressed --stdout > tmpResult +println "test : decompress from stdin into stdout" +zstd -dc < tmp.zst > $INTOVOID # combine decompression, stdin & stdout +zstd -dc - < tmp.zst > $INTOVOID +zstd -d < tmp.zst > $INTOVOID # implicit stdout when stdin is used +zstd -d - < tmp.zst > $INTOVOID +println "test : impose memory limitation (must fail)" +zstd -d -f tmp.zst -M2K -c > $INTOVOID && die "decompression needs more memory than allowed" +zstd -d -f tmp.zst --memlimit=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command +zstd -d -f tmp.zst --memory=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command +zstd -d -f tmp.zst --memlimit-decompress=2K -c > $INTOVOID && die "decompression needs more memory than allowed" # long command +println "test : overwrite protection" +zstd -q tmp && die "overwrite check failed!" +println "test : force overwrite" +zstd -q -f tmp +zstd -q --force tmp +println "test : overwrite readonly file" +rm -f tmpro tmpro.zst +println foo > tmpro.zst +println foo > tmpro +chmod 400 tmpro.zst +zstd -q tmpro && die "should have refused to overwrite read-only file" +zstd -q -f tmpro +println "test: --no-progress flag" +zstd tmpro -c --no-progress | zstd -d -f -o "$INTOVOID" --no-progress +zstd tmpro -cv --no-progress | zstd -dv -f -o "$INTOVOID" --no-progress +rm -f tmpro tmpro.zst +println "test: overwrite input file (must fail)" +zstd tmp -fo tmp && die "zstd compression overwrote the input file" +zstd tmp.zst -dfo tmp.zst && die "zstd decompression overwrote the input file" +println "test: detect that input file does not exist" +zstd nothere && die "zstd hasn't detected that input file does not exist" +println "test: --[no-]compress-literals" +zstd tmp -c --no-compress-literals -1 | zstd -t +zstd tmp -c --no-compress-literals --fast=1 | zstd -t +zstd tmp -c --no-compress-literals -19 | zstd -t +zstd tmp -c --compress-literals -1 | zstd -t +zstd tmp -c --compress-literals --fast=1 | zstd -t +zstd tmp -c --compress-literals -19 | zstd -t +zstd -b --fast=1 -i0e1 tmp --compress-literals +zstd -b --fast=1 -i0e1 tmp --no-compress-literals + +println "\n===> zstdgrep tests" +ln -sf "$ZSTD_BIN" zstdcat +rm -f tmp_grep +echo "1234" > tmp_grep +zstd -f tmp_grep +lines=$(ZCAT=./zstdcat $ZSTDGREP 2>&1 "1234" tmp_grep tmp_grep.zst | wc -l) +test 2 -eq $lines +ZCAT=./zstdcat $ZSTDGREP 2>&1 "1234" tmp_grep_bad.zst && die "Should have failed" +ZCAT=./zstdcat $ZSTDGREP 2>&1 "1234" tmp_grep_bad.zst | grep "No such file or directory" || true +rm -f tmp_grep* + +println "\n===> --exclude-compressed flag" +rm -rf precompressedFilterTestDir +mkdir -p precompressedFilterTestDir +datagen $size > precompressedFilterTestDir/input.5 +datagen $size > precompressedFilterTestDir/input.6 +zstd --exclude-compressed --long --rm -r precompressedFilterTestDir +datagen $size > precompressedFilterTestDir/input.7 +datagen $size > precompressedFilterTestDir/input.8 +zstd --exclude-compressed --long --rm -r precompressedFilterTestDir +test ! -f precompressedFilterTestDir/input.5.zst.zst +test ! -f precompressedFilterTestDir/input.6.zst.zst +file1timestamp=`$MTIME precompressedFilterTestDir/input.5.zst` +file2timestamp=`$MTIME precompressedFilterTestDir/input.7.zst` +if [ $file2timestamp -ge $file1timestamp ]; then + println "Test is successful. input.5.zst is precompressed and therefore not compressed/modified again." +else + println "Test is not successful" +fi +# File Extension check. +datagen $size > precompressedFilterTestDir/input.zstbar +zstd --exclude-compressed --long --rm -r precompressedFilterTestDir +# zstd should compress input.zstbar +test -f precompressedFilterTestDir/input.zstbar.zst +# Check without the --exclude-compressed flag +zstd --long --rm -r precompressedFilterTestDir +# Files should get compressed again without the --exclude-compressed flag. +test -f precompressedFilterTestDir/input.5.zst.zst +test -f precompressedFilterTestDir/input.6.zst.zst +println "Test completed" + + +println "\n===> recursive mode test " +# combination of -r with empty list of input file +zstd -c -r < tmp > tmp.zst + + +println "\n===> file removal" +zstd -f --rm tmp +test ! -f tmp # tmp should no longer be present +zstd -f -d --rm tmp.zst +test ! -f tmp.zst # tmp.zst should no longer be present +println "test : should quietly not remove non-regular file" +println hello > tmp +zstd tmp -f -o "$DEVDEVICE" 2>tmplog > "$INTOVOID" +grep -v "Refusing to remove non-regular file" tmplog +rm -f tmplog +zstd tmp -f -o "$INTOVOID" 2>&1 | grep -v "Refusing to remove non-regular file" +println "test : --rm on stdin" +println a | zstd --rm > $INTOVOID # --rm should remain silent +rm tmp +zstd -f tmp && die "tmp not present : should have failed" +test ! -f tmp.zst # tmp.zst should not be created +println "test : -d -f do not delete destination when source is not present" +touch tmp # create destination file +zstd -d -f tmp.zst && die "attempt to decompress a non existing file" +test -f tmp # destination file should still be present +println "test : -f do not delete destination when source is not present" +rm tmp # erase source file +touch tmp.zst # create destination file +zstd -f tmp && die "attempt to compress a non existing file" +test -f tmp.zst # destination file should still be present +rm -rf tmp* # may also erase tmp* directory from previous failed run + + +println "\n===> decompression only tests " +# the following test verifies that the decoder is compatible with RLE as first block +# older versions of zstd cli are not able to decode such corner case. +# As a consequence, the zstd cli do not generate them, to maintain compatibility with older versions. +dd bs=1048576 count=1 if=/dev/zero of=tmp +zstd -d -o tmp1 "$TESTDIR/golden-decompression/rle-first-block.zst" +$DIFF -s tmp1 tmp +rm tmp* + + +println "\n===> compress multiple files" +println hello > tmp1 +println world > tmp2 +zstd tmp1 tmp2 -o "$INTOVOID" -f +zstd tmp1 tmp2 -c | zstd -t +zstd tmp1 tmp2 -o tmp.zst +test ! -f tmp1.zst +test ! -f tmp2.zst +zstd tmp1 tmp2 +zstd -t tmp1.zst tmp2.zst +zstd -dc tmp1.zst tmp2.zst +zstd tmp1.zst tmp2.zst -o "$INTOVOID" -f +zstd -d tmp1.zst tmp2.zst -o tmp +touch tmpexists +zstd tmp1 tmp2 -f -o tmpexists +zstd tmp1 tmp2 -o tmpexists && die "should have refused to overwrite" +# Bug: PR #972 +if [ "$?" -eq 139 ]; then + die "should not have segfaulted" +fi +println "\n===> multiple files and shell completion " +datagen -s1 > tmp1 2> $INTOVOID +datagen -s2 -g100K > tmp2 2> $INTOVOID +datagen -s3 -g1M > tmp3 2> $INTOVOID +println "compress tmp* : " +zstd -f tmp* +test -f tmp1.zst +test -f tmp2.zst +test -f tmp3.zst +rm tmp1 tmp2 tmp3 +println "decompress tmp* : " +zstd -df ./*.zst +test -f tmp1 +test -f tmp2 +test -f tmp3 +println "compress tmp* into stdout > tmpall : " +zstd -c tmp1 tmp2 tmp3 > tmpall +test -f tmpall # should check size of tmpall (should be tmp1.zst + tmp2.zst + tmp3.zst) +println "decompress tmpall* into stdout > tmpdec : " +cp tmpall tmpall2 +zstd -dc tmpall* > tmpdec +test -f tmpdec # should check size of tmpdec (should be 2*(tmp1 + tmp2 + tmp3)) +println "compress multiple files including a missing one (notHere) : " +zstd -f tmp1 notHere tmp2 && die "missing file not detected!" +rm tmp* + + +if [ "$isWindows" = false ] ; then + println "\n===> zstd fifo named pipe test " + echo "Hello World!" > tmp_original + mkfifo tmp_named_pipe + # note : fifo test doesn't work in combination with `dd` or `cat` + echo "Hello World!" > tmp_named_pipe & + zstd tmp_named_pipe -o tmp_compressed + zstd -d -o tmp_decompressed tmp_compressed + $DIFF -s tmp_original tmp_decompressed + rm -rf tmp* +fi + + +if [ -n "$DEVNULLRIGHTS" ] ; then + # these tests requires sudo rights, which is uncommon. + # they are only triggered if DEVNULLRIGHTS macro is defined. + println "\n===> checking /dev/null permissions are unaltered " + datagen > tmp + sudoZstd tmp -o $INTOVOID # sudo rights could modify /dev/null permissions + sudoZstd tmp -c > $INTOVOID + zstd tmp -f -o tmp.zst + sudoZstd -d tmp.zst -c > $INTOVOID + sudoZstd -d tmp.zst -o $INTOVOID + ls -las $INTOVOID | grep "rw-rw-rw-" +fi + + +println "\n===> compress multiple files into an output directory, --output-dir-flat" +println henlo > tmp1 +mkdir tmpInputTestDir +mkdir tmpInputTestDir/we +mkdir tmpInputTestDir/we/must +mkdir tmpInputTestDir/we/must/go +mkdir tmpInputTestDir/we/must/go/deeper +println cool > tmpInputTestDir/we/must/go/deeper/tmp2 +mkdir tmpOutDir +zstd tmp1 tmpInputTestDir/we/must/go/deeper/tmp2 --output-dir-flat tmpOutDir +test -f tmpOutDir/tmp1.zst +test -f tmpOutDir/tmp2.zst +println "test : decompress multiple files into an output directory, --output-dir-flat" +mkdir tmpOutDirDecomp +zstd tmpOutDir -r -d --output-dir-flat tmpOutDirDecomp +test -f tmpOutDirDecomp/tmp2 +test -f tmpOutDirDecomp/tmp1 +rm -f tmpOutDirDecomp/* +zstd tmpOutDir -r -d --output-dir-flat=tmpOutDirDecomp +test -f tmpOutDirDecomp/tmp2 +test -f tmpOutDirDecomp/tmp1 +rm -rf tmp* + + +println "test : compress multiple files reading them from a file, --filelist=FILE" +println "Hello world!, file1" > tmp1 +println "Hello world!, file2" > tmp2 +println tmp1 > tmp_fileList +println tmp2 >> tmp_fileList +zstd -f --filelist=tmp_fileList +test -f tmp2.zst +test -f tmp1.zst + +println "test : reading file list from a symlink, --filelist=FILE" +rm -f *.zst +ln -s tmp_fileList tmp_symLink +zstd -f --filelist=tmp_symLink +test -f tmp2.zst +test -f tmp1.zst + +println "test : compress multiple files reading them from multiple files, --filelist=FILE" +rm -f *.zst +println "Hello world!, file3" > tmp3 +println "Hello world!, file4" > tmp4 +println tmp3 > tmp_fileList2 +println tmp4 >> tmp_fileList2 +zstd -f --filelist=tmp_fileList --filelist=tmp_fileList2 +test -f tmp1.zst +test -f tmp2.zst +test -f tmp3.zst +test -f tmp4.zst + +println "test : decompress multiple files reading them from a file, --filelist=FILE" +rm -f tmp1 tmp2 +println tmp1.zst > tmpZst +println tmp2.zst >> tmpZst +zstd -d -f --filelist=tmpZst +test -f tmp1 +test -f tmp2 + +println "test : decompress multiple files reading them from multiple files, --filelist=FILE" +rm -f tmp1 tmp2 tmp3 tmp4 +println tmp3.zst > tmpZst2 +println tmp4.zst >> tmpZst2 +zstd -d -f --filelist=tmpZst --filelist=tmpZst2 +test -f tmp1 +test -f tmp2 +test -f tmp3 +test -f tmp4 + +println "test : survive a list of files which is text garbage (--filelist=FILE)" +datagen > tmp_badList +zstd -f --filelist=tmp_badList && die "should have failed : list is text garbage" + +println "test : survive a list of files which is binary garbage (--filelist=FILE)" +datagen -P0 -g1M > tmp_badList +zstd -qq -f --filelist=tmp_badList && die "should have failed : list is binary garbage" # let's avoid printing binary garbage on console + +println "test : try to overflow internal list of files (--filelist=FILE)" +touch tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 +ls tmp* > tmpList +zstd -f tmp1 --filelist=tmpList --filelist=tmpList tmp2 tmp3 # can trigger an overflow of internal file list +rm -rf tmp* + +println "\n===> --[no-]content-size tests" + +datagen > tmp_contentsize +zstd -f tmp_contentsize +zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" +zstd -f --no-content-size tmp_contentsize +zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die +zstd -f --content-size tmp_contentsize +zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" +zstd -f --content-size --no-content-size tmp_contentsize +zstd -lv tmp_contentsize.zst | grep "Decompressed Size:" && die +rm -rf tmp* + +println "test : show-default-cparams regular" +datagen > tmp +zstd --show-default-cparams -f tmp +rm -rf tmp* + +println "test : show-default-cparams recursive" +mkdir tmp_files +datagen -g15000 > tmp_files/tmp1 +datagen -g129000 > tmp_files/tmp2 +datagen -g257000 > tmp_files/tmp3 +zstd --show-default-cparams -f -r tmp_files +rm -rf tmp* + +println "\n===> Advanced compression parameters " +println "Hello world!" | zstd --zstd=windowLog=21, - -o tmp.zst && die "wrong parameters not detected!" +println "Hello world!" | zstd --zstd=windowLo=21 - -o tmp.zst && die "wrong parameters not detected!" +println "Hello world!" | zstd --zstd=windowLog=21,slog - -o tmp.zst && die "wrong parameters not detected!" +println "Hello world!" | zstd --zstd=strategy=10 - -o tmp.zst && die "parameter out of bound not detected!" # > btultra2 : does not exist +test ! -f tmp.zst # tmp.zst should not be created +roundTripTest -g512K +roundTripTest -g512K " --zstd=mml=3,tlen=48,strat=6" +roundTripTest -g512K " --zstd=strat=6,wlog=23,clog=23,hlog=22,slog=6" +roundTripTest -g512K " --zstd=windowLog=23,chainLog=23,hashLog=22,searchLog=6,minMatch=3,targetLength=48,strategy=6" +roundTripTest -g512K " --single-thread --long --zstd=ldmHashLog=20,ldmMinMatch=64,ldmBucketSizeLog=1,ldmHashRateLog=7" +roundTripTest -g512K " --single-thread --long --zstd=lhlog=20,lmml=64,lblog=1,lhrlog=7" +roundTripTest -g64K "19 --zstd=strat=9" # btultra2 + + +println "\n===> Pass-Through mode " +println "Hello world 1!" | zstd -df +println "Hello world 2!" | zstd -dcf +println "Hello world 3!" > tmp1 +zstd -dcf tmp1 + + +println "\n===> frame concatenation " +println "hello " > hello.tmp +println "world!" > world.tmp +cat hello.tmp world.tmp > helloworld.tmp +zstd -c hello.tmp > hello.zst +zstd -c world.tmp > world.zst +cat hello.zst world.zst > helloworld.zst +zstd -dc helloworld.zst > result.tmp +cat result.tmp +$DIFF helloworld.tmp result.tmp +println "frame concatenation without checksum" +zstd -c hello.tmp > hello.zst --no-check +zstd -c world.tmp > world.zst --no-check +cat hello.zst world.zst > helloworld.zstd +zstd -dc helloworld.zst > result.tmp +$DIFF helloworld.tmp result.tmp +println "testing zstdcat symlink" +ln -sf "$ZSTD_BIN" zstdcat +$EXE_PREFIX ./zstdcat helloworld.zst > result.tmp +$DIFF helloworld.tmp result.tmp +ln -s helloworld.zst helloworld.link.zst +$EXE_PREFIX ./zstdcat helloworld.link.zst > result.tmp +$DIFF helloworld.tmp result.tmp +rm zstdcat +rm result.tmp +println "testing zcat symlink" +ln -sf "$ZSTD_BIN" zcat +$EXE_PREFIX ./zcat helloworld.zst > result.tmp +$DIFF helloworld.tmp result.tmp +$EXE_PREFIX ./zcat helloworld.link.zst > result.tmp +$DIFF helloworld.tmp result.tmp +rm zcat +rm ./*.tmp ./*.zstd +println "frame concatenation tests completed" + + +if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] && [ "$UNAME" != "OpenBSD" ] ; then +println "\n**** flush write error test **** " + +println "println foo | zstd > /dev/full" +println foo | zstd > /dev/full && die "write error not detected!" +println "println foo | zstd | zstd -d > /dev/full" +println foo | zstd | zstd -d > /dev/full && die "write error not detected!" + +fi + + +if [ "$isWindows" = false ] && [ "$UNAME" != 'SunOS' ] ; then + +println "\n===> symbolic link test " + +rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst +println "hello world" > hello.tmp +ln -s hello.tmp world.tmp +ln -s hello.tmp world2.tmp +zstd world.tmp hello.tmp || true +test -f hello.tmp.zst # regular file should have been compressed! +test ! -f world.tmp.zst # symbolic link should not have been compressed! +zstd world.tmp || true +test ! -f world.tmp.zst # symbolic link should not have been compressed! +zstd world.tmp world2.tmp || true +test ! -f world.tmp.zst # symbolic link should not have been compressed! +test ! -f world2.tmp.zst # symbolic link should not have been compressed! +zstd world.tmp hello.tmp -f +test -f world.tmp.zst # symbolic link should have been compressed with --force +rm -f hello.tmp world.tmp world2.tmp hello.tmp.zst world.tmp.zst + +fi + + +println "\n===> test sparse file support " + +datagen -g5M -P100 > tmpSparse +zstd tmpSparse -c | zstd -dv -o tmpSparseRegen +$DIFF -s tmpSparse tmpSparseRegen +zstd tmpSparse -c | zstd -dv --sparse -c > tmpOutSparse +$DIFF -s tmpSparse tmpOutSparse +zstd tmpSparse -c | zstd -dv --no-sparse -c > tmpOutNoSparse +$DIFF -s tmpSparse tmpOutNoSparse +ls -ls tmpSparse* # look at file size and block size on disk +datagen -s1 -g1200007 -P100 | zstd | zstd -dv --sparse -c > tmpSparseOdd # Odd size file (to not finish on an exact nb of blocks) +datagen -s1 -g1200007 -P100 | $DIFF -s - tmpSparseOdd +ls -ls tmpSparseOdd # look at file size and block size on disk +println "\n Sparse Compatibility with Console :" +println "Hello World 1 !" | zstd | zstd -d -c +println "Hello World 2 !" | zstd | zstd -d | cat +println "\n Sparse Compatibility with Append :" +datagen -P100 -g1M > tmpSparse1M +cat tmpSparse1M tmpSparse1M > tmpSparse2M +zstd -v -f tmpSparse1M -o tmpSparseCompressed +zstd -d -v -f tmpSparseCompressed -o tmpSparseRegenerated +zstd -d -v -f tmpSparseCompressed -c >> tmpSparseRegenerated +ls -ls tmpSparse* # look at file size and block size on disk +$DIFF tmpSparse2M tmpSparseRegenerated +rm tmpSparse* + + +println "\n===> stream-size mode" + +datagen -g11000 > tmp +println "test : basic file compression vs sized streaming compression" +file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) +stream_size=$(cat tmp | zstd -14 --stream-size=11000 | wc -c) +if [ "$stream_size" -gt "$file_size" ]; then + die "hinted compression larger than expected" +fi +println "test : sized streaming compression and decompression" +cat tmp | zstd -14 -f tmp -o --stream-size=11000 tmp.zst +zstd -df tmp.zst -o tmp_decompress +cmp tmp tmp_decompress || die "difference between original and decompressed file" +println "test : incorrect stream size" +cat tmp | zstd -14 -f -o tmp.zst --stream-size=11001 && die "should fail with incorrect stream size" + +println "\n===> zstd zero weight dict test " +rm -f tmp* +cp "$TESTDIR/dict-files/zero-weight-dict" tmp_input +zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input +zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp +$DIFF tmp_decomp tmp_input +rm -rf tmp* + +println "\n===> zstd (valid) zero weight dict test " +rm -f tmp* +# 0 has a non-zero weight in the dictionary +echo "0000000000000000000000000" > tmp_input +zstd -D "$TESTDIR/dict-files/zero-weight-dict" tmp_input +zstd -D "$TESTDIR/dict-files/zero-weight-dict" -d tmp_input.zst -o tmp_decomp +$DIFF tmp_decomp tmp_input +rm -rf tmp* + +println "\n===> size-hint mode" + +datagen -g11000 > tmp +datagen -g11000 > tmp2 +datagen > tmpDict +println "test : basic file compression vs hinted streaming compression" +file_size=$(zstd -14 -f tmp -o tmp.zst && wc -c < tmp.zst) +stream_size=$(cat tmp | zstd -14 --size-hint=11000 | wc -c) +if [ "$stream_size" -ge "$file_size" ]; then + die "hinted compression larger than expected" +fi +println "test : hinted streaming compression and decompression" +cat tmp | zstd -14 -f -o tmp.zst --size-hint=11000 +zstd -df tmp.zst -o tmp_decompress +cmp tmp tmp_decompress || die "difference between original and decompressed file" +println "test : hinted streaming compression with dictionary" +cat tmp | zstd -14 -f -D tmpDict --size-hint=11000 | zstd -t -D tmpDict +println "test : multiple file compression with hints and dictionary" +zstd -14 -f -D tmpDict --size-hint=11000 tmp tmp2 +zstd -14 -f -o tmp1_.zst -D tmpDict --size-hint=11000 tmp +zstd -14 -f -o tmp2_.zst -D tmpDict --size-hint=11000 tmp2 +cmp tmp.zst tmp1_.zst || die "first file's output differs" +cmp tmp2.zst tmp2_.zst || die "second file's output differs" +println "test : incorrect hinted stream sizes" +cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high +cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low +cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high +cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low + + +println "\n===> dictionary tests " + +println "- test with raw dict (content only) " +datagen > tmpDict +datagen -g1M | $MD5SUM > tmp1 +datagen -g1M | zstd -D tmpDict | zstd -D tmpDict -dvq | $MD5SUM > tmp2 +$DIFF -q tmp1 tmp2 +println "- Create first dictionary " +TESTFILE="$PRGDIR"/zstdcli.c +zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict +cp "$TESTFILE" tmp +println "- Test dictionary compression with tmpDict as an input file and dictionary" +zstd -f tmpDict -D tmpDict && die "compression error not detected!" +println "- Dictionary compression roundtrip" +zstd -f tmp -D tmpDict +zstd -d tmp.zst -D tmpDict -fo result +$DIFF "$TESTFILE" result +println "- Dictionary compression with btlazy2 strategy" +zstd -f tmp -D tmpDict --zstd=strategy=6 +zstd -d tmp.zst -D tmpDict -fo result +$DIFF "$TESTFILE" result +if [ -n "$hasMT" ] +then + println "- Test dictionary compression with multithreading " + datagen -g5M | zstd -T2 -D tmpDict | zstd -t -D tmpDict # fails with v1.3.2 +fi +println "- Create second (different) dictionary " +zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC +zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" +println "- Create dictionary with short dictID" +zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 +cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +println "- Create dictionary with wrong dictID parameter order (must fail)" +zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID -o 1 tmpDict1 && die "wrong order : --dictID must be followed by argument " +println "- Create dictionary with size limit" +zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K -v +println "- Create dictionary with small size limit" +zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict=1K -v +println "- Create dictionary with wrong parameter order (must fail)" +zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict3 --maxdict -v 4K && die "wrong order : --maxdict must be followed by argument " +println "- Compress without dictID" +zstd -f tmp -D tmpDict1 --no-dictID +zstd -d tmp.zst -D tmpDict -fo result +$DIFF "$TESTFILE" result +println "- Compress with wrong argument order (must fail)" +zstd tmp -Df tmpDict1 -c > $INTOVOID && die "-D must be followed by dictionary name " +println "- Compress multiple files with dictionary" +rm -rf dirTestDict +mkdir dirTestDict +cp "$TESTDIR"/*.c dirTestDict +cp "$PRGDIR"/*.c dirTestDict +cp "$PRGDIR"/*.h dirTestDict +$MD5SUM dirTestDict/* > tmph1 +zstd -f --rm dirTestDict/* -D tmpDictC +zstd -d --rm dirTestDict/*.zst -D tmpDictC # note : use internal checksum by default +case "$UNAME" in + Darwin) println "md5sum -c not supported on OS-X : test skipped" ;; # not compatible with OS-X's md5 + *) $MD5SUM -c tmph1 ;; +esac +rm -rf dirTestDict +println "- dictionary builder on bogus input" +println "Hello World" > tmp +zstd --train-legacy -q tmp && die "Dictionary training should fail : not enough input source" +datagen -P0 -g10M > tmp +zstd --train-legacy -q tmp && die "Dictionary training should fail : source is pure noise" +println "- Test -o before --train" +rm -f tmpDict dictionary +zstd -o tmpDict --train "$TESTDIR"/*.c "$PRGDIR"/*.c +test -f tmpDict +zstd --train "$TESTDIR"/*.c "$PRGDIR"/*.c +test -f dictionary +println "- Test dictionary training fails" +echo "000000000000000000000000000000000" > tmpz +zstd --train tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" +if [ -n "$hasMT" ] +then + zstd --train -T0 tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz tmpz && die "Dictionary training should fail : source is all zeros" + println "- Create dictionary with multithreading enabled" + zstd --train -T0 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict +fi +rm tmp* dictionary + + +println "\n===> fastCover dictionary builder : advanced options " +TESTFILE="$PRGDIR"/zstdcli.c +datagen > tmpDict +println "- Create first dictionary" +zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict +cp "$TESTFILE" tmp +zstd -f tmp -D tmpDict +zstd -d tmp.zst -D tmpDict -fo result +$DIFF "$TESTFILE" result +println "- Create second (different) dictionary" +zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC +zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" +zstd --train-fastcover=k=56,d=8 && die "Create dictionary without input file" +println "- Create dictionary with short dictID" +zstd --train-fastcover=k=46,d=8,f=15,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 +cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +println "- Create dictionaries with shrink-dict flag enabled" +zstd --train-fastcover=steps=1,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict +zstd --train-fastcover=steps=1,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict1 +zstd --train-fastcover=steps=1,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpShrinkDict2 +println "- Create dictionary with size limit" +zstd --train-fastcover=steps=1 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K +println "- Create dictionary using all samples for both training and testing" +zstd --train-fastcover=k=56,d=8,split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +println "- Create dictionary using f=16" +zstd --train-fastcover=k=56,d=8,f=16 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +zstd --train-fastcover=k=56,d=8,accel=15 -r "$TESTDIR"/*.c "$PRGDIR"/*.c && die "Created dictionary using accel=15" +println "- Create dictionary using accel=2" +zstd --train-fastcover=k=56,d=8,accel=2 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +println "- Create dictionary using accel=10" +zstd --train-fastcover=k=56,d=8,accel=10 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +println "- Create dictionary with multithreading" +zstd --train-fastcover -T4 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +println "- Test -o before --train-fastcover" +rm -f tmpDict dictionary +zstd -o tmpDict --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c +test -f tmpDict +zstd --train-fastcover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c +test -f dictionary +rm tmp* dictionary + + +println "\n===> legacy dictionary builder " + +TESTFILE="$PRGDIR"/zstdcli.c +datagen > tmpDict +println "- Create first dictionary" +zstd --train-legacy=selectivity=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict +cp "$TESTFILE" tmp +zstd -f tmp -D tmpDict +zstd -d tmp.zst -D tmpDict -fo result +$DIFF "$TESTFILE" result +zstd --train-legacy=s=8 && die "Create dictionary without input files (should error)" +println "- Create second (different) dictionary" +zstd --train-legacy=s=5 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC +zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" +println "- Create dictionary with short dictID" +zstd --train-legacy -s5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 +cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +println "- Create dictionary with size limit" +zstd --train-legacy -s9 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K +println "- Test -o before --train-legacy" +rm -f tmpDict dictionary +zstd -o tmpDict --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c +test -f tmpDict +zstd --train-legacy "$TESTDIR"/*.c "$PRGDIR"/*.c +test -f dictionary +rm tmp* dictionary + + +println "\n===> integrity tests " + +println "test one file (tmp1.zst) " +datagen > tmp1 +zstd tmp1 +zstd -t tmp1.zst +zstd --test tmp1.zst +println "test multiple files (*.zst) " +zstd -t ./*.zst +println "test bad files (*) " +zstd -t ./* && die "bad files not detected !" +zstd -t tmp1 && die "bad file not detected !" +cp tmp1 tmp2.zst +zstd -t tmp2.zst && die "bad file not detected !" +datagen -g0 > tmp3 +zstd -t tmp3 && die "bad file not detected !" # detects 0-sized files as bad +println "test --rm and --test combined " +zstd -t --rm tmp1.zst +test -f tmp1.zst # check file is still present +split -b16384 tmp1.zst tmpSplit. +zstd -t tmpSplit.* && die "bad file not detected !" +datagen | zstd -c | zstd -t + + +println "\n===> golden files tests " + +zstd -t -r "$TESTDIR/golden-compression" +zstd -c -r "$TESTDIR/golden-compression" | zstd -t + + +println "\n===> benchmark mode tests " + +println "bench one file" +datagen > tmp1 +zstd -bi0 tmp1 +println "bench multiple levels" +zstd -i0b0e3 tmp1 +println "bench negative level" +zstd -bi0 --fast tmp1 +println "with recursive and quiet modes" +zstd -rqi0b1e2 tmp1 +println "benchmark decompression only" +zstd -f tmp1 +zstd -b -d -i0 tmp1.zst + + +println "\n===> zstd compatibility tests " + +datagen > tmp +rm -f tmp.zst +zstd --format=zstd -f tmp +test -f tmp.zst + + +println "\n===> gzip compatibility tests " + +GZIPMODE=1 +zstd --format=gzip -V || GZIPMODE=0 +if [ $GZIPMODE -eq 1 ]; then + println "gzip support detected" + GZIPEXE=1 + gzip -V || GZIPEXE=0 + if [ $GZIPEXE -eq 1 ]; then + datagen > tmp + zstd --format=gzip -f tmp + gzip -t -v tmp.gz + gzip -f tmp + zstd -d -f -v tmp.gz + rm tmp* + else + println "gzip binary not detected" + fi +else + println "gzip mode not supported" +fi + + +println "\n===> gzip frame tests " + +if [ $GZIPMODE -eq 1 ]; then + datagen > tmp + zstd -f --format=gzip tmp + zstd -f tmp + cat tmp.gz tmp.zst tmp.gz tmp.zst | zstd -d -f -o tmp + truncateLastByte tmp.gz | zstd -t > $INTOVOID && die "incomplete frame not detected !" + rm tmp* +else + println "gzip mode not supported" +fi + +if [ $GZIPMODE -eq 1 ]; then + datagen > tmp + rm -f tmp.zst + zstd --format=gzip --format=zstd -f tmp + test -f tmp.zst +fi + +println "\n===> xz compatibility tests " + +LZMAMODE=1 +zstd --format=xz -V || LZMAMODE=0 +if [ $LZMAMODE -eq 1 ]; then + println "xz support detected" + XZEXE=1 + xz -Q -V && lzma -Q -V || XZEXE=0 + if [ $XZEXE -eq 1 ]; then + println "Testing zstd xz and lzma support" + datagen > tmp + zstd --format=lzma -f tmp + zstd --format=xz -f tmp + xz -Q -t -v tmp.xz + xz -Q -t -v tmp.lzma + xz -Q -f -k tmp + lzma -Q -f -k --lzma1 tmp + zstd -d -f -v tmp.xz + zstd -d -f -v tmp.lzma + rm tmp* + println "Creating symlinks" + ln -s "$ZSTD_BIN" ./xz + ln -s "$ZSTD_BIN" ./unxz + ln -s "$ZSTD_BIN" ./lzma + ln -s "$ZSTD_BIN" ./unlzma + println "Testing xz and lzma symlinks" + datagen > tmp + ./xz tmp + xz -Q -d tmp.xz + ./lzma tmp + lzma -Q -d tmp.lzma + println "Testing unxz and unlzma symlinks" + xz -Q tmp + ./xz -d tmp.xz + lzma -Q tmp + ./lzma -d tmp.lzma + rm xz unxz lzma unlzma + rm tmp* + else + println "xz binary not detected" + fi +else + println "xz mode not supported" +fi + + +println "\n===> xz frame tests " + +if [ $LZMAMODE -eq 1 ]; then + datagen > tmp + zstd -f --format=xz tmp + zstd -f --format=lzma tmp + zstd -f tmp + cat tmp.xz tmp.lzma tmp.zst tmp.lzma tmp.xz tmp.zst | zstd -d -f -o tmp + truncateLastByte tmp.xz | zstd -t > $INTOVOID && die "incomplete frame not detected !" + truncateLastByte tmp.lzma | zstd -t > $INTOVOID && die "incomplete frame not detected !" + rm tmp* +else + println "xz mode not supported" +fi + +println "\n===> lz4 compatibility tests " + +LZ4MODE=1 +zstd --format=lz4 -V || LZ4MODE=0 +if [ $LZ4MODE -eq 1 ]; then + println "lz4 support detected" + LZ4EXE=1 + lz4 -V || LZ4EXE=0 + if [ $LZ4EXE -eq 1 ]; then + datagen > tmp + zstd --format=lz4 -f tmp + lz4 -t -v tmp.lz4 + lz4 -f tmp + zstd -d -f -v tmp.lz4 + rm tmp* + else + println "lz4 binary not detected" + fi +else + println "lz4 mode not supported" +fi + + +if [ $LZ4MODE -eq 1 ]; then + println "\n===> lz4 frame tests " + datagen > tmp + zstd -f --format=lz4 tmp + zstd -f tmp + cat tmp.lz4 tmp.zst tmp.lz4 tmp.zst | zstd -d -f -o tmp + truncateLastByte tmp.lz4 | zstd -t > $INTOVOID && die "incomplete frame not detected !" + rm tmp* +else + println "\nlz4 mode not supported" +fi + + +println "\n===> suffix list test" + +! zstd -d tmp.abc 2> tmplg + +if [ $GZIPMODE -ne 1 ]; then + grep ".gz" tmplg > $INTOVOID && die "Unsupported suffix listed" +fi + +if [ $LZMAMODE -ne 1 ]; then + grep ".lzma" tmplg > $INTOVOID && die "Unsupported suffix listed" + grep ".xz" tmplg > $INTOVOID && die "Unsupported suffix listed" +fi + +if [ $LZ4MODE -ne 1 ]; then + grep ".lz4" tmplg > $INTOVOID && die "Unsupported suffix listed" +fi + + +println "\n===> tar extension tests " + +rm -f tmp tmp.tar tmp.tzst tmp.tgz tmp.txz tmp.tlz4 + +datagen > tmp +tar cf tmp.tar tmp +zstd tmp.tar -o tmp.tzst +rm tmp.tar +zstd -d tmp.tzst +[ -e tmp.tar ] || die ".tzst failed to decompress to .tar!" +rm -f tmp.tar tmp.tzst + +if [ $GZIPMODE -eq 1 ]; then + tar czf tmp.tgz tmp + zstd -d tmp.tgz + [ -e tmp.tar ] || die ".tgz failed to decompress to .tar!" + rm -f tmp.tar tmp.tgz +fi + +if [ $LZMAMODE -eq 1 ]; then + tar c tmp | zstd --format=xz > tmp.txz + zstd -d tmp.txz + [ -e tmp.tar ] || die ".txz failed to decompress to .tar!" + rm -f tmp.tar tmp.txz +fi + +if [ $LZ4MODE -eq 1 ]; then + tar c tmp | zstd --format=lz4 > tmp.tlz4 + zstd -d tmp.tlz4 + [ -e tmp.tar ] || die ".tlz4 failed to decompress to .tar!" + rm -f tmp.tar tmp.tlz4 +fi + +touch tmp.t tmp.tz tmp.tzs +! zstd -d tmp.t +! zstd -d tmp.tz +! zstd -d tmp.tzs + + +println "\n===> zstd round-trip tests " + +roundTripTest +roundTripTest -g15K # TableID==3 +roundTripTest -g127K # TableID==2 +roundTripTest -g255K # TableID==1 +roundTripTest -g522K # TableID==0 +roundTripTest -g519K 6 # greedy, hash chain +roundTripTest -g517K 16 # btlazy2 +roundTripTest -g516K 19 # btopt + +fileRoundTripTest -g500K + +println "\n===> zstd long distance matching round-trip tests " +roundTripTest -g0 "2 --single-thread --long" +roundTripTest -g1000K "1 --single-thread --long" +roundTripTest -g517K "6 --single-thread --long" +roundTripTest -g516K "16 --single-thread --long" +roundTripTest -g518K "19 --single-thread --long" +fileRoundTripTest -g5M "3 --single-thread --long" + + +roundTripTest -g96K "5 --single-thread" +if [ -n "$hasMT" ] +then + println "\n===> zstdmt round-trip tests " + roundTripTest -g4M "1 -T0" + roundTripTest -g8M "3 -T2" + roundTripTest -g8000K "2 --threads=2" + fileRoundTripTest -g4M "19 -T2 -B1M" + + println "\n===> zstdmt long distance matching round-trip tests " + roundTripTest -g8M "3 --long=24 -T2" + + println "\n===> ovLog tests " + datagen -g2MB > tmp + refSize=$(zstd tmp -6 -c --zstd=wlog=18 | wc -c) + ov9Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=9 | wc -c) + ov1Size=$(zstd tmp -6 -c --zstd=wlog=18,ovlog=1 | wc -c) + if [ "$refSize" -eq "$ov9Size" ]; then + echo ov9Size should be different from refSize + exit 1 + fi + if [ "$refSize" -eq "$ov1Size" ]; then + echo ov1Size should be different from refSize + exit 1 + fi + if [ "$ov9Size" -ge "$ov1Size" ]; then + echo ov9Size="$ov9Size" should be smaller than ov1Size="$ov1Size" + exit 1 + fi + +else + println "\n===> no multithreading, skipping zstdmt tests " +fi + +rm tmp* + +println "\n===> zstd --list/-l single frame tests " +datagen > tmp1 +datagen > tmp2 +datagen > tmp3 +zstd tmp* +zstd -l ./*.zst +zstd -lv ./*.zst | grep "Decompressed Size:" # check that decompressed size is present in header +zstd --list ./*.zst +zstd --list -v ./*.zst + +println "\n===> zstd --list/-l multiple frame tests " +cat tmp1.zst tmp2.zst > tmp12.zst +cat tmp12.zst tmp3.zst > tmp123.zst +zstd -l ./*.zst +zstd -lv ./*.zst + +println "\n===> zstd --list/-l error detection tests " +zstd -l tmp1 tmp1.zst && die "-l must fail on non-zstd file" +zstd --list tmp* && die "-l must fail on non-zstd file" +zstd -lv tmp1* && die "-l must fail on non-zstd file" +zstd --list -v tmp2 tmp12.zst && die "-l must fail on non-zstd file" + +println "test : detect truncated compressed file " +TEST_DATA_FILE=truncatable-input.txt +FULL_COMPRESSED_FILE=${TEST_DATA_FILE}.zst +TRUNCATED_COMPRESSED_FILE=truncated-input.txt.zst +datagen -g50000 > $TEST_DATA_FILE +zstd -f $TEST_DATA_FILE -o $FULL_COMPRESSED_FILE +dd bs=1 count=100 if=$FULL_COMPRESSED_FILE of=$TRUNCATED_COMPRESSED_FILE +zstd --list $TRUNCATED_COMPRESSED_FILE && die "-l must fail on truncated file" + +rm $TEST_DATA_FILE +rm $FULL_COMPRESSED_FILE +rm $TRUNCATED_COMPRESSED_FILE + +println "\n===> zstd --list/-l errors when presented with stdin / no files" +zstd -l && die "-l must fail on empty list of files" +zstd -l - && die "-l does not work on stdin" +zstd -l < tmp1.zst && die "-l does not work on stdin" +zstd -l - < tmp1.zst && die "-l does not work on stdin" +zstd -l - tmp1.zst && die "-l does not work on stdin" +zstd -l - tmp1.zst < tmp1.zst && die "-l does not work on stdin" +zstd -l tmp1.zst < tmp2.zst # this will check tmp1.zst, but not tmp2.zst, which is not an error : zstd simply doesn't read stdin in this case. It must not error just because stdin is not a tty + +println "\n===> zstd --list/-l test with null files " +datagen -g0 > tmp5 +zstd tmp5 +zstd -l tmp5.zst +zstd -l tmp5* && die "-l must fail on non-zstd file" +zstd -lv tmp5.zst | grep "Decompressed Size: 0.00 KB (0 B)" # check that 0 size is present in header +zstd -lv tmp5* && die "-l must fail on non-zstd file" + +println "\n===> zstd --list/-l test with no content size field " +datagen -g513K | zstd > tmp6.zst +zstd -l tmp6.zst +zstd -lv tmp6.zst | grep "Decompressed Size:" && die "Field :Decompressed Size: should not be available in this compressed file" + +println "\n===> zstd --list/-l test with no checksum " +zstd -f --no-check tmp1 +zstd -l tmp1.zst +zstd -lv tmp1.zst + +rm tmp* + + +println "\n===> zstd long distance matching tests " +roundTripTest -g0 " --single-thread --long" +roundTripTest -g9M "2 --single-thread --long" +# Test parameter parsing +roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB" +roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB" +roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB" +roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB" + + + + +if [ "$1" != "--test-large-data" ]; then + println "Skipping large data tests" + exit 0 +fi + + +############################################################################# + + +if [ -n "$hasMT" ] +then + println "\n===> adaptive mode " + roundTripTest -g270000000 " --adapt" + roundTripTest -g27000000 " --adapt=min=1,max=4" + println "===> test: --adapt must fail on incoherent bounds " + datagen > tmp + zstd -f -vv --adapt=min=10,max=9 tmp && die "--adapt must fail on incoherent bounds" + + println "\n===> rsyncable mode " + roundTripTest -g10M " --rsyncable" + roundTripTest -g10M " --rsyncable -B100K" + println "===> test: --rsyncable must fail with --single-thread" + zstd -f -vv --rsyncable --single-thread tmp && die "--rsyncable must fail with --single-thread" +fi + +println "\n===> patch-from tests" + +datagen -g1000 -P50 > tmp_dict +datagen -g1000 -P10 > tmp_patch +zstd --patch-from=tmp_dict tmp_patch -o tmp_patch_diff +zstd -d --patch-from=tmp_dict tmp_patch_diff -o tmp_patch_recon +$DIFF -s tmp_patch_recon tmp_patch +rm -rf tmp_* + +println "\n===> patch-from recursive tests" + +mkdir tmp_dir +datagen > tmp_dir/tmp1 +datagen > tmp_dir/tmp2 +datagen > tmp_dict +zstd --patch-from=tmp_dict -r tmp_dir && die +rm -rf tmp* + +println "\n===> patch-from long mode trigger larger file test" + +datagen -g5000000 > tmp_dict +datagen -g5000000 > tmp_patch +zstd -15 --patch-from=tmp_dict tmp_patch 2>&1 | grep "long mode automaticaly triggered" +rm -rf tmp* + +println "\n===> large files tests " + +roundTripTest -g270000000 1 +roundTripTest -g250000000 2 +roundTripTest -g230000000 3 + +roundTripTest -g140000000 -P60 4 +roundTripTest -g130000000 -P62 5 +roundTripTest -g120000000 -P65 6 + +roundTripTest -g70000000 -P70 7 +roundTripTest -g60000000 -P71 8 +roundTripTest -g50000000 -P73 9 + +roundTripTest -g35000000 -P75 10 +roundTripTest -g30000000 -P76 11 +roundTripTest -g25000000 -P78 12 + +roundTripTest -g18000013 -P80 13 +roundTripTest -g18000014 -P80 14 +roundTripTest -g18000015 -P81 15 +roundTripTest -g18000016 -P84 16 +roundTripTest -g18000017 -P88 17 +roundTripTest -g18000018 -P94 18 +roundTripTest -g18000019 -P96 19 + +roundTripTest -g5000000000 -P99 "1 --zstd=wlog=25" +roundTripTest -g3700000000 -P0 "1 --zstd=strategy=6,wlog=25" # ensure btlazy2 can survive an overflow rescale + +fileRoundTripTest -g4193M -P99 1 + + +println "\n===> zstd long, long distance matching round-trip tests " +roundTripTest -g270000000 "1 --single-thread --long" +roundTripTest -g130000000 -P60 "5 --single-thread --long" +roundTripTest -g35000000 -P70 "8 --single-thread --long" +roundTripTest -g18000001 -P80 "18 --single-thread --long" +# Test large window logs +roundTripTest -g700M -P50 "1 --single-thread --long=29" +roundTripTest -g600M -P50 "1 --single-thread --long --zstd=wlog=29,clog=28" + + +if [ -n "$hasMT" ] +then + println "\n===> zstdmt long round-trip tests " + roundTripTest -g80000000 -P99 "19 -T2" " " + roundTripTest -g5000000000 -P99 "1 -T2" " " + roundTripTest -g500000000 -P97 "1 -T999" " " + fileRoundTripTest -g4103M -P98 " -T0" " " + roundTripTest -g400000000 -P97 "1 --long=24 -T2" " " + # Exposes the bug in https://github.com/facebook/zstd/pull/1678 + # This test fails on 4 different travis builds at the time of writing + # because it needs to allocate 8 GB of memory. + # roundTripTest -g10G -P99 "1 -T1 --long=31 --zstd=clog=27 --fast=1000" +else + println "\n**** no multithreading, skipping zstdmt tests **** " +fi + + +println "\n===> cover dictionary builder : advanced options " + +TESTFILE="$PRGDIR"/zstdcli.c +datagen > tmpDict +println "- Create first dictionary" +zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict +cp "$TESTFILE" tmp +zstd -f tmp -D tmpDict +zstd -d tmp.zst -D tmpDict -fo result +$DIFF "$TESTFILE" result +zstd --train-cover=k=56,d=8 && die "Create dictionary without input file (should error)" +println "- Create second (different) dictionary" +zstd --train-cover=k=56,d=8 "$TESTDIR"/*.c "$PRGDIR"/*.c "$PRGDIR"/*.h -o tmpDictC +zstd -d tmp.zst -D tmpDictC -fo result && die "wrong dictionary not detected!" +println "- Create dictionary using shrink-dict flag" +zstd --train-cover=steps=256,shrink "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict +zstd --train-cover=steps=256,shrink=1 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict1 +zstd --train-cover=steps=256,shrink=5 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpShrinkDict2 +println "- Create dictionary with short dictID" +zstd --train-cover=k=46,d=8,split=80 "$TESTDIR"/*.c "$PRGDIR"/*.c --dictID=1 -o tmpDict1 +cmp tmpDict tmpDict1 && die "dictionaries should have different ID !" +println "- Create dictionary with size limit" +zstd --train-cover=steps=8 "$TESTDIR"/*.c "$PRGDIR"/*.c -o tmpDict2 --maxdict=4K +println "- Compare size of dictionary from 90% training samples with 80% training samples" +zstd --train-cover=split=90 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +zstd --train-cover=split=80 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +println "- Create dictionary using all samples for both training and testing" +zstd --train-cover=split=100 -r "$TESTDIR"/*.c "$PRGDIR"/*.c +println "- Test -o before --train-cover" +rm -f tmpDict dictionary +zstd -o tmpDict --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c +test -f tmpDict +zstd --train-cover "$TESTDIR"/*.c "$PRGDIR"/*.c +test -f dictionary +rm -f tmp* dictionary + +rm -f tmp* diff --git a/src/zstd/tests/poolTests.c b/src/zstd/tests/poolTests.c new file mode 100644 index 000000000..e1576ba85 --- /dev/null +++ b/src/zstd/tests/poolTests.c @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#include "pool.h" +#include "threading.h" +#include "util.h" +#include "timefn.h" +#include <stddef.h> +#include <stdio.h> + +#define ASSERT_TRUE(p) \ + do { \ + if (!(p)) { \ + return 1; \ + } \ + } while (0) +#define ASSERT_FALSE(p) ASSERT_TRUE(!(p)) +#define ASSERT_EQ(lhs, rhs) ASSERT_TRUE((lhs) == (rhs)) + +struct data { + ZSTD_pthread_mutex_t mutex; + unsigned data[16]; + size_t i; +}; + +static void fn(void *opaque) +{ + struct data *data = (struct data *)opaque; + ZSTD_pthread_mutex_lock(&data->mutex); + data->data[data->i] = (unsigned)(data->i); + ++data->i; + ZSTD_pthread_mutex_unlock(&data->mutex); +} + +static int testOrder(size_t numThreads, size_t queueSize) +{ + struct data data; + POOL_ctx* const ctx = POOL_create(numThreads, queueSize); + ASSERT_TRUE(ctx); + data.i = 0; + ASSERT_FALSE(ZSTD_pthread_mutex_init(&data.mutex, NULL)); + { size_t i; + for (i = 0; i < 16; ++i) { + POOL_add(ctx, &fn, &data); + } + } + POOL_free(ctx); + ASSERT_EQ(16, data.i); + { size_t i; + for (i = 0; i < data.i; ++i) { + ASSERT_EQ(i, data.data[i]); + } + } + ZSTD_pthread_mutex_destroy(&data.mutex); + return 0; +} + + +/* --- test deadlocks --- */ + +static void waitFn(void *opaque) { + (void)opaque; + UTIL_sleepMilli(1); +} + +/* Tests for deadlock */ +static int testWait(size_t numThreads, size_t queueSize) { + struct data data; + POOL_ctx* const ctx = POOL_create(numThreads, queueSize); + ASSERT_TRUE(ctx); + { size_t i; + for (i = 0; i < 16; ++i) { + POOL_add(ctx, &waitFn, &data); + } + } + POOL_free(ctx); + return 0; +} + + +/* --- test POOL_resize() --- */ + +typedef struct { + ZSTD_pthread_mutex_t mut; + int countdown; + int val; + int max; + ZSTD_pthread_cond_t cond; +} poolTest_t; + +static void waitLongFn(void *opaque) { + poolTest_t* const test = (poolTest_t*) opaque; + ZSTD_pthread_mutex_lock(&test->mut); + test->val++; + if (test->val > test->max) + test->max = test->val; + ZSTD_pthread_mutex_unlock(&test->mut); + + UTIL_sleepMilli(10); + + ZSTD_pthread_mutex_lock(&test->mut); + test->val--; + test->countdown--; + if (test->countdown == 0) + ZSTD_pthread_cond_signal(&test->cond); + ZSTD_pthread_mutex_unlock(&test->mut); +} + +static int testThreadReduction_internal(POOL_ctx* ctx, poolTest_t test) +{ + int const nbWaits = 16; + + test.countdown = nbWaits; + test.val = 0; + test.max = 0; + + { int i; + for (i=0; i<nbWaits; i++) + POOL_add(ctx, &waitLongFn, &test); + } + ZSTD_pthread_mutex_lock(&test.mut); + while (test.countdown > 0) + ZSTD_pthread_cond_wait(&test.cond, &test.mut); + ASSERT_EQ(test.val, 0); + ASSERT_EQ(test.max, 4); + ZSTD_pthread_mutex_unlock(&test.mut); + + ASSERT_EQ( POOL_resize(ctx, 2/*nbThreads*/) , 0 ); + test.countdown = nbWaits; + test.val = 0; + test.max = 0; + { int i; + for (i=0; i<nbWaits; i++) + POOL_add(ctx, &waitLongFn, &test); + } + ZSTD_pthread_mutex_lock(&test.mut); + while (test.countdown > 0) + ZSTD_pthread_cond_wait(&test.cond, &test.mut); + ASSERT_EQ(test.val, 0); + ASSERT_EQ(test.max, 2); + ZSTD_pthread_mutex_unlock(&test.mut); + + return 0; +} + +static int testThreadReduction(void) { + int result; + poolTest_t test; + POOL_ctx* const ctx = POOL_create(4 /*nbThreads*/, 2 /*queueSize*/); + + ASSERT_TRUE(ctx); + + memset(&test, 0, sizeof(test)); + ASSERT_FALSE( ZSTD_pthread_mutex_init(&test.mut, NULL) ); + ASSERT_FALSE( ZSTD_pthread_cond_init(&test.cond, NULL) ); + + result = testThreadReduction_internal(ctx, test); + + ZSTD_pthread_mutex_destroy(&test.mut); + ZSTD_pthread_cond_destroy(&test.cond); + POOL_free(ctx); + + return result; +} + + +/* --- test abrupt ending --- */ + +typedef struct { + ZSTD_pthread_mutex_t mut; + int val; +} abruptEndCanary_t; + +static void waitIncFn(void *opaque) { + abruptEndCanary_t* test = (abruptEndCanary_t*) opaque; + UTIL_sleepMilli(10); + ZSTD_pthread_mutex_lock(&test->mut); + test->val = test->val + 1; + ZSTD_pthread_mutex_unlock(&test->mut); +} + +static int testAbruptEnding_internal(abruptEndCanary_t test) +{ + int const nbWaits = 16; + + POOL_ctx* const ctx = POOL_create(3 /*numThreads*/, nbWaits /*queueSize*/); + ASSERT_TRUE(ctx); + test.val = 0; + + { int i; + for (i=0; i<nbWaits; i++) + POOL_add(ctx, &waitIncFn, &test); /* all jobs pushed into queue */ + } + ASSERT_EQ( POOL_resize(ctx, 1 /*numThreads*/) , 0 ); /* downsize numThreads, to try to break end condition */ + + POOL_free(ctx); /* must finish all jobs in queue before giving back control */ + ASSERT_EQ(test.val, nbWaits); + return 0; +} + +static int testAbruptEnding(void) { + int result; + abruptEndCanary_t test; + + memset(&test, 0, sizeof(test)); + ASSERT_FALSE( ZSTD_pthread_mutex_init(&test.mut, NULL) ); + + result = testAbruptEnding_internal(test); + + ZSTD_pthread_mutex_destroy(&test.mut); + return result; +} + + + +/* --- test launcher --- */ + +int main(int argc, const char **argv) { + size_t numThreads; + (void)argc; + (void)argv; + + if (POOL_create(0, 1)) { /* should not be possible */ + printf("FAIL: should not create POOL with 0 threads\n"); + return 1; + } + + for (numThreads = 1; numThreads <= 4; ++numThreads) { + size_t queueSize; + for (queueSize = 0; queueSize <= 2; ++queueSize) { + printf("queueSize==%u, numThreads=%u \n", + (unsigned)queueSize, (unsigned)numThreads); + if (testOrder(numThreads, queueSize)) { + printf("FAIL: testOrder\n"); + return 1; + } + printf("SUCCESS: testOrder\n"); + if (testWait(numThreads, queueSize)) { + printf("FAIL: testWait\n"); + return 1; + } + printf("SUCCESS: testWait\n"); + } + } + + if (testThreadReduction()) { + printf("FAIL: thread reduction not effective \n"); + return 1; + } else { + printf("SUCCESS: thread reduction effective \n"); + } + + if (testAbruptEnding()) { + printf("FAIL: jobs in queue not completed on early end \n"); + return 1; + } else { + printf("SUCCESS: all jobs in queue completed on early end \n"); + } + + printf("PASS: all POOL tests\n"); + + return 0; +} diff --git a/src/zstd/tests/rateLimiter.py b/src/zstd/tests/rateLimiter.py new file mode 100755 index 000000000..1068c4424 --- /dev/null +++ b/src/zstd/tests/rateLimiter.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 + +# ################################################################ +# Copyright (c) 2018-2020, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ########################################################################## + +# Rate limiter, replacement for pv +# this rate limiter does not "catch up" after a blocking period +# Limitations: +# - only accepts limit speed in MB/s + +import sys +import time + +MB = 1024 * 1024 +rate = float(sys.argv[1]) * MB +start = time.time() +total_read = 0 + +# sys.stderr.close() # remove error message, for Ctrl+C + +try: + buf = " " + while len(buf): + now = time.time() + to_read = max(int(rate * (now - start)), 1) + max_buf_size = 1 * MB + to_read = min(to_read, max_buf_size) + start = now + + buf = sys.stdin.buffer.read(to_read) + sys.stdout.buffer.write(buf) + +except (KeyboardInterrupt, BrokenPipeError) as e: + pass diff --git a/src/zstd/tests/regression/.gitignore b/src/zstd/tests/regression/.gitignore new file mode 100644 index 000000000..1b2618f41 --- /dev/null +++ b/src/zstd/tests/regression/.gitignore @@ -0,0 +1,3 @@ +# regression test artifacts +data-cache +test diff --git a/src/zstd/tests/regression/Makefile b/src/zstd/tests/regression/Makefile new file mode 100644 index 000000000..87c1c2b96 --- /dev/null +++ b/src/zstd/tests/regression/Makefile @@ -0,0 +1,59 @@ +# ################################################################ +# Copyright (c) 2015-2020, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +CFLAGS ?= -O3 + +CURL_CFLAGS := $(shell curl-config --cflags) +CURL_LDFLAGS := $(shell curl-config --libs) -pthread + +PROGDIR := ../../programs +LIBDIR := ../../lib +ZSTD_CPPFLAGS := -I$(PROGDIR) -I$(LIBDIR) -I$(LIBDIR)/common + +REGRESSION_CFLAGS = $(CFLAGS) $(CURL_CFLAGS) +REGRESSION_CPPFLAGS = $(CPPFLAGS) $(ZSTD_CPPFLAGS) +REGRESSION_LDFLAGS = $(LDFLAGS) $(CURL_LDFLAGS) + +all: test + +xxhash.o: $(LIBDIR)/common/xxhash.c $(LIBDIR)/common/xxhash.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +util.o: $(PROGDIR)/util.c $(PROGDIR)/util.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +data.o: data.c data.h $(PROGDIR)/util.h $(LIBDIR)/common/xxhash.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +config.o: config.c config.h levels.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +method.h: data.h config.h result.h + +method.o: method.c method.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +result.o: result.c result.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +test.o: test.c data.h config.h method.h + $(CC) $(REGRESSION_CFLAGS) $(REGRESSION_CPPFLAGS) $< -c -o $@ + +libzstd.a: + $(MAKE) -C $(LIBDIR) libzstd.a-mt + cp $(LIBDIR)/libzstd.a . + +test: test.o data.o config.o util.o method.o result.o xxhash.o libzstd.a + $(CC) $^ $(REGRESSION_LDFLAGS) -o $@ + +.PHONY: clean +clean: + $(MAKE) -C $(LIBDIR) clean + $(RM) *.o *.a test diff --git a/src/zstd/tests/regression/config.c b/src/zstd/tests/regression/config.c new file mode 100644 index 000000000..ed6b69235 --- /dev/null +++ b/src/zstd/tests/regression/config.c @@ -0,0 +1,278 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "config.h" + +/* Define a config for each fast level we want to test with. */ +#define FAST_LEVEL(x) \ + param_value_t const level_fast##x##_param_values[] = { \ + {.param = ZSTD_c_compressionLevel, .value = -x}, \ + }; \ + config_t const level_fast##x = { \ + .name = "level -" #x, \ + .cli_args = "--fast=" #x, \ + .param_values = PARAM_VALUES(level_fast##x##_param_values), \ + }; \ + config_t const level_fast##x##_dict = { \ + .name = "level -" #x " with dict", \ + .cli_args = "--fast=" #x, \ + .param_values = PARAM_VALUES(level_fast##x##_param_values), \ + .use_dictionary = 1, \ + }; + +/* Define a config for each level we want to test with. */ +#define LEVEL(x) \ + param_value_t const level_##x##_param_values[] = { \ + {.param = ZSTD_c_compressionLevel, .value = x}, \ + }; \ + config_t const level_##x = { \ + .name = "level " #x, \ + .cli_args = "-" #x, \ + .param_values = PARAM_VALUES(level_##x##_param_values), \ + }; \ + config_t const level_##x##_dict = { \ + .name = "level " #x " with dict", \ + .cli_args = "-" #x, \ + .param_values = PARAM_VALUES(level_##x##_param_values), \ + .use_dictionary = 1, \ + }; + +#define PARAM_VALUES(pv) \ + { .data = pv, .size = sizeof(pv) / sizeof((pv)[0]) } + +#include "levels.h" + +#undef LEVEL +#undef FAST_LEVEL + +static config_t no_pledged_src_size = { + .name = "no source size", + .cli_args = "", + .param_values = PARAM_VALUES(level_0_param_values), + .no_pledged_src_size = 1, +}; + +static param_value_t const ldm_param_values[] = { + {.param = ZSTD_c_enableLongDistanceMatching, .value = 1}, +}; + +static config_t ldm = { + .name = "long distance mode", + .cli_args = "--long", + .param_values = PARAM_VALUES(ldm_param_values), +}; + +static param_value_t const mt_param_values[] = { + {.param = ZSTD_c_nbWorkers, .value = 2}, +}; + +static config_t mt = { + .name = "multithreaded", + .cli_args = "-T2", + .param_values = PARAM_VALUES(mt_param_values), +}; + +static param_value_t const mt_ldm_param_values[] = { + {.param = ZSTD_c_nbWorkers, .value = 2}, + {.param = ZSTD_c_enableLongDistanceMatching, .value = 1}, +}; + +static config_t mt_ldm = { + .name = "multithreaded long distance mode", + .cli_args = "-T2 --long", + .param_values = PARAM_VALUES(mt_ldm_param_values), +}; + +static param_value_t mt_advanced_param_values[] = { + {.param = ZSTD_c_nbWorkers, .value = 2}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, +}; + +static config_t mt_advanced = { + .name = "multithreaded with advanced params", + .cli_args = "-T2 --no-compress-literals", + .param_values = PARAM_VALUES(mt_advanced_param_values), +}; + +static param_value_t const small_wlog_param_values[] = { + {.param = ZSTD_c_windowLog, .value = 10}, +}; + +static config_t small_wlog = { + .name = "small window log", + .cli_args = "--zstd=wlog=10", + .param_values = PARAM_VALUES(small_wlog_param_values), +}; + +static param_value_t const small_hlog_param_values[] = { + {.param = ZSTD_c_hashLog, .value = 6}, + {.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt}, +}; + +static config_t small_hlog = { + .name = "small hash log", + .cli_args = "--zstd=hlog=6,strat=7", + .param_values = PARAM_VALUES(small_hlog_param_values), +}; + +static param_value_t const small_clog_param_values[] = { + {.param = ZSTD_c_chainLog, .value = 6}, + {.param = ZSTD_c_strategy, .value = (int)ZSTD_btopt}, +}; + +static config_t small_clog = { + .name = "small chain log", + .cli_args = "--zstd=clog=6,strat=7", + .param_values = PARAM_VALUES(small_clog_param_values), +}; + +static param_value_t const uncompressed_literals_param_values[] = { + {.param = ZSTD_c_compressionLevel, .value = 3}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, +}; + +static config_t uncompressed_literals = { + .name = "uncompressed literals", + .cli_args = "-3 --no-compress-literals", + .param_values = PARAM_VALUES(uncompressed_literals_param_values), +}; + +static param_value_t const uncompressed_literals_opt_param_values[] = { + {.param = ZSTD_c_compressionLevel, .value = 19}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_uncompressed}, +}; + +static config_t uncompressed_literals_opt = { + .name = "uncompressed literals optimal", + .cli_args = "-19 --no-compress-literals", + .param_values = PARAM_VALUES(uncompressed_literals_opt_param_values), +}; + +static param_value_t const huffman_literals_param_values[] = { + {.param = ZSTD_c_compressionLevel, .value = -1}, + {.param = ZSTD_c_literalCompressionMode, .value = ZSTD_lcm_huffman}, +}; + +static config_t huffman_literals = { + .name = "huffman literals", + .cli_args = "--fast=1 --compress-literals", + .param_values = PARAM_VALUES(huffman_literals_param_values), +}; + +static param_value_t const explicit_params_param_values[] = { + {.param = ZSTD_c_checksumFlag, .value = 1}, + {.param = ZSTD_c_contentSizeFlag, .value = 0}, + {.param = ZSTD_c_dictIDFlag, .value = 0}, + {.param = ZSTD_c_strategy, .value = (int)ZSTD_greedy}, + {.param = ZSTD_c_windowLog, .value = 18}, + {.param = ZSTD_c_hashLog, .value = 21}, + {.param = ZSTD_c_chainLog, .value = 21}, + {.param = ZSTD_c_targetLength, .value = 100}, +}; + +static config_t explicit_params = { + .name = "explicit params", + .cli_args = "--no-check --no-dictID --zstd=strategy=3,wlog=18,hlog=21,clog=21,tlen=100", + .param_values = PARAM_VALUES(explicit_params_param_values), +}; + +static config_t const* g_configs[] = { + +#define FAST_LEVEL(x) &level_fast##x, &level_fast##x##_dict, +#define LEVEL(x) &level_##x, &level_##x##_dict, +#include "levels.h" +#undef LEVEL +#undef FAST_LEVEL + + &no_pledged_src_size, + &ldm, + &mt, + &mt_ldm, + &small_wlog, + &small_hlog, + &small_clog, + &explicit_params, + &uncompressed_literals, + &uncompressed_literals_opt, + &huffman_literals, + &mt_advanced, + NULL, +}; + +config_t const* const* configs = g_configs; + +int config_skip_data(config_t const* config, data_t const* data) { + return config->use_dictionary && !data_has_dict(data); +} + +int config_get_level(config_t const* config) +{ + param_values_t const params = config->param_values; + size_t i; + for (i = 0; i < params.size; ++i) { + if (params.data[i].param == ZSTD_c_compressionLevel) + return (int)params.data[i].value; + } + return CONFIG_NO_LEVEL; +} + +ZSTD_parameters config_get_zstd_params( + config_t const* config, + uint64_t srcSize, + size_t dictSize) +{ + ZSTD_parameters zparams = {}; + param_values_t const params = config->param_values; + int level = config_get_level(config); + if (level == CONFIG_NO_LEVEL) + level = 3; + zparams = ZSTD_getParams( + level, + config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : srcSize, + dictSize); + for (size_t i = 0; i < params.size; ++i) { + unsigned const value = params.data[i].value; + switch (params.data[i].param) { + case ZSTD_c_contentSizeFlag: + zparams.fParams.contentSizeFlag = value; + break; + case ZSTD_c_checksumFlag: + zparams.fParams.checksumFlag = value; + break; + case ZSTD_c_dictIDFlag: + zparams.fParams.noDictIDFlag = !value; + break; + case ZSTD_c_windowLog: + zparams.cParams.windowLog = value; + break; + case ZSTD_c_chainLog: + zparams.cParams.chainLog = value; + break; + case ZSTD_c_hashLog: + zparams.cParams.hashLog = value; + break; + case ZSTD_c_searchLog: + zparams.cParams.searchLog = value; + break; + case ZSTD_c_minMatch: + zparams.cParams.minMatch = value; + break; + case ZSTD_c_targetLength: + zparams.cParams.targetLength = value; + break; + case ZSTD_c_strategy: + zparams.cParams.strategy = (ZSTD_strategy)value; + break; + default: + break; + } + } + return zparams; +} diff --git a/src/zstd/tests/regression/config.h b/src/zstd/tests/regression/config.h new file mode 100644 index 000000000..aa563b9e9 --- /dev/null +++ b/src/zstd/tests/regression/config.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef CONFIG_H +#define CONFIG_H + +#include <stddef.h> + +#define ZSTD_STATIC_LINKING_ONLY +#include <zstd.h> + +#include "data.h" + +typedef struct { + ZSTD_cParameter param; + int value; +} param_value_t; + +typedef struct { + size_t size; + param_value_t const* data; +} param_values_t; + +/** + * The config tells the compression method what options to use. + */ +typedef struct { + const char* name; /**< Identifies the config in the results table */ + /** + * Optional arguments to pass to the CLI. If not set, CLI-based methods + * will skip this config. + */ + char const* cli_args; + /** + * Parameters to pass to the advanced API. If the advanced API isn't used, + * the parameters will be derived from these. + */ + param_values_t param_values; + /** + * Boolean parameter that says if we should use a dictionary. If the data + * doesn't have a dictionary, this config is skipped. Defaults to no. + */ + int use_dictionary; + /** + * Boolean parameter that says if we should pass the pledged source size + * when the method allows it. Defaults to yes. + */ + int no_pledged_src_size; +} config_t; + +/** + * Returns true if the config should skip this data. + * For instance, if the config requires a dictionary but the data doesn't have + * one. + */ +int config_skip_data(config_t const* config, data_t const* data); + +#define CONFIG_NO_LEVEL (-ZSTD_TARGETLENGTH_MAX - 1) +/** + * Returns the compression level specified by the config, or CONFIG_NO_LEVEL if + * no level is specified. Note that 0 is a valid compression level, meaning + * default. + */ +int config_get_level(config_t const* config); + +/** + * Returns the compression parameters specified by the config. + */ +ZSTD_parameters config_get_zstd_params( + config_t const* config, + uint64_t srcSize, + size_t dictSize); + +/** + * The NULL-terminated list of configs. + */ +extern config_t const* const* configs; + +#endif diff --git a/src/zstd/tests/regression/data.c b/src/zstd/tests/regression/data.c new file mode 100644 index 000000000..b75ac1192 --- /dev/null +++ b/src/zstd/tests/regression/data.c @@ -0,0 +1,613 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "data.h" + +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <string.h> + +#include <sys/stat.h> + +#include <curl/curl.h> + +#include "mem.h" +#include "util.h" +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + +/** + * Data objects + */ + +#define REGRESSION_RELEASE(x) \ + "https://github.com/facebook/zstd/releases/download/regression-data/" x + +data_t silesia = { + .name = "silesia", + .type = data_type_dir, + .data = + { + .url = REGRESSION_RELEASE("silesia.tar.zst"), + .xxhash64 = 0x48a199f92f93e977LL, + }, +}; + +data_t silesia_tar = { + .name = "silesia.tar", + .type = data_type_file, + .data = + { + .url = REGRESSION_RELEASE("silesia.tar.zst"), + .xxhash64 = 0x48a199f92f93e977LL, + }, +}; + +data_t github = { + .name = "github", + .type = data_type_dir, + .data = + { + .url = REGRESSION_RELEASE("github.tar.zst"), + .xxhash64 = 0xa9b1b44b020df292LL, + }, + .dict = + { + .url = REGRESSION_RELEASE("github.dict.zst"), + .xxhash64 = 0x1eddc6f737d3cb53LL, + + }, +}; + +static data_t* g_data[] = { + &silesia, + &silesia_tar, + &github, + NULL, +}; + +data_t const* const* data = (data_t const* const*)g_data; + +/** + * data helpers. + */ + +int data_has_dict(data_t const* data) { + return data->dict.url != NULL; +} + +/** + * data buffer helper functions (documented in header). + */ + +data_buffer_t data_buffer_create(size_t const capacity) { + data_buffer_t buffer = {}; + + buffer.data = (uint8_t*)malloc(capacity); + if (buffer.data == NULL) + return buffer; + buffer.capacity = capacity; + return buffer; +} + +data_buffer_t data_buffer_read(char const* filename) { + data_buffer_t buffer = {}; + + uint64_t const size = UTIL_getFileSize(filename); + if (size == UTIL_FILESIZE_UNKNOWN) { + fprintf(stderr, "unknown size for %s\n", filename); + return buffer; + } + + buffer.data = (uint8_t*)malloc(size); + if (buffer.data == NULL) { + fprintf(stderr, "malloc failed\n"); + return buffer; + } + buffer.capacity = size; + + FILE* file = fopen(filename, "rb"); + if (file == NULL) { + fprintf(stderr, "file null\n"); + goto err; + } + buffer.size = fread(buffer.data, 1, buffer.capacity, file); + fclose(file); + if (buffer.size != buffer.capacity) { + fprintf(stderr, "read %zu != %zu\n", buffer.size, buffer.capacity); + goto err; + } + + return buffer; +err: + free(buffer.data); + memset(&buffer, 0, sizeof(buffer)); + return buffer; +} + +data_buffer_t data_buffer_get_data(data_t const* data) { + data_buffer_t const kEmptyBuffer = {}; + + if (data->type != data_type_file) + return kEmptyBuffer; + + return data_buffer_read(data->data.path); +} + +data_buffer_t data_buffer_get_dict(data_t const* data) { + data_buffer_t const kEmptyBuffer = {}; + + if (!data_has_dict(data)) + return kEmptyBuffer; + + return data_buffer_read(data->dict.path); +} + +int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2) { + size_t const size = + buffer1.size < buffer2.size ? buffer1.size : buffer2.size; + int const cmp = memcmp(buffer1.data, buffer2.data, size); + if (cmp != 0) + return cmp; + if (buffer1.size < buffer2.size) + return -1; + if (buffer1.size == buffer2.size) + return 0; + assert(buffer1.size > buffer2.size); + return 1; +} + +void data_buffer_free(data_buffer_t buffer) { + free(buffer.data); +} + +/** + * data filenames helpers. + */ + +FileNamesTable* data_filenames_get(data_t const* data) +{ + char const* const path = data->data.path; + return UTIL_createExpandedFNT(&path, 1, 0 /* followLinks */ ); +} + +/** + * data buffers helpers. + */ + +data_buffers_t data_buffers_get(data_t const* data) { + data_buffers_t buffers = {.size = 0}; + FileNamesTable* const filenames = data_filenames_get(data); + if (filenames == NULL) return buffers; + if (filenames->tableSize == 0) { + UTIL_freeFileNamesTable(filenames); + return buffers; + } + + data_buffer_t* buffersPtr = + (data_buffer_t*)malloc(filenames->tableSize * sizeof(*buffersPtr)); + if (buffersPtr == NULL) { + UTIL_freeFileNamesTable(filenames); + return buffers; + } + buffers.buffers = (data_buffer_t const*)buffersPtr; + buffers.size = filenames->tableSize; + + for (size_t i = 0; i < filenames->tableSize; ++i) { + buffersPtr[i] = data_buffer_read(filenames->fileNames[i]); + if (buffersPtr[i].data == NULL) { + data_buffers_t const kEmptyBuffer = {}; + data_buffers_free(buffers); + UTIL_freeFileNamesTable(filenames); + return kEmptyBuffer; + } + } + + UTIL_freeFileNamesTable(filenames); + return buffers; +} + +/** + * Frees the data buffers. + */ +void data_buffers_free(data_buffers_t buffers) { + free((data_buffer_t*)buffers.buffers); +} + +/** + * Initialization and download functions. + */ + +static char* g_data_dir = NULL; + +/* mkdir -p */ +static int ensure_directory_exists(char const* indir) { + char* const dir = strdup(indir); + char* end = dir; + int ret = 0; + if (dir == NULL) { + ret = EINVAL; + goto out; + } + do { + /* Find the next directory level. */ + for (++end; *end != '\0' && *end != '/'; ++end) + ; + /* End the string there, make the directory, and restore the string. */ + char const save = *end; + *end = '\0'; + int const isdir = UTIL_isDirectory(dir); + ret = mkdir(dir, S_IRWXU); + *end = save; + /* Its okay if the directory already exists. */ + if (ret == 0 || (errno == EEXIST && isdir)) + continue; + ret = errno; + fprintf(stderr, "mkdir() failed\n"); + goto out; + } while (*end != '\0'); + + ret = 0; +out: + free(dir); + return ret; +} + +/** Concatenate 3 strings into a new buffer. */ +static char* cat3(char const* str1, char const* str2, char const* str3) { + size_t const size1 = strlen(str1); + size_t const size2 = strlen(str2); + size_t const size3 = str3 == NULL ? 0 : strlen(str3); + size_t const size = size1 + size2 + size3 + 1; + char* const dst = (char*)malloc(size); + if (dst == NULL) + return NULL; + strcpy(dst, str1); + strcpy(dst + size1, str2); + if (str3 != NULL) + strcpy(dst + size1 + size2, str3); + assert(strlen(dst) == size1 + size2 + size3); + return dst; +} + +static char* cat2(char const* str1, char const* str2) { + return cat3(str1, str2, NULL); +} + +/** + * State needed by the curl callback. + * It takes data from curl, hashes it, and writes it to the file. + */ +typedef struct { + FILE* file; + XXH64_state_t xxhash64; + int error; +} curl_data_t; + +/** Create the curl state. */ +static curl_data_t curl_data_create( + data_resource_t const* resource, + data_type_t type) { + curl_data_t cdata = {}; + + XXH64_reset(&cdata.xxhash64, 0); + + assert(UTIL_isDirectory(g_data_dir)); + + if (type == data_type_file) { + /* Decompress the resource and store to the path. */ + char* cmd = cat3("zstd -dqfo '", resource->path, "'"); + if (cmd == NULL) { + cdata.error = ENOMEM; + return cdata; + } + cdata.file = popen(cmd, "w"); + free(cmd); + } else { + /* Decompress and extract the resource to the cache directory. */ + char* cmd = cat3("zstd -dc | tar -x -C '", g_data_dir, "'"); + if (cmd == NULL) { + cdata.error = ENOMEM; + return cdata; + } + cdata.file = popen(cmd, "w"); + free(cmd); + } + if (cdata.file == NULL) { + cdata.error = errno; + } + + return cdata; +} + +/** Free the curl state. */ +static int curl_data_free(curl_data_t cdata) { + return pclose(cdata.file); +} + +/** curl callback. Updates the hash, and writes to the file. */ +static size_t curl_write(void* data, size_t size, size_t count, void* ptr) { + curl_data_t* cdata = (curl_data_t*)ptr; + size_t const written = fwrite(data, size, count, cdata->file); + XXH64_update(&cdata->xxhash64, data, written * size); + return written; +} + +static int curl_download_resource( + CURL* curl, + data_resource_t const* resource, + data_type_t type) { + curl_data_t cdata; + /* Download the data. */ + if (curl_easy_setopt(curl, CURLOPT_URL, resource->url) != 0) + return EINVAL; + if (curl_easy_setopt(curl, CURLOPT_WRITEDATA, &cdata) != 0) + return EINVAL; + cdata = curl_data_create(resource, type); + if (cdata.error != 0) + return cdata.error; + int const curl_err = curl_easy_perform(curl); + int const close_err = curl_data_free(cdata); + if (curl_err) { + fprintf( + stderr, + "downloading '%s' for '%s' failed\n", + resource->url, + resource->path); + return EIO; + } + if (close_err) { + fprintf(stderr, "writing data to '%s' failed\n", resource->path); + return EIO; + } + /* check that the file exists. */ + if (type == data_type_file && !UTIL_isRegularFile(resource->path)) { + fprintf(stderr, "output file '%s' does not exist\n", resource->path); + return EIO; + } + if (type == data_type_dir && !UTIL_isDirectory(resource->path)) { + fprintf( + stderr, "output directory '%s' does not exist\n", resource->path); + return EIO; + } + /* Check that the hash matches. */ + if (XXH64_digest(&cdata.xxhash64) != resource->xxhash64) { + fprintf( + stderr, + "checksum does not match: 0x%llxLL != 0x%llxLL\n", + (unsigned long long)XXH64_digest(&cdata.xxhash64), + (unsigned long long)resource->xxhash64); + return EINVAL; + } + + return 0; +} + +/** Download a single data object. */ +static int curl_download_datum(CURL* curl, data_t const* data) { + int ret; + ret = curl_download_resource(curl, &data->data, data->type); + if (ret != 0) + return ret; + if (data_has_dict(data)) { + ret = curl_download_resource(curl, &data->dict, data_type_file); + if (ret != 0) + return ret; + } + return ret; +} + +/** Download all the data. */ +static int curl_download_data(data_t const* const* data) { + if (curl_global_init(CURL_GLOBAL_ALL) != 0) + return EFAULT; + + curl_data_t cdata = {}; + CURL* curl = curl_easy_init(); + int err = EFAULT; + + if (curl == NULL) + return EFAULT; + + if (curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1L) != 0) + goto out; + if (curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L) != 0) + goto out; + if (curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write) != 0) + goto out; + + assert(data != NULL); + for (; *data != NULL; ++data) { + if (curl_download_datum(curl, *data) != 0) + goto out; + } + + err = 0; +out: + curl_easy_cleanup(curl); + curl_global_cleanup(); + return err; +} + +/** Fill the path member variable of the data objects. */ +static int data_create_paths(data_t* const* data, char const* dir) { + size_t const dirlen = strlen(dir); + assert(data != NULL); + for (; *data != NULL; ++data) { + data_t* const datum = *data; + datum->data.path = cat3(dir, "/", datum->name); + if (datum->data.path == NULL) + return ENOMEM; + if (data_has_dict(datum)) { + datum->dict.path = cat2(datum->data.path, ".dict"); + if (datum->dict.path == NULL) + return ENOMEM; + } + } + return 0; +} + +/** Free the path member variable of the data objects. */ +static void data_free_paths(data_t* const* data) { + assert(data != NULL); + for (; *data != NULL; ++data) { + data_t* datum = *data; + free((void*)datum->data.path); + free((void*)datum->dict.path); + datum->data.path = NULL; + datum->dict.path = NULL; + } +} + +static char const kStampName[] = "STAMP"; + +static void xxh_update_le(XXH64_state_t* state, uint64_t data) { + if (!MEM_isLittleEndian()) + data = MEM_swap64(data); + XXH64_update(state, &data, sizeof(data)); +} + +/** Hash the data to create the stamp. */ +static uint64_t stamp_hash(data_t const* const* data) { + XXH64_state_t state; + + XXH64_reset(&state, 0); + assert(data != NULL); + for (; *data != NULL; ++data) { + data_t const* datum = *data; + /* We don't care about the URL that we fetch from. */ + /* The path is derived from the name. */ + XXH64_update(&state, datum->name, strlen(datum->name)); + xxh_update_le(&state, datum->data.xxhash64); + xxh_update_le(&state, datum->dict.xxhash64); + xxh_update_le(&state, datum->type); + } + return XXH64_digest(&state); +} + +/** Check if the stamp matches the stamp in the cache directory. */ +static int stamp_check(char const* dir, data_t const* const* data) { + char* stamp = cat3(dir, "/", kStampName); + uint64_t const expected = stamp_hash(data); + XXH64_canonical_t actual; + FILE* stampfile = NULL; + int matches = 0; + + if (stamp == NULL) + goto out; + if (!UTIL_isRegularFile(stamp)) { + fprintf(stderr, "stamp does not exist: recreating the data cache\n"); + goto out; + } + + stampfile = fopen(stamp, "rb"); + if (stampfile == NULL) { + fprintf(stderr, "could not open stamp: recreating the data cache\n"); + goto out; + } + + size_t b; + if ((b = fread(&actual, sizeof(actual), 1, stampfile)) != 1) { + fprintf(stderr, "invalid stamp: recreating the data cache\n"); + goto out; + } + + matches = (expected == XXH64_hashFromCanonical(&actual)); + if (matches) + fprintf(stderr, "stamp matches: reusing the cached data\n"); + else + fprintf(stderr, "stamp does not match: recreating the data cache\n"); + +out: + free(stamp); + if (stampfile != NULL) + fclose(stampfile); + return matches; +} + +/** On success write a new stamp, on failure delete the old stamp. */ +static int +stamp_write(char const* dir, data_t const* const* data, int const data_err) { + char* stamp = cat3(dir, "/", kStampName); + FILE* stampfile = NULL; + int err = EIO; + + if (stamp == NULL) + return ENOMEM; + + if (data_err != 0) { + err = data_err; + goto out; + } + XXH64_canonical_t hash; + + XXH64_canonicalFromHash(&hash, stamp_hash(data)); + + stampfile = fopen(stamp, "wb"); + if (stampfile == NULL) + goto out; + if (fwrite(&hash, sizeof(hash), 1, stampfile) != 1) + goto out; + err = 0; + fprintf(stderr, "stamped new data cache\n"); +out: + if (err != 0) + /* Ignore errors. */ + unlink(stamp); + free(stamp); + if (stampfile != NULL) + fclose(stampfile); + return err; +} + +int data_init(char const* dir) { + int err; + + if (dir == NULL) + return EINVAL; + + /* This must be first to simplify logic. */ + err = ensure_directory_exists(dir); + if (err != 0) + return err; + + /* Save the cache directory. */ + g_data_dir = strdup(dir); + if (g_data_dir == NULL) + return ENOMEM; + + err = data_create_paths(g_data, dir); + if (err != 0) + return err; + + /* If the stamp matches then we are good to go. + * This must be called before any modifications to the data cache. + * After this point, we MUST call stamp_write() to update the STAMP, + * since we've updated the data cache. + */ + if (stamp_check(dir, data)) + return 0; + + err = curl_download_data(data); + if (err != 0) + goto out; + +out: + /* This must be last, since it must know if data_init() succeeded. */ + stamp_write(dir, data, err); + return err; +} + +void data_finish(void) { + data_free_paths(g_data); + free(g_data_dir); + g_data_dir = NULL; +} diff --git a/src/zstd/tests/regression/data.h b/src/zstd/tests/regression/data.h new file mode 100644 index 000000000..90ed22f19 --- /dev/null +++ b/src/zstd/tests/regression/data.h @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef DATA_H +#define DATA_H + +#include <stddef.h> +#include <stdint.h> + +typedef enum { + data_type_file = 1, /**< This data is a file. *.zst */ + data_type_dir = 2, /**< This data is a directory. *.tar.zst */ +} data_type_t; + +typedef struct { + char const* url; /**< Where to get this resource. */ + uint64_t xxhash64; /**< Hash of the url contents. */ + char const* path; /**< The path of the unpacked resource (derived). */ +} data_resource_t; + +typedef struct { + data_resource_t data; + data_resource_t dict; + data_type_t type; /**< The type of the data. */ + char const* name; /**< The logical name of the data (no extension). */ +} data_t; + +/** + * The NULL-terminated list of data objects. + */ +extern data_t const* const* data; + + +int data_has_dict(data_t const* data); + +/** + * Initializes the data module and downloads the data necessary. + * Caches the downloads in dir. We add a stamp file in the directory after + * a successful download. If a stamp file already exists, and matches our + * current data stamp, we will use the cached data without downloading. + * + * @param dir The directory to cache the downloaded data into. + * + * @returns 0 on success. + */ +int data_init(char const* dir); + +/** + * Must be called at exit to free resources allocated by data_init(). + */ +void data_finish(void); + +typedef struct { + uint8_t* data; + size_t size; + size_t capacity; +} data_buffer_t; + +/** + * Read the file that data points to into a buffer. + * NOTE: data must be a file, not a directory. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_get_data(data_t const* data); + +/** + * Read the dictionary that the data points to into a buffer. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_get_dict(data_t const* data); + +/** + * Read the contents of filename into a buffer. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_read(char const* filename); + +/** + * Create a buffer with the specified capacity. + * + * @returns The buffer, which is NULL on failure. + */ +data_buffer_t data_buffer_create(size_t capacity); + +/** + * Calls memcmp() on the contents [0, size) of both buffers. + */ +int data_buffer_compare(data_buffer_t buffer1, data_buffer_t buffer2); + +/** + * Frees an allocated buffer. + */ +void data_buffer_free(data_buffer_t buffer); + + +typedef struct { + data_buffer_t const* buffers; + size_t size; +} data_buffers_t; + +/** + * @returns a list of buffers for every file in data. It is zero sized on error. + */ +data_buffers_t data_buffers_get(data_t const* data); + +/** + * Frees the data buffers. + */ +void data_buffers_free(data_buffers_t buffers); + +#endif diff --git a/src/zstd/tests/regression/levels.h b/src/zstd/tests/regression/levels.h new file mode 100644 index 000000000..5e7d40a7d --- /dev/null +++ b/src/zstd/tests/regression/levels.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef LEVEL +# error LEVEL(x) must be defined +#endif +#ifndef FAST_LEVEL +# error FAST_LEVEL(x) must be defined +#endif + +/** + * The levels are chosen to trigger every strategy in every source size, + * as well as some fast levels and the default level. + * If you change the compression levels, you should probably update these. + */ + +FAST_LEVEL(5) + +FAST_LEVEL(3) + +FAST_LEVEL(1) +LEVEL(0) +LEVEL(1) + +LEVEL(3) +LEVEL(4) +LEVEL(5) +LEVEL(6) +LEVEL(7) + +LEVEL(9) + +LEVEL(13) + +LEVEL(16) + +LEVEL(19) diff --git a/src/zstd/tests/regression/method.c b/src/zstd/tests/regression/method.c new file mode 100644 index 000000000..3c949a278 --- /dev/null +++ b/src/zstd/tests/regression/method.c @@ -0,0 +1,688 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "method.h" + +#include <stdio.h> +#include <stdlib.h> + +#define ZSTD_STATIC_LINKING_ONLY +#include <zstd.h> + +#define MIN(x, y) ((x) < (y) ? (x) : (y)) + +static char const* g_zstdcli = NULL; + +void method_set_zstdcli(char const* zstdcli) { + g_zstdcli = zstdcli; +} + +/** + * Macro to get a pointer of type, given ptr, which is a member variable with + * the given name, member. + * + * method_state_t* base = ...; + * buffer_state_t* state = container_of(base, buffer_state_t, base); + */ +#define container_of(ptr, type, member) \ + ((type*)(ptr == NULL ? NULL : (char*)(ptr)-offsetof(type, member))) + +/** State to reuse the same buffers between compression calls. */ +typedef struct { + method_state_t base; + data_buffers_t inputs; /**< The input buffer for each file. */ + data_buffer_t dictionary; /**< The dictionary. */ + data_buffer_t compressed; /**< The compressed data buffer. */ + data_buffer_t decompressed; /**< The decompressed data buffer. */ +} buffer_state_t; + +static size_t buffers_max_size(data_buffers_t buffers) { + size_t max = 0; + for (size_t i = 0; i < buffers.size; ++i) { + if (buffers.buffers[i].size > max) + max = buffers.buffers[i].size; + } + return max; +} + +static method_state_t* buffer_state_create(data_t const* data) { + buffer_state_t* state = (buffer_state_t*)calloc(1, sizeof(buffer_state_t)); + if (state == NULL) + return NULL; + state->base.data = data; + state->inputs = data_buffers_get(data); + state->dictionary = data_buffer_get_dict(data); + size_t const max_size = buffers_max_size(state->inputs); + state->compressed = data_buffer_create(ZSTD_compressBound(max_size)); + state->decompressed = data_buffer_create(max_size); + return &state->base; +} + +static void buffer_state_destroy(method_state_t* base) { + if (base == NULL) + return; + buffer_state_t* state = container_of(base, buffer_state_t, base); + free(state); +} + +static int buffer_state_bad( + buffer_state_t const* state, + config_t const* config) { + if (state == NULL) { + fprintf(stderr, "buffer_state_t is NULL\n"); + return 1; + } + if (state->inputs.size == 0 || state->compressed.data == NULL || + state->decompressed.data == NULL) { + fprintf(stderr, "buffer state allocation failure\n"); + return 1; + } + if (config->use_dictionary && state->dictionary.data == NULL) { + fprintf(stderr, "dictionary loading failed\n"); + return 1; + } + return 0; +} + +static result_t simple_compress(method_state_t* base, config_t const* config) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + /* Keep the tests short by skipping directories, since behavior shouldn't + * change. + */ + if (base->data->type != data_type_file) + return result_error(result_error_skip); + + if (config->use_dictionary || config->no_pledged_src_size) + return result_error(result_error_skip); + + /* If the config doesn't specify a level, skip. */ + int const level = config_get_level(config); + if (level == CONFIG_NO_LEVEL) + return result_error(result_error_skip); + + data_buffer_t const input = state->inputs.buffers[0]; + + /* Compress, decompress, and check the result. */ + state->compressed.size = ZSTD_compress( + state->compressed.data, + state->compressed.capacity, + input.data, + input.size, + level); + if (ZSTD_isError(state->compressed.size)) + return result_error(result_error_compression_error); + + state->decompressed.size = ZSTD_decompress( + state->decompressed.data, + state->decompressed.capacity, + state->compressed.data, + state->compressed.size); + if (ZSTD_isError(state->decompressed.size)) + return result_error(result_error_decompression_error); + if (data_buffer_compare(input, state->decompressed)) + return result_error(result_error_round_trip_error); + + result_data_t data; + data.total_size = state->compressed.size; + return result_data(data); +} + +static result_t compress_cctx_compress( + method_state_t* base, + config_t const* config) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + if (config->no_pledged_src_size) + return result_error(result_error_skip); + + if (base->data->type != data_type_dir) + return result_error(result_error_skip); + + int const level = config_get_level(config); + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + if (cctx == NULL || dctx == NULL) { + fprintf(stderr, "context creation failed\n"); + return result_error(result_error_system_error); + } + + result_t result; + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t const input = state->inputs.buffers[i]; + ZSTD_parameters const params = + config_get_zstd_params(config, input.size, state->dictionary.size); + + if (level == CONFIG_NO_LEVEL) + state->compressed.size = ZSTD_compress_advanced( + cctx, + state->compressed.data, + state->compressed.capacity, + input.data, + input.size, + config->use_dictionary ? state->dictionary.data : NULL, + config->use_dictionary ? state->dictionary.size : 0, + params); + else if (config->use_dictionary) + state->compressed.size = ZSTD_compress_usingDict( + cctx, + state->compressed.data, + state->compressed.capacity, + input.data, + input.size, + state->dictionary.data, + state->dictionary.size, + level); + else + state->compressed.size = ZSTD_compressCCtx( + cctx, + state->compressed.data, + state->compressed.capacity, + input.data, + input.size, + level); + + if (ZSTD_isError(state->compressed.size)) { + result = result_error(result_error_compression_error); + goto out; + } + + if (config->use_dictionary) + state->decompressed.size = ZSTD_decompress_usingDict( + dctx, + state->decompressed.data, + state->decompressed.capacity, + state->compressed.data, + state->compressed.size, + state->dictionary.data, + state->dictionary.size); + else + state->decompressed.size = ZSTD_decompressDCtx( + dctx, + state->decompressed.data, + state->decompressed.capacity, + state->compressed.data, + state->compressed.size); + if (ZSTD_isError(state->decompressed.size)) { + result = result_error(result_error_decompression_error); + goto out; + } + if (data_buffer_compare(input, state->decompressed)) { + result = result_error(result_error_round_trip_error); + goto out; + } + + data.total_size += state->compressed.size; + } + + result = result_data(data); +out: + ZSTD_freeCCtx(cctx); + ZSTD_freeDCtx(dctx); + return result; +} + +/** Generic state creation function. */ +static method_state_t* method_state_create(data_t const* data) { + method_state_t* state = (method_state_t*)malloc(sizeof(method_state_t)); + if (state == NULL) + return NULL; + state->data = data; + return state; +} + +static void method_state_destroy(method_state_t* state) { + free(state); +} + +static result_t cli_compress(method_state_t* state, config_t const* config) { + if (config->cli_args == NULL) + return result_error(result_error_skip); + + /* We don't support no pledged source size with directories. Too slow. */ + if (state->data->type == data_type_dir && config->no_pledged_src_size) + return result_error(result_error_skip); + + if (g_zstdcli == NULL) + return result_error(result_error_system_error); + + /* '<zstd>' -cqr <args> [-D '<dict>'] '<file/dir>' */ + char cmd[1024]; + size_t const cmd_size = snprintf( + cmd, + sizeof(cmd), + "'%s' -cqr %s %s%s%s %s '%s'", + g_zstdcli, + config->cli_args, + config->use_dictionary ? "-D '" : "", + config->use_dictionary ? state->data->dict.path : "", + config->use_dictionary ? "'" : "", + config->no_pledged_src_size ? "<" : "", + state->data->data.path); + if (cmd_size >= sizeof(cmd)) { + fprintf(stderr, "command too large: %s\n", cmd); + return result_error(result_error_system_error); + } + FILE* zstd = popen(cmd, "r"); + if (zstd == NULL) { + fprintf(stderr, "failed to popen command: %s\n", cmd); + return result_error(result_error_system_error); + } + + char out[4096]; + size_t total_size = 0; + while (1) { + size_t const size = fread(out, 1, sizeof(out), zstd); + total_size += size; + if (size != sizeof(out)) + break; + } + if (ferror(zstd) || pclose(zstd) != 0) { + fprintf(stderr, "zstd failed with command: %s\n", cmd); + return result_error(result_error_compression_error); + } + + result_data_t const data = {.total_size = total_size}; + return result_data(data); +} + +static int advanced_config( + ZSTD_CCtx* cctx, + buffer_state_t* state, + config_t const* config) { + ZSTD_CCtx_reset(cctx, ZSTD_reset_session_and_parameters); + for (size_t p = 0; p < config->param_values.size; ++p) { + param_value_t const pv = config->param_values.data[p]; + if (ZSTD_isError(ZSTD_CCtx_setParameter(cctx, pv.param, pv.value))) { + return 1; + } + } + if (config->use_dictionary) { + if (ZSTD_isError(ZSTD_CCtx_loadDictionary( + cctx, state->dictionary.data, state->dictionary.size))) { + return 1; + } + } + return 0; +} + +static result_t advanced_one_pass_compress_output_adjustment( + method_state_t* base, + config_t const* config, + size_t const subtract) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + result_t result; + + if (!cctx || advanced_config(cctx, state, config)) { + result = result_error(result_error_compression_error); + goto out; + } + + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t const input = state->inputs.buffers[i]; + + if (!config->no_pledged_src_size) { + if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) { + result = result_error(result_error_compression_error); + goto out; + } + } + size_t const size = ZSTD_compress2( + cctx, + state->compressed.data, + ZSTD_compressBound(input.size) - subtract, + input.data, + input.size); + if (ZSTD_isError(size)) { + result = result_error(result_error_compression_error); + goto out; + } + data.total_size += size; + } + + result = result_data(data); +out: + ZSTD_freeCCtx(cctx); + return result; +} + +static result_t advanced_one_pass_compress( + method_state_t* base, + config_t const* config) { + return advanced_one_pass_compress_output_adjustment(base, config, 0); +} + +static result_t advanced_one_pass_compress_small_output( + method_state_t* base, + config_t const* config) { + return advanced_one_pass_compress_output_adjustment(base, config, 1); +} + +static result_t advanced_streaming_compress( + method_state_t* base, + config_t const* config) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + ZSTD_CCtx* cctx = ZSTD_createCCtx(); + result_t result; + + if (!cctx || advanced_config(cctx, state, config)) { + result = result_error(result_error_compression_error); + goto out; + } + + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t input = state->inputs.buffers[i]; + + if (!config->no_pledged_src_size) { + if (ZSTD_isError(ZSTD_CCtx_setPledgedSrcSize(cctx, input.size))) { + result = result_error(result_error_compression_error); + goto out; + } + } + + while (input.size > 0) { + ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)}; + input.data += in.size; + input.size -= in.size; + ZSTD_EndDirective const op = + input.size > 0 ? ZSTD_e_continue : ZSTD_e_end; + size_t ret = 0; + while (in.pos < in.size || (op == ZSTD_e_end && ret != 0)) { + ZSTD_outBuffer out = {state->compressed.data, + MIN(state->compressed.capacity, 1024)}; + ret = ZSTD_compressStream2(cctx, &out, &in, op); + if (ZSTD_isError(ret)) { + result = result_error(result_error_compression_error); + goto out; + } + data.total_size += out.pos; + } + } + } + + result = result_data(data); +out: + ZSTD_freeCCtx(cctx); + return result; +} + +static int init_cstream( + buffer_state_t* state, + ZSTD_CStream* zcs, + config_t const* config, + int const advanced, + ZSTD_CDict** cdict) +{ + size_t zret; + if (advanced) { + ZSTD_parameters const params = config_get_zstd_params(config, 0, 0); + ZSTD_CDict* dict = NULL; + if (cdict) { + if (!config->use_dictionary) + return 1; + *cdict = ZSTD_createCDict_advanced( + state->dictionary.data, + state->dictionary.size, + ZSTD_dlm_byRef, + ZSTD_dct_auto, + params.cParams, + ZSTD_defaultCMem); + if (!*cdict) { + return 1; + } + zret = ZSTD_initCStream_usingCDict_advanced( + zcs, *cdict, params.fParams, ZSTD_CONTENTSIZE_UNKNOWN); + } else { + zret = ZSTD_initCStream_advanced( + zcs, + config->use_dictionary ? state->dictionary.data : NULL, + config->use_dictionary ? state->dictionary.size : 0, + params, + ZSTD_CONTENTSIZE_UNKNOWN); + } + } else { + int const level = config_get_level(config); + if (level == CONFIG_NO_LEVEL) + return 1; + if (cdict) { + if (!config->use_dictionary) + return 1; + *cdict = ZSTD_createCDict( + state->dictionary.data, + state->dictionary.size, + level); + if (!*cdict) { + return 1; + } + zret = ZSTD_initCStream_usingCDict(zcs, *cdict); + } else if (config->use_dictionary) { + zret = ZSTD_initCStream_usingDict( + zcs, + state->dictionary.data, + state->dictionary.size, + level); + } else { + zret = ZSTD_initCStream(zcs, level); + } + } + if (ZSTD_isError(zret)) { + return 1; + } + return 0; +} + +static result_t old_streaming_compress_internal( + method_state_t* base, + config_t const* config, + int const advanced, + int const cdict) { + buffer_state_t* state = container_of(base, buffer_state_t, base); + + if (buffer_state_bad(state, config)) + return result_error(result_error_system_error); + + + ZSTD_CStream* zcs = ZSTD_createCStream(); + ZSTD_CDict* cd = NULL; + result_t result; + if (zcs == NULL) { + result = result_error(result_error_compression_error); + goto out; + } + if (!advanced && config_get_level(config) == CONFIG_NO_LEVEL) { + result = result_error(result_error_skip); + goto out; + } + if (cdict && !config->use_dictionary) { + result = result_error(result_error_skip); + goto out; + } + if (init_cstream(state, zcs, config, advanced, cdict ? &cd : NULL)) { + result = result_error(result_error_compression_error); + goto out; + } + + result_data_t data = {.total_size = 0}; + for (size_t i = 0; i < state->inputs.size; ++i) { + data_buffer_t input = state->inputs.buffers[i]; + size_t zret = ZSTD_resetCStream( + zcs, + config->no_pledged_src_size ? ZSTD_CONTENTSIZE_UNKNOWN : input.size); + if (ZSTD_isError(zret)) { + result = result_error(result_error_compression_error); + goto out; + } + + while (input.size > 0) { + ZSTD_inBuffer in = {input.data, MIN(input.size, 4096)}; + input.data += in.size; + input.size -= in.size; + ZSTD_EndDirective const op = + input.size > 0 ? ZSTD_e_continue : ZSTD_e_end; + zret = 0; + while (in.pos < in.size || (op == ZSTD_e_end && zret != 0)) { + ZSTD_outBuffer out = {state->compressed.data, + MIN(state->compressed.capacity, 1024)}; + if (op == ZSTD_e_continue || in.pos < in.size) + zret = ZSTD_compressStream(zcs, &out, &in); + else + zret = ZSTD_endStream(zcs, &out); + if (ZSTD_isError(zret)) { + result = result_error(result_error_compression_error); + goto out; + } + data.total_size += out.pos; + } + } + } + + result = result_data(data); +out: + ZSTD_freeCStream(zcs); + ZSTD_freeCDict(cd); + return result; +} + +static result_t old_streaming_compress( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 0, /* cdict */ 0); +} + +static result_t old_streaming_compress_advanced( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 1, /* cdict */ 0); +} + +static result_t old_streaming_compress_cdict( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 0, /* cdict */ 1); +} + +static result_t old_streaming_compress_cdict_advanced( + method_state_t* base, + config_t const* config) +{ + return old_streaming_compress_internal( + base, config, /* advanced */ 1, /* cdict */ 1); +} + +method_t const simple = { + .name = "compress simple", + .create = buffer_state_create, + .compress = simple_compress, + .destroy = buffer_state_destroy, +}; + +method_t const compress_cctx = { + .name = "compress cctx", + .create = buffer_state_create, + .compress = compress_cctx_compress, + .destroy = buffer_state_destroy, +}; + +method_t const advanced_one_pass = { + .name = "advanced one pass", + .create = buffer_state_create, + .compress = advanced_one_pass_compress, + .destroy = buffer_state_destroy, +}; + +method_t const advanced_one_pass_small_out = { + .name = "advanced one pass small out", + .create = buffer_state_create, + .compress = advanced_one_pass_compress, + .destroy = buffer_state_destroy, +}; + +method_t const advanced_streaming = { + .name = "advanced streaming", + .create = buffer_state_create, + .compress = advanced_streaming_compress, + .destroy = buffer_state_destroy, +}; + +method_t const old_streaming = { + .name = "old streaming", + .create = buffer_state_create, + .compress = old_streaming_compress, + .destroy = buffer_state_destroy, +}; + +method_t const old_streaming_advanced = { + .name = "old streaming advanced", + .create = buffer_state_create, + .compress = old_streaming_compress_advanced, + .destroy = buffer_state_destroy, +}; + +method_t const old_streaming_cdict = { + .name = "old streaming cdcit", + .create = buffer_state_create, + .compress = old_streaming_compress_cdict, + .destroy = buffer_state_destroy, +}; + +method_t const old_streaming_advanced_cdict = { + .name = "old streaming advanced cdict", + .create = buffer_state_create, + .compress = old_streaming_compress_cdict_advanced, + .destroy = buffer_state_destroy, +}; + +method_t const cli = { + .name = "zstdcli", + .create = method_state_create, + .compress = cli_compress, + .destroy = method_state_destroy, +}; + +static method_t const* g_methods[] = { + &simple, + &compress_cctx, + &cli, + &advanced_one_pass, + &advanced_one_pass_small_out, + &advanced_streaming, + &old_streaming, + &old_streaming_advanced, + &old_streaming_cdict, + &old_streaming_advanced_cdict, + NULL, +}; + +method_t const* const* methods = g_methods; diff --git a/src/zstd/tests/regression/method.h b/src/zstd/tests/regression/method.h new file mode 100644 index 000000000..6884e5418 --- /dev/null +++ b/src/zstd/tests/regression/method.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef METHOD_H +#define METHOD_H + +#include <stddef.h> + +#include "data.h" +#include "config.h" +#include "result.h" + +/** + * The base class for state that methods keep. + * All derived method state classes must have a member of this type. + */ +typedef struct { + data_t const* data; +} method_state_t; + +/** + * A method that compresses the data using config. + */ +typedef struct { + char const* name; /**< The identifier for this method in the results. */ + /** + * Creates a state that must contain a member variable of method_state_t, + * and returns a pointer to that member variable. + * + * This method can be used to do expensive work that only depends on the + * data, like loading the data file into a buffer. + */ + method_state_t* (*create)(data_t const* data); + /** + * Compresses the data in the state using the given config. + * + * @param state A pointer to the state returned by create(). + * + * @returns The total compressed size on success, or an error code. + */ + result_t (*compress)(method_state_t* state, config_t const* config); + /** + * Frees the state. + */ + void (*destroy)(method_state_t* state); +} method_t; + +/** + * Set the zstd cli path. Must be called before any methods are used. + */ +void method_set_zstdcli(char const* zstdcli); + +/** + * A NULL-terminated list of methods. + */ +extern method_t const* const* methods; + +#endif diff --git a/src/zstd/tests/regression/result.c b/src/zstd/tests/regression/result.c new file mode 100644 index 000000000..2911722cd --- /dev/null +++ b/src/zstd/tests/regression/result.c @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "result.h" + +char const* result_get_error_string(result_t result) { + switch (result_get_error(result)) { + case result_error_ok: + return "okay"; + case result_error_skip: + return "skip"; + case result_error_system_error: + return "system error"; + case result_error_compression_error: + return "compression error"; + case result_error_decompression_error: + return "decompression error"; + case result_error_round_trip_error: + return "round trip error"; + } +} diff --git a/src/zstd/tests/regression/result.h b/src/zstd/tests/regression/result.h new file mode 100644 index 000000000..0085c2adf --- /dev/null +++ b/src/zstd/tests/regression/result.h @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef RESULT_H +#define RESULT_H + +#include <stddef.h> + +/** + * The error type enum. + */ +typedef enum { + result_error_ok, /**< No error. */ + result_error_skip, /**< This method was skipped. */ + result_error_system_error, /**< Some internal error happened. */ + result_error_compression_error, /**< Compression failed. */ + result_error_decompression_error, /**< Decompression failed. */ + result_error_round_trip_error, /**< Data failed to round trip. */ +} result_error_t; + +/** + * The success type. + */ +typedef struct { + size_t total_size; /**< The total compressed size. */ +} result_data_t; + +/** + * The result type. + * Do not access the member variables directory, use the helper functions. + */ +typedef struct { + result_error_t internal_error; + result_data_t internal_data; +} result_t; + +/** + * Create a result of the error type. + */ +static result_t result_error(result_error_t error); +/** + * Create a result of the success type. + */ +static result_t result_data(result_data_t data); + +/** + * Check if the result is an error or skip. + */ +static int result_is_error(result_t result); +/** + * Check if the result error is skip. + */ +static int result_is_skip(result_t result); +/** + * Get the result error or okay. + */ +static result_error_t result_get_error(result_t result); +/** + * Get the result data. The result MUST be checked with result_is_error() first. + */ +static result_data_t result_get_data(result_t result); + +static result_t result_error(result_error_t error) { + result_t result = { + .internal_error = error, + }; + return result; +} + +static result_t result_data(result_data_t data) { + result_t result = { + .internal_error = result_error_ok, + .internal_data = data, + }; + return result; +} + +static int result_is_error(result_t result) { + return result_get_error(result) != result_error_ok; +} + +static int result_is_skip(result_t result) { + return result_get_error(result) == result_error_skip; +} + +static result_error_t result_get_error(result_t result) { + return result.internal_error; +} + +char const* result_get_error_string(result_t result); + +static result_data_t result_get_data(result_t result) { + return result.internal_data; +} + +#endif diff --git a/src/zstd/tests/regression/results.csv b/src/zstd/tests/regression/results.csv new file mode 100644 index 000000000..4db42a488 --- /dev/null +++ b/src/zstd/tests/regression/results.csv @@ -0,0 +1,636 @@ +Data, Config, Method, Total compressed size +silesia.tar, level -5, compress simple, 6738558 +silesia.tar, level -3, compress simple, 6446362 +silesia.tar, level -1, compress simple, 6186038 +silesia.tar, level 0, compress simple, 4861374 +silesia.tar, level 1, compress simple, 5334825 +silesia.tar, level 3, compress simple, 4861374 +silesia.tar, level 4, compress simple, 4799583 +silesia.tar, level 5, compress simple, 4722271 +silesia.tar, level 6, compress simple, 4672231 +silesia.tar, level 7, compress simple, 4606657 +silesia.tar, level 9, compress simple, 4554099 +silesia.tar, level 13, compress simple, 4491706 +silesia.tar, level 16, compress simple, 4381265 +silesia.tar, level 19, compress simple, 4281551 +silesia.tar, uncompressed literals, compress simple, 4861374 +silesia.tar, uncompressed literals optimal, compress simple, 4281551 +silesia.tar, huffman literals, compress simple, 6186038 +silesia, level -5, compress cctx, 6737567 +silesia, level -3, compress cctx, 6444663 +silesia, level -1, compress cctx, 6178442 +silesia, level 0, compress cctx, 4849491 +silesia, level 1, compress cctx, 5313144 +silesia, level 3, compress cctx, 4849491 +silesia, level 4, compress cctx, 4786913 +silesia, level 5, compress cctx, 4710178 +silesia, level 6, compress cctx, 4659996 +silesia, level 7, compress cctx, 4596234 +silesia, level 9, compress cctx, 4543862 +silesia, level 13, compress cctx, 4482073 +silesia, level 16, compress cctx, 4377389 +silesia, level 19, compress cctx, 4293262 +silesia, long distance mode, compress cctx, 4849491 +silesia, multithreaded, compress cctx, 4849491 +silesia, multithreaded long distance mode, compress cctx, 4849491 +silesia, small window log, compress cctx, 7078156 +silesia, small hash log, compress cctx, 6554898 +silesia, small chain log, compress cctx, 4931093 +silesia, explicit params, compress cctx, 4794609 +silesia, uncompressed literals, compress cctx, 4849491 +silesia, uncompressed literals optimal, compress cctx, 4293262 +silesia, huffman literals, compress cctx, 6178442 +silesia, multithreaded with advanced params, compress cctx, 4849491 +github, level -5, compress cctx, 205285 +github, level -5 with dict, compress cctx, 47294 +github, level -3, compress cctx, 190643 +github, level -3 with dict, compress cctx, 48047 +github, level -1, compress cctx, 175568 +github, level -1 with dict, compress cctx, 43527 +github, level 0, compress cctx, 136311 +github, level 0 with dict, compress cctx, 41534 +github, level 1, compress cctx, 142450 +github, level 1 with dict, compress cctx, 42157 +github, level 3, compress cctx, 136311 +github, level 3 with dict, compress cctx, 41534 +github, level 4, compress cctx, 136144 +github, level 4 with dict, compress cctx, 41725 +github, level 5, compress cctx, 135106 +github, level 5 with dict, compress cctx, 38934 +github, level 6, compress cctx, 135108 +github, level 6 with dict, compress cctx, 38628 +github, level 7, compress cctx, 135108 +github, level 7 with dict, compress cctx, 38741 +github, level 9, compress cctx, 135108 +github, level 9 with dict, compress cctx, 39335 +github, level 13, compress cctx, 133717 +github, level 13 with dict, compress cctx, 39923 +github, level 16, compress cctx, 133717 +github, level 16 with dict, compress cctx, 37568 +github, level 19, compress cctx, 133717 +github, level 19 with dict, compress cctx, 37567 +github, long distance mode, compress cctx, 141101 +github, multithreaded, compress cctx, 141101 +github, multithreaded long distance mode, compress cctx, 141101 +github, small window log, compress cctx, 141101 +github, small hash log, compress cctx, 138943 +github, small chain log, compress cctx, 139239 +github, explicit params, compress cctx, 140924 +github, uncompressed literals, compress cctx, 136311 +github, uncompressed literals optimal, compress cctx, 133717 +github, huffman literals, compress cctx, 175568 +github, multithreaded with advanced params, compress cctx, 141101 +silesia, level -5, zstdcli, 6882514 +silesia, level -3, zstdcli, 6568406 +silesia, level -1, zstdcli, 6183433 +silesia, level 0, zstdcli, 4849539 +silesia, level 1, zstdcli, 5314157 +silesia, level 3, zstdcli, 4849539 +silesia, level 4, zstdcli, 4786961 +silesia, level 5, zstdcli, 4710226 +silesia, level 6, zstdcli, 4660044 +silesia, level 7, zstdcli, 4596282 +silesia, level 9, zstdcli, 4543910 +silesia, level 13, zstdcli, 4482121 +silesia, level 16, zstdcli, 4377437 +silesia, level 19, zstdcli, 4293310 +silesia, long distance mode, zstdcli, 4839698 +silesia, multithreaded, zstdcli, 4849539 +silesia, multithreaded long distance mode, zstdcli, 4839698 +silesia, small window log, zstdcli, 7104616 +silesia, small hash log, zstdcli, 6554946 +silesia, small chain log, zstdcli, 4931141 +silesia, explicit params, zstdcli, 4797048 +silesia, uncompressed literals, zstdcli, 5128008 +silesia, uncompressed literals optimal, zstdcli, 4325482 +silesia, huffman literals, zstdcli, 5331158 +silesia, multithreaded with advanced params, zstdcli, 5128008 +silesia.tar, level -5, zstdcli, 6738906 +silesia.tar, level -3, zstdcli, 6448409 +silesia.tar, level -1, zstdcli, 6186908 +silesia.tar, level 0, zstdcli, 4861462 +silesia.tar, level 1, zstdcli, 5336255 +silesia.tar, level 3, zstdcli, 4861462 +silesia.tar, level 4, zstdcli, 4800482 +silesia.tar, level 5, zstdcli, 4723312 +silesia.tar, level 6, zstdcli, 4673616 +silesia.tar, level 7, zstdcli, 4608346 +silesia.tar, level 9, zstdcli, 4554702 +silesia.tar, level 13, zstdcli, 4491710 +silesia.tar, level 16, zstdcli, 4381269 +silesia.tar, level 19, zstdcli, 4281555 +silesia.tar, no source size, zstdcli, 4861458 +silesia.tar, long distance mode, zstdcli, 4853140 +silesia.tar, multithreaded, zstdcli, 4861462 +silesia.tar, multithreaded long distance mode, zstdcli, 4853140 +silesia.tar, small window log, zstdcli, 7095284 +silesia.tar, small hash log, zstdcli, 6587841 +silesia.tar, small chain log, zstdcli, 4943269 +silesia.tar, explicit params, zstdcli, 4822318 +silesia.tar, uncompressed literals, zstdcli, 5129548 +silesia.tar, uncompressed literals optimal, zstdcli, 4320914 +silesia.tar, huffman literals, zstdcli, 5347560 +silesia.tar, multithreaded with advanced params, zstdcli, 5129548 +github, level -5, zstdcli, 207285 +github, level -5 with dict, zstdcli, 48718 +github, level -3, zstdcli, 192643 +github, level -3 with dict, zstdcli, 47395 +github, level -1, zstdcli, 177568 +github, level -1 with dict, zstdcli, 45170 +github, level 0, zstdcli, 138311 +github, level 0 with dict, zstdcli, 43148 +github, level 1, zstdcli, 144450 +github, level 1 with dict, zstdcli, 43682 +github, level 3, zstdcli, 138311 +github, level 3 with dict, zstdcli, 43148 +github, level 4, zstdcli, 138144 +github, level 4 with dict, zstdcli, 43251 +github, level 5, zstdcli, 137106 +github, level 5 with dict, zstdcli, 40938 +github, level 6, zstdcli, 137108 +github, level 6 with dict, zstdcli, 40632 +github, level 7, zstdcli, 137108 +github, level 7 with dict, zstdcli, 40766 +github, level 9, zstdcli, 137108 +github, level 9 with dict, zstdcli, 41326 +github, level 13, zstdcli, 135717 +github, level 13 with dict, zstdcli, 41716 +github, level 16, zstdcli, 135717 +github, level 16 with dict, zstdcli, 39577 +github, level 19, zstdcli, 135717 +github, level 19 with dict, zstdcli, 39576 +github, long distance mode, zstdcli, 138311 +github, multithreaded, zstdcli, 138311 +github, multithreaded long distance mode, zstdcli, 138311 +github, small window log, zstdcli, 138311 +github, small hash log, zstdcli, 137467 +github, small chain log, zstdcli, 138314 +github, explicit params, zstdcli, 136140 +github, uncompressed literals, zstdcli, 167915 +github, uncompressed literals optimal, zstdcli, 158824 +github, huffman literals, zstdcli, 144450 +github, multithreaded with advanced params, zstdcli, 167915 +silesia, level -5, advanced one pass, 6737567 +silesia, level -3, advanced one pass, 6444663 +silesia, level -1, advanced one pass, 6178442 +silesia, level 0, advanced one pass, 4849491 +silesia, level 1, advanced one pass, 5313144 +silesia, level 3, advanced one pass, 4849491 +silesia, level 4, advanced one pass, 4786913 +silesia, level 5, advanced one pass, 4710178 +silesia, level 6, advanced one pass, 4659996 +silesia, level 7, advanced one pass, 4596234 +silesia, level 9, advanced one pass, 4543862 +silesia, level 13, advanced one pass, 4482073 +silesia, level 16, advanced one pass, 4377389 +silesia, level 19, advanced one pass, 4293262 +silesia, no source size, advanced one pass, 4849491 +silesia, long distance mode, advanced one pass, 4839650 +silesia, multithreaded, advanced one pass, 4849491 +silesia, multithreaded long distance mode, advanced one pass, 4839650 +silesia, small window log, advanced one pass, 7089646 +silesia, small hash log, advanced one pass, 6554898 +silesia, small chain log, advanced one pass, 4931093 +silesia, explicit params, advanced one pass, 4797035 +silesia, uncompressed literals, advanced one pass, 5127960 +silesia, uncompressed literals optimal, advanced one pass, 4325434 +silesia, huffman literals, advanced one pass, 5326210 +silesia, multithreaded with advanced params, advanced one pass, 5127960 +silesia.tar, level -5, advanced one pass, 6738558 +silesia.tar, level -3, advanced one pass, 6446362 +silesia.tar, level -1, advanced one pass, 6186038 +silesia.tar, level 0, advanced one pass, 4861374 +silesia.tar, level 1, advanced one pass, 5334825 +silesia.tar, level 3, advanced one pass, 4861374 +silesia.tar, level 4, advanced one pass, 4799583 +silesia.tar, level 5, advanced one pass, 4722271 +silesia.tar, level 6, advanced one pass, 4672231 +silesia.tar, level 7, advanced one pass, 4606657 +silesia.tar, level 9, advanced one pass, 4554099 +silesia.tar, level 13, advanced one pass, 4491706 +silesia.tar, level 16, advanced one pass, 4381265 +silesia.tar, level 19, advanced one pass, 4281551 +silesia.tar, no source size, advanced one pass, 4861374 +silesia.tar, long distance mode, advanced one pass, 4848046 +silesia.tar, multithreaded, advanced one pass, 4860726 +silesia.tar, multithreaded long distance mode, advanced one pass, 4847343 +silesia.tar, small window log, advanced one pass, 7095237 +silesia.tar, small hash log, advanced one pass, 6587833 +silesia.tar, small chain log, advanced one pass, 4943266 +silesia.tar, explicit params, advanced one pass, 4808543 +silesia.tar, uncompressed literals, advanced one pass, 5129447 +silesia.tar, uncompressed literals optimal, advanced one pass, 4320910 +silesia.tar, huffman literals, advanced one pass, 5347283 +silesia.tar, multithreaded with advanced params, advanced one pass, 5129766 +github, level -5, advanced one pass, 205285 +github, level -5 with dict, advanced one pass, 46718 +github, level -3, advanced one pass, 190643 +github, level -3 with dict, advanced one pass, 45395 +github, level -1, advanced one pass, 175568 +github, level -1 with dict, advanced one pass, 43170 +github, level 0, advanced one pass, 136311 +github, level 0 with dict, advanced one pass, 41148 +github, level 1, advanced one pass, 142450 +github, level 1 with dict, advanced one pass, 41682 +github, level 3, advanced one pass, 136311 +github, level 3 with dict, advanced one pass, 41148 +github, level 4, advanced one pass, 136144 +github, level 4 with dict, advanced one pass, 41251 +github, level 5, advanced one pass, 135106 +github, level 5 with dict, advanced one pass, 38938 +github, level 6, advanced one pass, 135108 +github, level 6 with dict, advanced one pass, 38632 +github, level 7, advanced one pass, 135108 +github, level 7 with dict, advanced one pass, 38766 +github, level 9, advanced one pass, 135108 +github, level 9 with dict, advanced one pass, 39326 +github, level 13, advanced one pass, 133717 +github, level 13 with dict, advanced one pass, 39716 +github, level 16, advanced one pass, 133717 +github, level 16 with dict, advanced one pass, 37577 +github, level 19, advanced one pass, 133717 +github, level 19 with dict, advanced one pass, 37576 +github, no source size, advanced one pass, 136311 +github, long distance mode, advanced one pass, 136311 +github, multithreaded, advanced one pass, 136311 +github, multithreaded long distance mode, advanced one pass, 136311 +github, small window log, advanced one pass, 136311 +github, small hash log, advanced one pass, 135467 +github, small chain log, advanced one pass, 136314 +github, explicit params, advanced one pass, 137670 +github, uncompressed literals, advanced one pass, 165915 +github, uncompressed literals optimal, advanced one pass, 156824 +github, huffman literals, advanced one pass, 142450 +github, multithreaded with advanced params, advanced one pass, 165915 +silesia, level -5, advanced one pass small out, 6737567 +silesia, level -3, advanced one pass small out, 6444663 +silesia, level -1, advanced one pass small out, 6178442 +silesia, level 0, advanced one pass small out, 4849491 +silesia, level 1, advanced one pass small out, 5313144 +silesia, level 3, advanced one pass small out, 4849491 +silesia, level 4, advanced one pass small out, 4786913 +silesia, level 5, advanced one pass small out, 4710178 +silesia, level 6, advanced one pass small out, 4659996 +silesia, level 7, advanced one pass small out, 4596234 +silesia, level 9, advanced one pass small out, 4543862 +silesia, level 13, advanced one pass small out, 4482073 +silesia, level 16, advanced one pass small out, 4377389 +silesia, level 19, advanced one pass small out, 4293262 +silesia, no source size, advanced one pass small out, 4849491 +silesia, long distance mode, advanced one pass small out, 4839650 +silesia, multithreaded, advanced one pass small out, 4849491 +silesia, multithreaded long distance mode, advanced one pass small out, 4839650 +silesia, small window log, advanced one pass small out, 7089646 +silesia, small hash log, advanced one pass small out, 6554898 +silesia, small chain log, advanced one pass small out, 4931093 +silesia, explicit params, advanced one pass small out, 4797035 +silesia, uncompressed literals, advanced one pass small out, 5127960 +silesia, uncompressed literals optimal, advanced one pass small out, 4325434 +silesia, huffman literals, advanced one pass small out, 5326210 +silesia, multithreaded with advanced params, advanced one pass small out, 5127960 +silesia.tar, level -5, advanced one pass small out, 6738558 +silesia.tar, level -3, advanced one pass small out, 6446362 +silesia.tar, level -1, advanced one pass small out, 6186038 +silesia.tar, level 0, advanced one pass small out, 4861374 +silesia.tar, level 1, advanced one pass small out, 5334825 +silesia.tar, level 3, advanced one pass small out, 4861374 +silesia.tar, level 4, advanced one pass small out, 4799583 +silesia.tar, level 5, advanced one pass small out, 4722271 +silesia.tar, level 6, advanced one pass small out, 4672231 +silesia.tar, level 7, advanced one pass small out, 4606657 +silesia.tar, level 9, advanced one pass small out, 4554099 +silesia.tar, level 13, advanced one pass small out, 4491706 +silesia.tar, level 16, advanced one pass small out, 4381265 +silesia.tar, level 19, advanced one pass small out, 4281551 +silesia.tar, no source size, advanced one pass small out, 4861374 +silesia.tar, long distance mode, advanced one pass small out, 4848046 +silesia.tar, multithreaded, advanced one pass small out, 4860726 +silesia.tar, multithreaded long distance mode, advanced one pass small out, 4847343 +silesia.tar, small window log, advanced one pass small out, 7095237 +silesia.tar, small hash log, advanced one pass small out, 6587833 +silesia.tar, small chain log, advanced one pass small out, 4943266 +silesia.tar, explicit params, advanced one pass small out, 4808543 +silesia.tar, uncompressed literals, advanced one pass small out, 5129447 +silesia.tar, uncompressed literals optimal, advanced one pass small out, 4320910 +silesia.tar, huffman literals, advanced one pass small out, 5347283 +silesia.tar, multithreaded with advanced params, advanced one pass small out, 5129766 +github, level -5, advanced one pass small out, 205285 +github, level -5 with dict, advanced one pass small out, 46718 +github, level -3, advanced one pass small out, 190643 +github, level -3 with dict, advanced one pass small out, 45395 +github, level -1, advanced one pass small out, 175568 +github, level -1 with dict, advanced one pass small out, 43170 +github, level 0, advanced one pass small out, 136311 +github, level 0 with dict, advanced one pass small out, 41148 +github, level 1, advanced one pass small out, 142450 +github, level 1 with dict, advanced one pass small out, 41682 +github, level 3, advanced one pass small out, 136311 +github, level 3 with dict, advanced one pass small out, 41148 +github, level 4, advanced one pass small out, 136144 +github, level 4 with dict, advanced one pass small out, 41251 +github, level 5, advanced one pass small out, 135106 +github, level 5 with dict, advanced one pass small out, 38938 +github, level 6, advanced one pass small out, 135108 +github, level 6 with dict, advanced one pass small out, 38632 +github, level 7, advanced one pass small out, 135108 +github, level 7 with dict, advanced one pass small out, 38766 +github, level 9, advanced one pass small out, 135108 +github, level 9 with dict, advanced one pass small out, 39326 +github, level 13, advanced one pass small out, 133717 +github, level 13 with dict, advanced one pass small out, 39716 +github, level 16, advanced one pass small out, 133717 +github, level 16 with dict, advanced one pass small out, 37577 +github, level 19, advanced one pass small out, 133717 +github, level 19 with dict, advanced one pass small out, 37576 +github, no source size, advanced one pass small out, 136311 +github, long distance mode, advanced one pass small out, 136311 +github, multithreaded, advanced one pass small out, 136311 +github, multithreaded long distance mode, advanced one pass small out, 136311 +github, small window log, advanced one pass small out, 136311 +github, small hash log, advanced one pass small out, 135467 +github, small chain log, advanced one pass small out, 136314 +github, explicit params, advanced one pass small out, 137670 +github, uncompressed literals, advanced one pass small out, 165915 +github, uncompressed literals optimal, advanced one pass small out, 156824 +github, huffman literals, advanced one pass small out, 142450 +github, multithreaded with advanced params, advanced one pass small out, 165915 +silesia, level -5, advanced streaming, 6882466 +silesia, level -3, advanced streaming, 6568358 +silesia, level -1, advanced streaming, 6183385 +silesia, level 0, advanced streaming, 4849491 +silesia, level 1, advanced streaming, 5314109 +silesia, level 3, advanced streaming, 4849491 +silesia, level 4, advanced streaming, 4786913 +silesia, level 5, advanced streaming, 4710178 +silesia, level 6, advanced streaming, 4659996 +silesia, level 7, advanced streaming, 4596234 +silesia, level 9, advanced streaming, 4543862 +silesia, level 13, advanced streaming, 4482073 +silesia, level 16, advanced streaming, 4377389 +silesia, level 19, advanced streaming, 4293262 +silesia, no source size, advanced streaming, 4849455 +silesia, long distance mode, advanced streaming, 4839650 +silesia, multithreaded, advanced streaming, 4849491 +silesia, multithreaded long distance mode, advanced streaming, 4839650 +silesia, small window log, advanced streaming, 7105714 +silesia, small hash log, advanced streaming, 6554898 +silesia, small chain log, advanced streaming, 4931093 +silesia, explicit params, advanced streaming, 4797048 +silesia, uncompressed literals, advanced streaming, 5127960 +silesia, uncompressed literals optimal, advanced streaming, 4325434 +silesia, huffman literals, advanced streaming, 5331110 +silesia, multithreaded with advanced params, advanced streaming, 5127960 +silesia.tar, level -5, advanced streaming, 6982738 +silesia.tar, level -3, advanced streaming, 6641264 +silesia.tar, level -1, advanced streaming, 6190789 +silesia.tar, level 0, advanced streaming, 4861376 +silesia.tar, level 1, advanced streaming, 5336879 +silesia.tar, level 3, advanced streaming, 4861376 +silesia.tar, level 4, advanced streaming, 4799583 +silesia.tar, level 5, advanced streaming, 4722276 +silesia.tar, level 6, advanced streaming, 4672240 +silesia.tar, level 7, advanced streaming, 4606657 +silesia.tar, level 9, advanced streaming, 4554106 +silesia.tar, level 13, advanced streaming, 4491707 +silesia.tar, level 16, advanced streaming, 4381284 +silesia.tar, level 19, advanced streaming, 4281511 +silesia.tar, no source size, advanced streaming, 4861372 +silesia.tar, long distance mode, advanced streaming, 4848046 +silesia.tar, multithreaded, advanced streaming, 4861458 +silesia.tar, multithreaded long distance mode, advanced streaming, 4853136 +silesia.tar, small window log, advanced streaming, 7112148 +silesia.tar, small hash log, advanced streaming, 6587834 +silesia.tar, small chain log, advanced streaming, 4943271 +silesia.tar, explicit params, advanced streaming, 4808570 +silesia.tar, uncompressed literals, advanced streaming, 5129450 +silesia.tar, uncompressed literals optimal, advanced streaming, 4320841 +silesia.tar, huffman literals, advanced streaming, 5352306 +silesia.tar, multithreaded with advanced params, advanced streaming, 5129544 +github, level -5, advanced streaming, 205285 +github, level -5 with dict, advanced streaming, 46718 +github, level -3, advanced streaming, 190643 +github, level -3 with dict, advanced streaming, 45395 +github, level -1, advanced streaming, 175568 +github, level -1 with dict, advanced streaming, 43170 +github, level 0, advanced streaming, 136311 +github, level 0 with dict, advanced streaming, 41148 +github, level 1, advanced streaming, 142450 +github, level 1 with dict, advanced streaming, 41682 +github, level 3, advanced streaming, 136311 +github, level 3 with dict, advanced streaming, 41148 +github, level 4, advanced streaming, 136144 +github, level 4 with dict, advanced streaming, 41251 +github, level 5, advanced streaming, 135106 +github, level 5 with dict, advanced streaming, 38938 +github, level 6, advanced streaming, 135108 +github, level 6 with dict, advanced streaming, 38632 +github, level 7, advanced streaming, 135108 +github, level 7 with dict, advanced streaming, 38766 +github, level 9, advanced streaming, 135108 +github, level 9 with dict, advanced streaming, 39326 +github, level 13, advanced streaming, 133717 +github, level 13 with dict, advanced streaming, 39716 +github, level 16, advanced streaming, 133717 +github, level 16 with dict, advanced streaming, 37577 +github, level 19, advanced streaming, 133717 +github, level 19 with dict, advanced streaming, 37576 +github, no source size, advanced streaming, 136311 +github, long distance mode, advanced streaming, 136311 +github, multithreaded, advanced streaming, 136311 +github, multithreaded long distance mode, advanced streaming, 136311 +github, small window log, advanced streaming, 136311 +github, small hash log, advanced streaming, 135467 +github, small chain log, advanced streaming, 136314 +github, explicit params, advanced streaming, 137670 +github, uncompressed literals, advanced streaming, 165915 +github, uncompressed literals optimal, advanced streaming, 156824 +github, huffman literals, advanced streaming, 142450 +github, multithreaded with advanced params, advanced streaming, 165915 +silesia, level -5, old streaming, 6882466 +silesia, level -3, old streaming, 6568358 +silesia, level -1, old streaming, 6183385 +silesia, level 0, old streaming, 4849491 +silesia, level 1, old streaming, 5314109 +silesia, level 3, old streaming, 4849491 +silesia, level 4, old streaming, 4786913 +silesia, level 5, old streaming, 4710178 +silesia, level 6, old streaming, 4659996 +silesia, level 7, old streaming, 4596234 +silesia, level 9, old streaming, 4543862 +silesia, level 13, old streaming, 4482073 +silesia, level 16, old streaming, 4377389 +silesia, level 19, old streaming, 4293262 +silesia, no source size, old streaming, 4849455 +silesia, uncompressed literals, old streaming, 4849491 +silesia, uncompressed literals optimal, old streaming, 4293262 +silesia, huffman literals, old streaming, 6183385 +silesia.tar, level -5, old streaming, 6982738 +silesia.tar, level -3, old streaming, 6641264 +silesia.tar, level -1, old streaming, 6190789 +silesia.tar, level 0, old streaming, 4861376 +silesia.tar, level 1, old streaming, 5336879 +silesia.tar, level 3, old streaming, 4861376 +silesia.tar, level 4, old streaming, 4799583 +silesia.tar, level 5, old streaming, 4722276 +silesia.tar, level 6, old streaming, 4672240 +silesia.tar, level 7, old streaming, 4606657 +silesia.tar, level 9, old streaming, 4554106 +silesia.tar, level 13, old streaming, 4491707 +silesia.tar, level 16, old streaming, 4381284 +silesia.tar, level 19, old streaming, 4281511 +silesia.tar, no source size, old streaming, 4861372 +silesia.tar, uncompressed literals, old streaming, 4861376 +silesia.tar, uncompressed literals optimal, old streaming, 4281511 +silesia.tar, huffman literals, old streaming, 6190789 +github, level -5, old streaming, 205285 +github, level -5 with dict, old streaming, 46718 +github, level -3, old streaming, 190643 +github, level -3 with dict, old streaming, 45395 +github, level -1, old streaming, 175568 +github, level -1 with dict, old streaming, 43170 +github, level 0, old streaming, 136311 +github, level 0 with dict, old streaming, 41148 +github, level 1, old streaming, 142450 +github, level 1 with dict, old streaming, 41682 +github, level 3, old streaming, 136311 +github, level 3 with dict, old streaming, 41148 +github, level 4, old streaming, 136144 +github, level 4 with dict, old streaming, 41251 +github, level 5, old streaming, 135106 +github, level 5 with dict, old streaming, 38938 +github, level 6, old streaming, 135108 +github, level 6 with dict, old streaming, 38632 +github, level 7, old streaming, 135108 +github, level 7 with dict, old streaming, 38766 +github, level 9, old streaming, 135108 +github, level 9 with dict, old streaming, 39326 +github, level 13, old streaming, 133717 +github, level 13 with dict, old streaming, 39716 +github, level 16, old streaming, 133717 +github, level 16 with dict, old streaming, 37577 +github, level 19, old streaming, 133717 +github, level 19 with dict, old streaming, 37576 +github, no source size, old streaming, 140631 +github, uncompressed literals, old streaming, 136311 +github, uncompressed literals optimal, old streaming, 133717 +github, huffman literals, old streaming, 175568 +silesia, level -5, old streaming advanced, 6882466 +silesia, level -3, old streaming advanced, 6568358 +silesia, level -1, old streaming advanced, 6183385 +silesia, level 0, old streaming advanced, 4849491 +silesia, level 1, old streaming advanced, 5314109 +silesia, level 3, old streaming advanced, 4849491 +silesia, level 4, old streaming advanced, 4786913 +silesia, level 5, old streaming advanced, 4710178 +silesia, level 6, old streaming advanced, 4659996 +silesia, level 7, old streaming advanced, 4596234 +silesia, level 9, old streaming advanced, 4543862 +silesia, level 13, old streaming advanced, 4482073 +silesia, level 16, old streaming advanced, 4377389 +silesia, level 19, old streaming advanced, 4293262 +silesia, no source size, old streaming advanced, 4849455 +silesia, long distance mode, old streaming advanced, 4849491 +silesia, multithreaded, old streaming advanced, 4849491 +silesia, multithreaded long distance mode, old streaming advanced, 4849491 +silesia, small window log, old streaming advanced, 7105714 +silesia, small hash log, old streaming advanced, 6554898 +silesia, small chain log, old streaming advanced, 4931093 +silesia, explicit params, old streaming advanced, 4797048 +silesia, uncompressed literals, old streaming advanced, 4849491 +silesia, uncompressed literals optimal, old streaming advanced, 4293262 +silesia, huffman literals, old streaming advanced, 6183385 +silesia, multithreaded with advanced params, old streaming advanced, 4849491 +silesia.tar, level -5, old streaming advanced, 6982738 +silesia.tar, level -3, old streaming advanced, 6641264 +silesia.tar, level -1, old streaming advanced, 6190789 +silesia.tar, level 0, old streaming advanced, 4861376 +silesia.tar, level 1, old streaming advanced, 5336879 +silesia.tar, level 3, old streaming advanced, 4861376 +silesia.tar, level 4, old streaming advanced, 4799583 +silesia.tar, level 5, old streaming advanced, 4722276 +silesia.tar, level 6, old streaming advanced, 4672240 +silesia.tar, level 7, old streaming advanced, 4606657 +silesia.tar, level 9, old streaming advanced, 4554106 +silesia.tar, level 13, old streaming advanced, 4491707 +silesia.tar, level 16, old streaming advanced, 4381284 +silesia.tar, level 19, old streaming advanced, 4281511 +silesia.tar, no source size, old streaming advanced, 4861372 +silesia.tar, long distance mode, old streaming advanced, 4861376 +silesia.tar, multithreaded, old streaming advanced, 4861376 +silesia.tar, multithreaded long distance mode, old streaming advanced, 4861376 +silesia.tar, small window log, old streaming advanced, 7112151 +silesia.tar, small hash log, old streaming advanced, 6587834 +silesia.tar, small chain log, old streaming advanced, 4943271 +silesia.tar, explicit params, old streaming advanced, 4808570 +silesia.tar, uncompressed literals, old streaming advanced, 4861376 +silesia.tar, uncompressed literals optimal, old streaming advanced, 4281511 +silesia.tar, huffman literals, old streaming advanced, 6190789 +silesia.tar, multithreaded with advanced params, old streaming advanced, 4861376 +github, level -5, old streaming advanced, 216734 +github, level -5 with dict, old streaming advanced, 49562 +github, level -3, old streaming advanced, 192160 +github, level -3 with dict, old streaming advanced, 44956 +github, level -1, old streaming advanced, 181108 +github, level -1 with dict, old streaming advanced, 42383 +github, level 0, old streaming advanced, 141090 +github, level 0 with dict, old streaming advanced, 41113 +github, level 1, old streaming advanced, 143682 +github, level 1 with dict, old streaming advanced, 42430 +github, level 3, old streaming advanced, 141090 +github, level 3 with dict, old streaming advanced, 41113 +github, level 4, old streaming advanced, 141090 +github, level 4 with dict, old streaming advanced, 41084 +github, level 5, old streaming advanced, 139391 +github, level 5 with dict, old streaming advanced, 39159 +github, level 6, old streaming advanced, 139394 +github, level 6 with dict, old streaming advanced, 38749 +github, level 7, old streaming advanced, 138675 +github, level 7 with dict, old streaming advanced, 38746 +github, level 9, old streaming advanced, 138675 +github, level 9 with dict, old streaming advanced, 38987 +github, level 13, old streaming advanced, 138675 +github, level 13 with dict, old streaming advanced, 39724 +github, level 16, old streaming advanced, 138675 +github, level 16 with dict, old streaming advanced, 40771 +github, level 19, old streaming advanced, 133717 +github, level 19 with dict, old streaming advanced, 37576 +github, no source size, old streaming advanced, 140631 +github, long distance mode, old streaming advanced, 141090 +github, multithreaded, old streaming advanced, 141090 +github, multithreaded long distance mode, old streaming advanced, 141090 +github, small window log, old streaming advanced, 141090 +github, small hash log, old streaming advanced, 141578 +github, small chain log, old streaming advanced, 139258 +github, explicit params, old streaming advanced, 140930 +github, uncompressed literals, old streaming advanced, 141090 +github, uncompressed literals optimal, old streaming advanced, 133717 +github, huffman literals, old streaming advanced, 181108 +github, multithreaded with advanced params, old streaming advanced, 141090 +github, level -5 with dict, old streaming cdcit, 46718 +github, level -3 with dict, old streaming cdcit, 45395 +github, level -1 with dict, old streaming cdcit, 43170 +github, level 0 with dict, old streaming cdcit, 41148 +github, level 1 with dict, old streaming cdcit, 41682 +github, level 3 with dict, old streaming cdcit, 41148 +github, level 4 with dict, old streaming cdcit, 41251 +github, level 5 with dict, old streaming cdcit, 38938 +github, level 6 with dict, old streaming cdcit, 38632 +github, level 7 with dict, old streaming cdcit, 38766 +github, level 9 with dict, old streaming cdcit, 39326 +github, level 13 with dict, old streaming cdcit, 39716 +github, level 16 with dict, old streaming cdcit, 37577 +github, level 19 with dict, old streaming cdcit, 37576 +github, level -5 with dict, old streaming advanced cdict, 49562 +github, level -3 with dict, old streaming advanced cdict, 44956 +github, level -1 with dict, old streaming advanced cdict, 42383 +github, level 0 with dict, old streaming advanced cdict, 41113 +github, level 1 with dict, old streaming advanced cdict, 42430 +github, level 3 with dict, old streaming advanced cdict, 41113 +github, level 4 with dict, old streaming advanced cdict, 41084 +github, level 5 with dict, old streaming advanced cdict, 39158 +github, level 6 with dict, old streaming advanced cdict, 38748 +github, level 7 with dict, old streaming advanced cdict, 38744 +github, level 9 with dict, old streaming advanced cdict, 38986 +github, level 13 with dict, old streaming advanced cdict, 39724 +github, level 16 with dict, old streaming advanced cdict, 40771 +github, level 19 with dict, old streaming advanced cdict, 37576 diff --git a/src/zstd/tests/regression/test.c b/src/zstd/tests/regression/test.c new file mode 100644 index 000000000..ff2cdba30 --- /dev/null +++ b/src/zstd/tests/regression/test.c @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2016-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include <assert.h> +#include <getopt.h> +#include <stdio.h> +#include <string.h> + +#include "config.h" +#include "data.h" +#include "method.h" + +static int g_max_name_len = 0; + +/** Check if a name contains a comma or is too long. */ +static int is_name_bad(char const* name) { + if (name == NULL) + return 1; + int const len = strlen(name); + if (len > g_max_name_len) + g_max_name_len = len; + for (; *name != '\0'; ++name) + if (*name == ',') + return 1; + return 0; +} + +/** Check if any of the names contain a comma. */ +static int are_names_bad() { + for (size_t method = 0; methods[method] != NULL; ++method) + if (is_name_bad(methods[method]->name)) { + fprintf(stderr, "method name %s is bad\n", methods[method]->name); + return 1; + } + for (size_t datum = 0; data[datum] != NULL; ++datum) + if (is_name_bad(data[datum]->name)) { + fprintf(stderr, "data name %s is bad\n", data[datum]->name); + return 1; + } + for (size_t config = 0; configs[config] != NULL; ++config) + if (is_name_bad(configs[config]->name)) { + fprintf(stderr, "config name %s is bad\n", configs[config]->name); + return 1; + } + return 0; +} + +/** + * Option parsing using getopt. + * When you add a new option update: long_options, long_extras, and + * short_options. + */ + +/** Option variables filled by parse_args. */ +static char const* g_output = NULL; +static char const* g_diff = NULL; +static char const* g_cache = NULL; +static char const* g_zstdcli = NULL; +static char const* g_config = NULL; +static char const* g_data = NULL; +static char const* g_method = NULL; + +typedef enum { + required_option, + optional_option, + help_option, +} option_type; + +/** + * Extra state that we need to keep per-option that we can't store in getopt. + */ +struct option_extra { + int id; /**< The short option name, used as an id. */ + char const* help; /**< The help message. */ + option_type opt_type; /**< The option type: required, optional, or help. */ + char const** value; /**< The value to set or NULL if no_argument. */ +}; + +/** The options. */ +static struct option long_options[] = { + {"cache", required_argument, NULL, 'c'}, + {"output", required_argument, NULL, 'o'}, + {"zstd", required_argument, NULL, 'z'}, + {"config", required_argument, NULL, 128}, + {"data", required_argument, NULL, 129}, + {"method", required_argument, NULL, 130}, + {"diff", required_argument, NULL, 'd'}, + {"help", no_argument, NULL, 'h'}, +}; + +static size_t const nargs = sizeof(long_options) / sizeof(long_options[0]); + +/** The extra info for the options. Must be in the same order as the options. */ +static struct option_extra long_extras[] = { + {'c', "the cache directory", required_option, &g_cache}, + {'o', "write the results here", required_option, &g_output}, + {'z', "zstd cli tool", required_option, &g_zstdcli}, + {128, "use this config", optional_option, &g_config}, + {129, "use this data", optional_option, &g_data}, + {130, "use this method", optional_option, &g_method}, + {'d', "compare the results to this file", optional_option, &g_diff}, + {'h', "display this message", help_option, NULL}, +}; + +/** The short options. Must correspond to the options. */ +static char const short_options[] = "c:d:ho:z:"; + +/** Return the help string for the option type. */ +static char const* required_message(option_type opt_type) { + switch (opt_type) { + case required_option: + return "[required]"; + case optional_option: + return "[optional]"; + case help_option: + return ""; + default: + assert(0); + return NULL; + } +} + +/** Print the help for the program. */ +static void print_help(void) { + fprintf(stderr, "regression test runner\n"); + size_t const nargs = sizeof(long_options) / sizeof(long_options[0]); + for (size_t i = 0; i < nargs; ++i) { + if (long_options[i].val < 128) { + /* Long / short - help [option type] */ + fprintf( + stderr, + "--%s / -%c \t- %s %s\n", + long_options[i].name, + long_options[i].val, + long_extras[i].help, + required_message(long_extras[i].opt_type)); + } else { + /* Short / long - help [option type] */ + fprintf( + stderr, + "--%s \t- %s %s\n", + long_options[i].name, + long_extras[i].help, + required_message(long_extras[i].opt_type)); + } + } +} + +/** Parse the arguments. Return 0 on success. Print help on failure. */ +static int parse_args(int argc, char** argv) { + int option_index = 0; + int c; + + while (1) { + c = getopt_long(argc, argv, short_options, long_options, &option_index); + if (c == -1) + break; + + int found = 0; + for (size_t i = 0; i < nargs; ++i) { + if (c == long_extras[i].id && long_extras[i].value != NULL) { + *long_extras[i].value = optarg; + found = 1; + break; + } + } + if (found) + continue; + + switch (c) { + case 'h': + case '?': + default: + print_help(); + return 1; + } + } + + int bad = 0; + for (size_t i = 0; i < nargs; ++i) { + if (long_extras[i].opt_type != required_option) + continue; + if (long_extras[i].value == NULL) + continue; + if (*long_extras[i].value != NULL) + continue; + fprintf( + stderr, + "--%s is a required argument but is not set\n", + long_options[i].name); + bad = 1; + } + if (bad) { + fprintf(stderr, "\n"); + print_help(); + return 1; + } + + return 0; +} + +/** Helper macro to print to stderr and a file. */ +#define tprintf(file, ...) \ + do { \ + fprintf(file, __VA_ARGS__); \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) +/** Helper macro to flush stderr and a file. */ +#define tflush(file) \ + do { \ + fflush(file); \ + fflush(stderr); \ + } while (0) + +void tprint_names( + FILE* results, + char const* data_name, + char const* config_name, + char const* method_name) { + int const data_padding = g_max_name_len - strlen(data_name); + int const config_padding = g_max_name_len - strlen(config_name); + int const method_padding = g_max_name_len - strlen(method_name); + + tprintf( + results, + "%s, %*s%s, %*s%s, %*s", + data_name, + data_padding, + "", + config_name, + config_padding, + "", + method_name, + method_padding, + ""); +} + +/** + * Run all the regression tests and record the results table to results and + * stderr progressively. + */ +static int run_all(FILE* results) { + tprint_names(results, "Data", "Config", "Method"); + tprintf(results, "Total compressed size\n"); + for (size_t method = 0; methods[method] != NULL; ++method) { + if (g_method != NULL && strcmp(methods[method]->name, g_method)) + continue; + for (size_t datum = 0; data[datum] != NULL; ++datum) { + if (g_data != NULL && strcmp(data[datum]->name, g_data)) + continue; + /* Create the state common to all configs */ + method_state_t* state = methods[method]->create(data[datum]); + for (size_t config = 0; configs[config] != NULL; ++config) { + if (g_config != NULL && strcmp(configs[config]->name, g_config)) + continue; + if (config_skip_data(configs[config], data[datum])) + continue; + /* Print the result for the (method, data, config) tuple. */ + result_t const result = + methods[method]->compress(state, configs[config]); + if (result_is_skip(result)) + continue; + tprint_names( + results, + data[datum]->name, + configs[config]->name, + methods[method]->name); + if (result_is_error(result)) { + tprintf(results, "%s\n", result_get_error_string(result)); + } else { + tprintf( + results, + "%llu\n", + (unsigned long long)result_get_data(result).total_size); + } + tflush(results); + } + methods[method]->destroy(state); + } + } + return 0; +} + +/** memcmp() the old results file and the new results file. */ +static int diff_results(char const* actual_file, char const* expected_file) { + data_buffer_t const actual = data_buffer_read(actual_file); + data_buffer_t const expected = data_buffer_read(expected_file); + int ret = 1; + + if (actual.data == NULL) { + fprintf(stderr, "failed to open results '%s' for diff\n", actual_file); + goto out; + } + if (expected.data == NULL) { + fprintf( + stderr, + "failed to open previous results '%s' for diff\n", + expected_file); + goto out; + } + + ret = data_buffer_compare(actual, expected); + if (ret != 0) { + fprintf( + stderr, + "actual results '%s' does not match expected results '%s'\n", + actual_file, + expected_file); + } else { + fprintf(stderr, "actual results match expected results\n"); + } +out: + data_buffer_free(actual); + data_buffer_free(expected); + return ret; +} + +int main(int argc, char** argv) { + /* Parse args and validate modules. */ + int ret = parse_args(argc, argv); + if (ret != 0) + return ret; + + if (are_names_bad()) + return 1; + + /* Initialize modules. */ + method_set_zstdcli(g_zstdcli); + ret = data_init(g_cache); + if (ret != 0) { + fprintf(stderr, "data_init() failed with error=%s\n", strerror(ret)); + return 1; + } + + /* Run the regression tests. */ + ret = 1; + FILE* results = fopen(g_output, "w"); + if (results == NULL) { + fprintf(stderr, "Failed to open the output file\n"); + goto out; + } + ret = run_all(results); + fclose(results); + + if (ret != 0) + goto out; + + if (g_diff) + /* Diff the new results with the previous results. */ + ret = diff_results(g_output, g_diff); + +out: + data_finish(); + return ret; +} diff --git a/src/zstd/tests/roundTripCrash.c b/src/zstd/tests/roundTripCrash.c new file mode 100644 index 000000000..c117d2c26 --- /dev/null +++ b/src/zstd/tests/roundTripCrash.c @@ -0,0 +1,241 @@ +/* + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + This program takes a file in input, + performs a zstd round-trip test (compression - decompress) + compares the result with original + and generates a crash (double free) on corruption detection. +*/ + +/*=========================================== +* Dependencies +*==========================================*/ +#include <stddef.h> /* size_t */ +#include <stdlib.h> /* malloc, free, exit */ +#include <stdio.h> /* fprintf */ +#include <string.h> /* strcmp */ +#include <sys/types.h> /* stat */ +#include <sys/stat.h> /* stat */ +#include "xxhash.h" + +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + +/*=========================================== +* Macros +*==========================================*/ +#define MIN(a,b) ( (a) < (b) ? (a) : (b) ) + +static void crash(int errorCode){ + /* abort if AFL/libfuzzer, exit otherwise */ + #ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* could also use __AFL_COMPILER */ + abort(); + #else + exit(errorCode); + #endif +} + +#define CHECK_Z(f) { \ + size_t const err = f; \ + if (ZSTD_isError(err)) { \ + fprintf(stderr, \ + "Error=> %s: %s", \ + #f, ZSTD_getErrorName(err)); \ + crash(1); \ +} } + +/** roundTripTest() : +* Compresses `srcBuff` into `compressedBuff`, +* then decompresses `compressedBuff` into `resultBuff`. +* Compression level used is derived from first content byte. +* @return : result of decompression, which should be == `srcSize` +* or an error code if either compression or decompression fails. +* Note : `compressedBuffCapacity` should be `>= ZSTD_compressBound(srcSize)` +* for compression to be guaranteed to work */ +static size_t roundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcBuffSize) +{ + static const int maxClevel = 19; + size_t const hashLength = MIN(128, srcBuffSize); + unsigned const h32 = XXH32(srcBuff, hashLength, 0); + int const cLevel = h32 % maxClevel; + size_t const cSize = ZSTD_compress(compressedBuff, compressedBuffCapacity, srcBuff, srcBuffSize, cLevel); + if (ZSTD_isError(cSize)) { + fprintf(stderr, "Compression error : %s \n", ZSTD_getErrorName(cSize)); + return cSize; + } + return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, cSize); +} + +/** cctxParamRoundTripTest() : + * Same as roundTripTest() except allows experimenting with ZSTD_CCtx_params. */ +static size_t cctxParamRoundTripTest(void* resultBuff, size_t resultBuffCapacity, + void* compressedBuff, size_t compressedBuffCapacity, + const void* srcBuff, size_t srcBuffSize) +{ + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_CCtx_params* const cctxParams = ZSTD_createCCtxParams(); + ZSTD_inBuffer inBuffer = { srcBuff, srcBuffSize, 0 }; + ZSTD_outBuffer outBuffer = { compressedBuff, compressedBuffCapacity, 0 }; + + static const int maxClevel = 19; + size_t const hashLength = MIN(128, srcBuffSize); + unsigned const h32 = XXH32(srcBuff, hashLength, 0); + int const cLevel = h32 % maxClevel; + + /* Set parameters */ + CHECK_Z( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_compressionLevel, cLevel) ); + CHECK_Z( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_nbWorkers, 2) ); + CHECK_Z( ZSTD_CCtxParams_setParameter(cctxParams, ZSTD_c_overlapLog, 5) ); + + + /* Apply parameters */ + CHECK_Z( ZSTD_CCtx_setParametersUsingCCtxParams(cctx, cctxParams) ); + + CHECK_Z (ZSTD_compressStream2(cctx, &outBuffer, &inBuffer, ZSTD_e_end) ); + + ZSTD_freeCCtxParams(cctxParams); + ZSTD_freeCCtx(cctx); + + return ZSTD_decompress(resultBuff, resultBuffCapacity, compressedBuff, outBuffer.pos); +} + +static size_t checkBuffers(const void* buff1, const void* buff2, size_t buffSize) +{ + const char* ip1 = (const char*)buff1; + const char* ip2 = (const char*)buff2; + size_t pos; + + for (pos=0; pos<buffSize; pos++) + if (ip1[pos]!=ip2[pos]) + break; + + return pos; +} + +static void roundTripCheck(const void* srcBuff, size_t srcBuffSize, int testCCtxParams) +{ + size_t const cBuffSize = ZSTD_compressBound(srcBuffSize); + void* cBuff = malloc(cBuffSize); + void* rBuff = malloc(cBuffSize); + + if (!cBuff || !rBuff) { + fprintf(stderr, "not enough memory ! \n"); + exit (1); + } + + { size_t const result = testCCtxParams ? + cctxParamRoundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize) + : roundTripTest(rBuff, cBuffSize, cBuff, cBuffSize, srcBuff, srcBuffSize); + if (ZSTD_isError(result)) { + fprintf(stderr, "roundTripTest error : %s \n", ZSTD_getErrorName(result)); + crash(1); + } + if (result != srcBuffSize) { + fprintf(stderr, "Incorrect regenerated size : %u != %u\n", (unsigned)result, (unsigned)srcBuffSize); + crash(1); + } + if (checkBuffers(srcBuff, rBuff, srcBuffSize) != srcBuffSize) { + fprintf(stderr, "Silent decoding corruption !!!"); + crash(1); + } + } + + free(cBuff); + free(rBuff); +} + + +static size_t getFileSize(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (r || !(statbuf.st_mode & S_IFREG)) return 0; /* No good... */ +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (r || !S_ISREG(statbuf.st_mode)) return 0; /* No good... */ +#endif + return (size_t)statbuf.st_size; +} + + +static int isDirectory(const char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); + if (!r && (statbuf.st_mode & _S_IFDIR)) return 1; +#else + struct stat statbuf; + r = stat(infilename, &statbuf); + if (!r && S_ISDIR(statbuf.st_mode)) return 1; +#endif + return 0; +} + + +/** loadFile() : +* requirement : `buffer` size >= `fileSize` */ +static void loadFile(void* buffer, const char* fileName, size_t fileSize) +{ + FILE* const f = fopen(fileName, "rb"); + if (isDirectory(fileName)) { + fprintf(stderr, "Ignoring %s directory \n", fileName); + exit(2); + } + if (f==NULL) { + fprintf(stderr, "Impossible to open %s \n", fileName); + exit(3); + } + { size_t const readSize = fread(buffer, 1, fileSize, f); + if (readSize != fileSize) { + fprintf(stderr, "Error reading %s \n", fileName); + exit(5); + } } + fclose(f); +} + + +static void fileCheck(const char* fileName, int testCCtxParams) +{ + size_t const fileSize = getFileSize(fileName); + void* const buffer = malloc(fileSize + !fileSize /* avoid 0 */); + if (!buffer) { + fprintf(stderr, "not enough memory \n"); + exit(4); + } + loadFile(buffer, fileName, fileSize); + roundTripCheck(buffer, fileSize, testCCtxParams); + free (buffer); +} + +int main(int argCount, const char** argv) { + int argNb = 1; + int testCCtxParams = 0; + if (argCount < 2) { + fprintf(stderr, "Error : no argument : need input file \n"); + exit(9); + } + + if (!strcmp(argv[argNb], "--cctxParams")) { + testCCtxParams = 1; + argNb++; + } + + fileCheck(argv[argNb], testCCtxParams); + fprintf(stderr, "no pb detected\n"); + return 0; +} diff --git a/src/zstd/tests/seqgen.c b/src/zstd/tests/seqgen.c new file mode 100644 index 000000000..29c0c4054 --- /dev/null +++ b/src/zstd/tests/seqgen.c @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2017-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include "seqgen.h" +#include "mem.h" +#include <string.h> + +#define MIN(a, b) ((a) < (b) ? (a) : (b)) + +static const size_t kMatchBytes = 128; + +#define SEQ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static BYTE SEQ_randByte(unsigned* src) +{ + static const U32 prime1 = 2654435761U; + static const U32 prime2 = 2246822519U; + U32 rand32 = *src; + rand32 *= prime1; + rand32 ^= prime2; + rand32 = SEQ_rotl32(rand32, 13); + *src = rand32; + return (BYTE)(rand32 >> 5); +} + +SEQ_stream SEQ_initStream(unsigned seed) +{ + SEQ_stream stream; + stream.state = 0; + XXH64_reset(&stream.xxh, 0); + stream.seed = seed; + return stream; +} + +/* Generates a single guard byte, then match length + 1 of a different byte, + * then another guard byte. + */ +static size_t SEQ_gen_matchLength(SEQ_stream* stream, unsigned value, + SEQ_outBuffer* out) +{ + typedef enum { + ml_first_byte = 0, + ml_match_bytes, + ml_last_byte, + } ml_state; + BYTE* const ostart = (BYTE*)out->dst; + BYTE* const oend = ostart + out->size; + BYTE* op = ostart + out->pos; + + switch ((ml_state)stream->state) { + case ml_first_byte: + /* Generate a single byte and pick a different byte for the match */ + if (op >= oend) { + stream->bytesLeft = 1; + break; + } + *op = SEQ_randByte(&stream->seed) & 0xFF; + do { + stream->saved = SEQ_randByte(&stream->seed) & 0xFF; + } while (*op == stream->saved); + ++op; + /* State transition */ + stream->state = ml_match_bytes; + stream->bytesLeft = value + 1; + /* fall-through */ + case ml_match_bytes: { + /* Copy matchLength + 1 bytes to the output buffer */ + size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op)); + if (setLength > 0) { + memset(op, stream->saved, setLength); + op += setLength; + stream->bytesLeft -= setLength; + } + if (stream->bytesLeft > 0) + break; + /* State transition */ + stream->state = ml_last_byte; + } + /* fall-through */ + case ml_last_byte: + /* Generate a single byte and pick a different byte for the match */ + if (op >= oend) { + stream->bytesLeft = 1; + break; + } + do { + *op = SEQ_randByte(&stream->seed) & 0xFF; + } while (*op == stream->saved); + ++op; + /* State transition */ + /* fall-through */ + default: + stream->state = 0; + stream->bytesLeft = 0; + break; + } + XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos); + out->pos = op - ostart; + return stream->bytesLeft; +} + +/* Saves the current seed then generates kMatchBytes random bytes >= 128. + * Generates literal length - kMatchBytes random bytes < 128. + * Generates another kMatchBytes using the saved seed to generate a match. + * This way the match is easy to find for the compressors. + */ +static size_t SEQ_gen_litLength(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out) +{ + typedef enum { + ll_start = 0, + ll_run_bytes, + ll_literals, + ll_run_match, + } ll_state; + BYTE* const ostart = (BYTE*)out->dst; + BYTE* const oend = ostart + out->size; + BYTE* op = ostart + out->pos; + + switch ((ll_state)stream->state) { + case ll_start: + stream->state = ll_run_bytes; + stream->saved = stream->seed; + stream->bytesLeft = MIN(kMatchBytes, value); + /* fall-through */ + case ll_run_bytes: + while (stream->bytesLeft > 0 && op < oend) { + *op++ = SEQ_randByte(&stream->seed) | 0x80; + --stream->bytesLeft; + } + if (stream->bytesLeft > 0) + break; + /* State transition */ + stream->state = ll_literals; + stream->bytesLeft = value - MIN(kMatchBytes, value); + /* fall-through */ + case ll_literals: + while (stream->bytesLeft > 0 && op < oend) { + *op++ = SEQ_randByte(&stream->seed) & 0x7F; + --stream->bytesLeft; + } + if (stream->bytesLeft > 0) + break; + /* State transition */ + stream->state = ll_run_match; + stream->bytesLeft = MIN(kMatchBytes, value); + /* fall-through */ + case ll_run_match: { + while (stream->bytesLeft > 0 && op < oend) { + *op++ = SEQ_randByte(&stream->saved) | 0x80; + --stream->bytesLeft; + } + if (stream->bytesLeft > 0) + break; + } + /* fall-through */ + default: + stream->state = 0; + stream->bytesLeft = 0; + break; + } + XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos); + out->pos = op - ostart; + return stream->bytesLeft; +} + +/* Saves the current seed then generates kMatchBytes random bytes >= 128. + * Generates offset - kMatchBytes of zeros to get a large offset without + * polluting the hash tables. + * Generates another kMatchBytes using the saved seed to generate a with the + * required offset. + */ +static size_t SEQ_gen_offset(SEQ_stream* stream, unsigned value, SEQ_outBuffer* out) +{ + typedef enum { + of_start = 0, + of_run_bytes, + of_offset, + of_run_match, + } of_state; + BYTE* const ostart = (BYTE*)out->dst; + BYTE* const oend = ostart + out->size; + BYTE* op = ostart + out->pos; + + switch ((of_state)stream->state) { + case of_start: + stream->state = of_run_bytes; + stream->saved = stream->seed; + stream->bytesLeft = MIN(value, kMatchBytes); + /* fall-through */ + case of_run_bytes: { + while (stream->bytesLeft > 0 && op < oend) { + *op++ = SEQ_randByte(&stream->seed) | 0x80; + --stream->bytesLeft; + } + if (stream->bytesLeft > 0) + break; + /* State transition */ + stream->state = of_offset; + stream->bytesLeft = value - MIN(value, kMatchBytes); + } + /* fall-through */ + case of_offset: { + /* Copy matchLength + 1 bytes to the output buffer */ + size_t const setLength = MIN(stream->bytesLeft, (size_t)(oend - op)); + if (setLength > 0) { + memset(op, 0, setLength); + op += setLength; + stream->bytesLeft -= setLength; + } + if (stream->bytesLeft > 0) + break; + /* State transition */ + stream->state = of_run_match; + stream->bytesLeft = MIN(value, kMatchBytes); + } + /* fall-through */ + case of_run_match: { + while (stream->bytesLeft > 0 && op < oend) { + *op++ = SEQ_randByte(&stream->saved) | 0x80; + --stream->bytesLeft; + } + if (stream->bytesLeft > 0) + break; + } + /* fall-through */ + default: + stream->state = 0; + stream->bytesLeft = 0; + break; + } + XXH64_update(&stream->xxh, ostart + out->pos, (op - ostart) - out->pos); + out->pos = op - ostart; + return stream->bytesLeft; +} + +/* Returns the number of bytes left to generate. + * Must pass the same type/value until it returns 0. + */ +size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value, SEQ_outBuffer* out) +{ + switch (type) { + case SEQ_gen_ml: return SEQ_gen_matchLength(stream, value, out); + case SEQ_gen_ll: return SEQ_gen_litLength(stream, value, out); + case SEQ_gen_of: return SEQ_gen_offset(stream, value, out); + case SEQ_gen_max: /* fall-through */ + default: return 0; + } +} + +/* Returns the xxhash of the data produced so far */ +XXH64_hash_t SEQ_digest(SEQ_stream const* stream) +{ + return XXH64_digest(&stream->xxh); +} diff --git a/src/zstd/tests/seqgen.h b/src/zstd/tests/seqgen.h new file mode 100644 index 000000000..808099ba9 --- /dev/null +++ b/src/zstd/tests/seqgen.h @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017-2020, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef SEQGEN_H +#define SEQGEN_H + +#define XXH_STATIC_LINKING_ONLY + +#include "xxhash.h" +#include <stddef.h> /* size_t */ + +typedef enum { + SEQ_gen_ml = 0, + SEQ_gen_ll, + SEQ_gen_of, + SEQ_gen_max /* Must be the last value */ +} SEQ_gen_type; + +/* Internal state, do not use */ +typedef struct { + XXH64_state_t xxh; /* xxh state for all the data produced so far (seed=0) */ + unsigned seed; + int state; /* enum to control state machine (clean=0) */ + unsigned saved; + size_t bytesLeft; +} SEQ_stream; + +SEQ_stream SEQ_initStream(unsigned seed); + +typedef struct { + void* dst; + size_t size; + size_t pos; +} SEQ_outBuffer; + +/* Returns non-zero until the current type/value has been generated. + * Must pass the same type/value until it returns 0. + * + * Recommended to pick a value in the middle of the range you want, since there + * may be some noise that causes actual results to be slightly different. + * We try to be more accurate for smaller values. + * + * NOTE: Very small values don't work well (< 6). + */ +size_t SEQ_gen(SEQ_stream* stream, SEQ_gen_type type, unsigned value, + SEQ_outBuffer* out); + +/* Returns the xxhash of the data produced so far */ +XXH64_hash_t SEQ_digest(SEQ_stream const* stream); + +#endif /* SEQGEN_H */ diff --git a/src/zstd/tests/test-license.py b/src/zstd/tests/test-license.py new file mode 100755 index 000000000..522884dba --- /dev/null +++ b/src/zstd/tests/test-license.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 + +# ################################################################ +# Copyright (c) 2016-2020, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +import datetime +import enum +import glob +import os +import sys + +YEAR = datetime.datetime.now().year + +YEAR_STR = str(YEAR) + +ROOT = os.path.join(os.path.dirname(__file__), "..") + +RELDIRS = [ + "doc", + "examples", + "lib", + "programs", + "tests", +] + +DIRS = [os.path.join(ROOT, d) for d in RELDIRS] + +class File(enum.Enum): + C = 1 + H = 2 + MAKE = 3 + PY = 4 + +SUFFIX = { + File.C: ".c", + File.H: ".h", + File.MAKE: "Makefile", + File.PY: ".py", +} + +# License should certainly be in the first 10 KB. +MAX_BYTES = 10000 +MAX_LINES = 50 + +LICENSE_LINES = [ + "This source code is licensed under both the BSD-style license (found in the", + "LICENSE file in the root directory of this source tree) and the GPLv2 (found", + "in the COPYING file in the root directory of this source tree).", + "You may select, at your option, one of the above-listed licenses.", +] + +COPYRIGHT_EXCEPTIONS = { + # From zstdmt + "threading.c", + "threading.h", + # From divsufsort + "divsufsort.c", + "divsufsort.h", +} + +LICENSE_EXCEPTIONS = { + # From divsufsort + "divsufsort.c", + "divsufsort.h", +} + + +def valid_copyright(lines): + for line in lines: + line = line.strip() + if "Copyright" not in line: + continue + if "present" in line: + return (False, f"Copyright line '{line}' contains 'present'!") + if "Facebook, Inc" not in line: + return (False, f"Copyright line '{line}' does not contain 'Facebook, Inc'") + if YEAR_STR not in line: + return (False, f"Copyright line '{line}' does not contain {YEAR}") + if " (c) " not in line: + return (False, f"Copyright line '{line}' does not contain ' (c) '!") + return (True, "") + return (False, "Copyright not found!") + + +def valid_license(lines): + for b in range(len(lines)): + if LICENSE_LINES[0] not in lines[b]: + continue + for l in range(len(LICENSE_LINES)): + if LICENSE_LINES[l] not in lines[b + l]: + message = f"""Invalid license line found starting on line {b + l}! +Expected: '{LICENSE_LINES[l]}' +Actual: '{lines[b + l]}'""" + return (False, message) + return (True, "") + return (False, "License not found!") + + +def valid_file(filename): + with open(filename, "r") as f: + lines = f.readlines(MAX_BYTES) + lines = lines[:min(len(lines), MAX_LINES)] + + ok = True + if os.path.basename(filename) not in COPYRIGHT_EXCEPTIONS: + c_ok, c_msg = valid_copyright(lines) + if not c_ok: + print(f"{filename}: {c_msg}") + ok = False + if os.path.basename(filename) not in LICENSE_EXCEPTIONS: + l_ok, l_msg = valid_license(lines) + if not l_ok: + print(f"{filename}: {l_msg}") + ok = False + return ok + + +def main(): + invalid_files = [] + for directory in DIRS: + for suffix in SUFFIX.values(): + files = set(glob.glob(f"{directory}/*{suffix}")) + files |= set(glob.glob(f"{directory}/**/*{suffix}")) + for filename in files: + if not valid_file(filename): + invalid_files.append(filename) + if len(invalid_files) > 0: + print(f"Invalid files: {invalid_files}") + else: + print("Pass!") + return len(invalid_files) + +if __name__ == "__main__": + sys.exit(main())
\ No newline at end of file diff --git a/src/zstd/tests/test-zstd-versions.py b/src/zstd/tests/test-zstd-versions.py new file mode 100755 index 000000000..fa2171752 --- /dev/null +++ b/src/zstd/tests/test-zstd-versions.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +"""Test zstd interoperability between versions""" + +# ################################################################ +# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# You may select, at your option, one of the above-listed licenses. +# ################################################################ + +import filecmp +import glob +import hashlib +import os +import shutil +import sys +import subprocess +from subprocess import Popen, PIPE + +repo_url = 'https://github.com/facebook/zstd.git' +tmp_dir_name = 'tests/versionsTest' +make_cmd = 'make' +git_cmd = 'git' +test_dat_src = 'README.md' +test_dat = 'test_dat' +head = 'vdevel' +dict_source = 'dict_source' +dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c ' +dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h' + + +def execute(command, print_output=False, print_error=True, param_shell=False): + popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell) + stdout_lines, stderr_lines = popen.communicate() + stderr_lines = stderr_lines.decode("utf-8") + stdout_lines = stdout_lines.decode("utf-8") + if print_output: + print(stdout_lines) + print(stderr_lines) + if popen.returncode is not None and popen.returncode != 0: + if not print_output and print_error: + print(stderr_lines) + return popen.returncode + + +def proc(cmd_args, pipe=True, dummy=False): + if dummy: + return + if pipe: + subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE) + else: + subproc = Popen(cmd_args) + return subproc.communicate() + + +def make(args, pipe=True): + return proc([make_cmd] + args, pipe) + + +def git(args, pipe=True): + return proc([git_cmd] + args, pipe) + + +def get_git_tags(): + stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]']) + tags = stdout.decode('utf-8').split() + return tags + + +def create_dict(tag, dict_source_path): + dict_name = 'dict.' + tag + if not os.path.isfile(dict_name): + cFiles = glob.glob(dict_source_path + "/*.c") + hFiles = glob.glob(dict_source_path + "/*.h") + if tag == 'v0.5.0': + result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) + else: + result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True) + if result == 0: + print(dict_name + ' created') + else: + print('ERROR: creating of ' + dict_name + ' failed') + else: + print(dict_name + ' already exists') + + +def dict_compress_sample(tag, sample): + dict_name = 'dict.' + tag + DEVNULL = open(os.devnull, 'wb') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst') + if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst') + # zstdFiles = glob.glob("*.zst*") + # print(zstdFiles) + print(tag + " : dict compression completed") + + +def compress_sample(tag, sample): + DEVNULL = open(os.devnull, 'wb') + if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst') + if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst') + if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst') + if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst') + if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0: + os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst') + # zstdFiles = glob.glob("*.zst*") + # print(zstdFiles) + print(tag + " : compression completed") + + +# http://stackoverflow.com/a/19711609/2132223 +def sha1_of_file(filepath): + with open(filepath, 'rb') as f: + return hashlib.sha1(f.read()).hexdigest() + + +def remove_duplicates(): + list_of_zst = sorted(glob.glob('*.zst')) + for i, ref_zst in enumerate(list_of_zst): + if not os.path.isfile(ref_zst): + continue + for j in range(i + 1, len(list_of_zst)): + compared_zst = list_of_zst[j] + if not os.path.isfile(compared_zst): + continue + if filecmp.cmp(ref_zst, compared_zst): + os.remove(compared_zst) + print('duplicated : {} == {}'.format(ref_zst, compared_zst)) + + +def decompress_zst(tag): + dec_error = 0 + list_zst = sorted(glob.glob('*_nodict.zst')) + for file_zst in list_zst: + print(file_zst, end=' ') + print(tag, end=' ') + file_dec = file_zst + '_d64_' + tag + '.dec' + if tag <= 'v0.5.0': + params = ['./zstd.' + tag, '-df', file_zst, file_dec] + else: + params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec] + if execute(params) == 0: + if not filecmp.cmp(file_dec, test_dat): + print('ERR !! ') + dec_error = 1 + else: + print('OK ') + else: + print('command does not work') + dec_error = 1 + return dec_error + + +def decompress_dict(tag): + dec_error = 0 + list_zst = sorted(glob.glob('*_dictio.zst')) + for file_zst in list_zst: + dict_tag = file_zst[0:len(file_zst)-11] # remove "_dictio.zst" + if head in dict_tag: # find vdevel + dict_tag = head + else: + dict_tag = dict_tag[dict_tag.rfind('v'):] + if tag == 'v0.6.0' and dict_tag < 'v0.6.0': + continue + dict_name = 'dict.' + dict_tag + print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ') + file_dec = file_zst + '_d64_' + tag + '.dec' + if tag <= 'v0.5.0': + params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec] + else: + params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec] + if execute(params) == 0: + if not filecmp.cmp(file_dec, test_dat): + print('ERR !! ') + dec_error = 1 + else: + print('OK ') + else: + print('command does not work') + dec_error = 1 + return dec_error + + +if __name__ == '__main__': + error_code = 0 + base_dir = os.getcwd() + '/..' # /path/to/zstd + tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest + clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd + dict_source_path = tmp_dir + '/' + dict_source # /path/to/zstd/tests/versionsTest/dict_source + programs_dir = base_dir + '/programs' # /path/to/zstd/programs + os.makedirs(tmp_dir, exist_ok=True) + + # since Travis clones limited depth, we should clone full repository + if not os.path.isdir(clone_dir): + git(['clone', repo_url, clone_dir]) + + shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat) + + # Retrieve all release tags + print('Retrieve all release tags :') + os.chdir(clone_dir) + alltags = get_git_tags() + [head] + tags = [t for t in alltags if t >= 'v0.5.0'] + print(tags) + + # Build all release zstd + for tag in tags: + os.chdir(base_dir) + dst_zstd = '{}/zstd.{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/zstd.<TAG> + if not os.path.isfile(dst_zstd) or tag == head: + if tag != head: + r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/<TAG> + os.makedirs(r_dir, exist_ok=True) + os.chdir(clone_dir) + git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False) + if tag == 'v0.5.0': + os.chdir(r_dir + '/dictBuilder') # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder + make(['clean', 'dictBuilder'], False) + shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag)) + os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest/<TAG>/programs + make(['clean', 'zstd'], False) + else: + os.chdir(programs_dir) + make(['zstd'], False) + shutil.copy2('zstd', dst_zstd) + + # remove any remaining *.zst and *.dec from previous test + os.chdir(tmp_dir) + for compressed in glob.glob("*.zst"): + os.remove(compressed) + for dec in glob.glob("*.dec"): + os.remove(dec) + + # copy *.c and *.h to a temporary directory ("dict_source") + if not os.path.isdir(dict_source_path): + os.mkdir(dict_source_path) + print('cp ' + dict_files + ' ' + dict_source_path) + execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True) + + print('Compress test.dat by all released zstd') + + error_code = 0 + for tag in tags: + print(tag) + if tag >= 'v0.5.0': + create_dict(tag, dict_source_path) + dict_compress_sample(tag, test_dat) + remove_duplicates() + error_code += decompress_dict(tag) + compress_sample(tag, test_dat) + remove_duplicates() + error_code += decompress_zst(tag) + + print('') + print('Enumerate different compressed files') + zstds = sorted(glob.glob('*.zst')) + for zstd in zstds: + print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd)) + + if error_code != 0: + print('====== ERROR !!! =======') + + sys.exit(error_code) diff --git a/src/zstd/tests/zbufftest.c b/src/zstd/tests/zbufftest.c new file mode 100644 index 000000000..cd3706af4 --- /dev/null +++ b/src/zstd/tests/zbufftest.c @@ -0,0 +1,625 @@ +/* + * Copyright (c) 2015-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*-************************************ +* Compiler specific +**************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +#endif + + +/*-************************************ +* Includes +**************************************/ +#include <stdlib.h> /* free */ +#include <stdio.h> /* fgets, sscanf */ +#include <string.h> /* strcmp */ +#include "timefn.h" /* UTIL_time_t */ +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel */ +#include "zstd.h" /* ZSTD_compressBound */ +#define ZBUFF_STATIC_LINKING_ONLY /* ZBUFF_createCCtx_advanced */ +#include "zbuff.h" /* ZBUFF_isError */ +#include "datagen.h" /* RDG_genBuffer */ +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" /* XXH64_* */ +#include "util.h" +#include "assert.h" + + +/*-************************************ +* Constants +**************************************/ +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +static const U32 nbTestsDefault = 10000; +#define COMPRESSIBLE_NOISE_LENGTH (10 MB) +#define FUZ_COMPRESSIBILITY_DEFAULT 50 +static const U32 prime1 = 2654435761U; +static const U32 prime2 = 2246822519U; + + + +/*-************************************ +* Display Macros +**************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static U32 g_displayLevel = 2; + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ + { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stderr); } } + +static U64 g_clockTime = 0; + + +/*-******************************************************* +* Fuzzer functions +*********************************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b)?(a):(b)) +#define MAX(a,b) ((a)>(b)?(a):(b)) +/*! FUZ_rand() : + @return : a 27 bits random value, from a 32-bits `seed`. + `seed` is also modified */ +# define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static unsigned int FUZ_rand(unsigned int* seedPtr) +{ + U32 rand32 = *seedPtr; + rand32 *= prime1; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *seedPtr = rand32; + return rand32 >> 5; +} + + +/* +static unsigned FUZ_highbit32(U32 v32) +{ + unsigned nbBits = 0; + if (v32==0) return 0; + for ( ; v32 ; v32>>=1) nbBits++; + return nbBits; +} +*/ + +static void* ZBUFF_allocFunction(void* opaque, size_t size) +{ + void* address = malloc(size); + (void)opaque; + /* DISPLAYLEVEL(4, "alloc %p, %d opaque=%p \n", address, (int)size, opaque); */ + return address; +} + +static void ZBUFF_freeFunction(void* opaque, void* address) +{ + (void)opaque; + /* if (address) DISPLAYLEVEL(4, "free %p opaque=%p \n", address, opaque); */ + free(address); +} + +static int basicUnitTests(U32 seed, double compressibility, ZSTD_customMem customMem) +{ + int testResult = 0; + size_t CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; + void* CNBuffer = malloc(CNBufferSize); + size_t const skippableFrameSize = 11; + size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + void* compressedBuffer = malloc(compressedBufferSize); + size_t const decodedBufferSize = CNBufferSize; + void* decodedBuffer = malloc(decodedBufferSize); + size_t cSize, readSize, readSkipSize, genSize; + U32 testNb=0; + ZBUFF_CCtx* zc = ZBUFF_createCCtx_advanced(customMem); + ZBUFF_DCtx* zd = ZBUFF_createDCtx_advanced(customMem); + + /* Create compressible test buffer */ + if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { + DISPLAY("Not enough memory, aborting\n"); + goto _output_error; + } + RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + + /* generate skippable frame */ + MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); + cSize = skippableFrameSize + 8; + + /* Basic compression test */ + DISPLAYLEVEL(4, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZBUFF_compressInitDictionary(zc, CNBuffer, 128 KB, 1); + readSize = CNBufferSize; + genSize = compressedBufferSize; + { size_t const r = ZBUFF_compressContinue(zc, ((char*)compressedBuffer)+cSize, &genSize, CNBuffer, &readSize); + if (ZBUFF_isError(r)) goto _output_error; } + if (readSize != CNBufferSize) goto _output_error; /* entire input should be consumed */ + cSize += genSize; + genSize = compressedBufferSize - cSize; + { size_t const r = ZBUFF_compressEnd(zc, ((char*)compressedBuffer)+cSize, &genSize); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + cSize += genSize; + DISPLAYLEVEL(4, "OK (%u bytes : %.2f%%)\n", (U32)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + /* skippable frame test */ + DISPLAYLEVEL(4, "test%3i : decompress skippable frame : ", testNb++); + ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + readSkipSize = cSize; + genSize = CNBufferSize; + { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, compressedBuffer, &readSkipSize); + if (r != 0) goto _output_error; } + if (genSize != 0) goto _output_error; /* skippable frame len is 0 */ + DISPLAYLEVEL(4, "OK \n"); + + /* Basic decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + readSize = cSize - readSkipSize; + genSize = CNBufferSize; + { size_t const r = ZBUFF_decompressContinue(zd, decodedBuffer, &genSize, ((char*)compressedBuffer)+readSkipSize, &readSize); + if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (readSize+readSkipSize != cSize) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + DISPLAYLEVEL(4, "test%3i : ZBUFF_recommendedCInSize : ", testNb++); { assert(ZBUFF_recommendedCInSize() != 0); } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : ZBUFF_recommendedCOutSize : ", testNb++); { assert(ZBUFF_recommendedCOutSize() != 0); } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : ZBUFF_recommendedDInSize : ", testNb++); { assert(ZBUFF_recommendedDInSize() != 0); } DISPLAYLEVEL(4, "OK \n"); + DISPLAYLEVEL(4, "test%3i : ZBUFF_recommendedDOutSize : ", testNb++); { assert(ZBUFF_recommendedDOutSize() != 0); } DISPLAYLEVEL(4, "OK \n"); + + /* check regenerated data is byte exact */ + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + { size_t i; + for (i=0; i<CNBufferSize; i++) { + if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error; + } } + DISPLAYLEVEL(4, "OK \n"); + + /* Byte-by-byte decompression test */ + DISPLAYLEVEL(4, "test%3i : decompress byte-by-byte : ", testNb++); + { size_t r, pIn=0, pOut=0; + do + { ZBUFF_decompressInitDictionary(zd, CNBuffer, 128 KB); + r = 1; + while (r) { + size_t inS = 1; + size_t outS = 1; + r = ZBUFF_decompressContinue(zd, ((BYTE*)decodedBuffer)+pOut, &outS, ((BYTE*)compressedBuffer)+pIn, &inS); + pIn += inS; + pOut += outS; + } + readSize = pIn; + genSize = pOut; + } while (genSize==0); + } + if (genSize != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (readSize != cSize) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(4, "OK \n"); + + /* check regenerated data is byte exact */ + DISPLAYLEVEL(4, "test%3i : check decompressed result : ", testNb++); + { size_t i; + for (i=0; i<CNBufferSize; i++) { + if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error; + } } + DISPLAYLEVEL(4, "OK \n"); + +_end: + ZBUFF_freeCCtx(zc); + ZBUFF_freeDCtx(zd); + free(CNBuffer); + free(compressedBuffer); + free(decodedBuffer); + return testResult; + +_output_error: + testResult = 1; + DISPLAY("Error detected in Unit tests ! \n"); + goto _end; +} + + +static size_t findDiff(const void* buf1, const void* buf2, size_t max) +{ + const BYTE* b1 = (const BYTE*)buf1; + const BYTE* b2 = (const BYTE*)buf2; + size_t u; + for (u=0; u<max; u++) { + if (b1[u] != b2[u]) break; + } + return u; +} + +static size_t FUZ_rLogLength(U32* seed, U32 logLength) +{ + size_t const lengthMask = ((size_t)1 << logLength) - 1; + return (lengthMask+1) + (FUZ_rand(seed) & lengthMask); +} + +static size_t FUZ_randomLength(U32* seed, U32 maxLog) +{ + U32 const logLength = FUZ_rand(seed) % maxLog; + return FUZ_rLogLength(seed, logLength); +} + +#define CHECK(cond, ...) if (cond) { DISPLAY("Error => "); DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u) \n", seed, testNb); goto _output_error; } + +static int fuzzerTests(U32 seed, U32 nbTests, unsigned startTest, double compressibility) +{ + static const U32 maxSrcLog = 24; + static const U32 maxSampleLog = 19; + BYTE* cNoiseBuffer[5]; + size_t const srcBufferSize = (size_t)1<<maxSrcLog; + BYTE* copyBuffer; + size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog); + BYTE* cBuffer; + size_t const cBufferSize = ZSTD_compressBound(srcBufferSize); + BYTE* dstBuffer; + size_t dstBufferSize = srcBufferSize; + U32 result = 0; + U32 testNb = 0; + U32 coreSeed = seed; + ZBUFF_CCtx* zc; + ZBUFF_DCtx* zd; + UTIL_time_t startClock = UTIL_getTime(); + + /* allocations */ + zc = ZBUFF_createCCtx(); + zd = ZBUFF_createDCtx(); + cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); + copyBuffer= (BYTE*)malloc (copyBufferSize); + dstBuffer = (BYTE*)malloc (dstBufferSize); + cBuffer = (BYTE*)malloc (cBufferSize); + CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || + !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd, + "Not enough memory, fuzzer tests cancelled"); + + /* Create initial samples */ + RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ + RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ + RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); + RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ + RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ + memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */ + + /* catch up testNb */ + for (testNb=1; testNb < startTest; testNb++) + FUZ_rand(&coreSeed); + + /* test loop */ + for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) { + U32 lseed; + const BYTE* srcBuffer; + const BYTE* dict; + size_t maxTestSize, dictSize; + size_t cSize, totalTestSize, totalCSize, totalGenSize; + size_t errorCode; + U32 n, nbChunks; + XXH64_state_t xxhState; + U64 crcOrig; + + /* init */ + DISPLAYUPDATE(2, "\r%6u", testNb); + if (nbTests >= testNb) DISPLAYUPDATE(2, "/%6u ", nbTests); + FUZ_rand(&coreSeed); + lseed = coreSeed ^ prime1; + + /* states full reset (unsynchronized) */ + /* some issues only happen when reusing states in a specific sequence of parameters */ + if ((FUZ_rand(&lseed) & 0xFF) == 131) { ZBUFF_freeCCtx(zc); zc = ZBUFF_createCCtx(); } + if ((FUZ_rand(&lseed) & 0xFF) == 132) { ZBUFF_freeDCtx(zd); zd = ZBUFF_createDCtx(); } + + /* srcBuffer selection [0-4] */ + { U32 buffNb = FUZ_rand(&lseed) & 0x7F; + if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ + else { + buffNb >>= 3; + if (buffNb & 7) { + const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ + buffNb = tnb[buffNb >> 3]; + } else { + const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ + buffNb = tnb[buffNb >> 3]; + } } + srcBuffer = cNoiseBuffer[buffNb]; + } + + /* compression init */ + { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const cLevel = (FUZ_rand(&lseed) % (ZSTD_maxCLevel() - (testLog/3))) + 1; + maxTestSize = FUZ_rLogLength(&lseed, testLog); + dictSize = (FUZ_rand(&lseed)==1) ? FUZ_randomLength(&lseed, maxSampleLog) : 0; + /* random dictionary selection */ + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; + } + { ZSTD_parameters params = ZSTD_getParams(cLevel, 0, dictSize); + params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; + params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; + { size_t const initError = ZBUFF_compressInit_advanced(zc, dict, dictSize, params, ZSTD_CONTENTSIZE_UNKNOWN); + CHECK (ZBUFF_isError(initError),"init error : %s", ZBUFF_getErrorName(initError)); + } } } + + /* multi-segments compression test */ + XXH64_reset(&xxhState, 0); + nbChunks = (FUZ_rand(&lseed) & 127) + 2; + for (n=0, cSize=0, totalTestSize=0 ; (n<nbChunks) && (totalTestSize < maxTestSize) ; n++) { + /* compress random chunk into random size dst buffer */ + { size_t readChunkSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); + size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - readChunkSize); + + size_t const compressionError = ZBUFF_compressContinue(zc, cBuffer+cSize, &dstBuffSize, srcBuffer+srcStart, &readChunkSize); + CHECK (ZBUFF_isError(compressionError), "compression error : %s", ZBUFF_getErrorName(compressionError)); + + XXH64_update(&xxhState, srcBuffer+srcStart, readChunkSize); + memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, readChunkSize); + cSize += dstBuffSize; + totalTestSize += readChunkSize; + } + + /* random flush operation, to mess around */ + if ((FUZ_rand(&lseed) & 15) == 0) { + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); + size_t const flushError = ZBUFF_compressFlush(zc, cBuffer+cSize, &dstBuffSize); + CHECK (ZBUFF_isError(flushError), "flush error : %s", ZBUFF_getErrorName(flushError)); + cSize += dstBuffSize; + } } + + /* final frame epilogue */ + { size_t remainingToFlush = (size_t)(-1); + while (remainingToFlush) { + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); + U32 const enoughDstSize = dstBuffSize >= remainingToFlush; + remainingToFlush = ZBUFF_compressEnd(zc, cBuffer+cSize, &dstBuffSize); + CHECK (ZBUFF_isError(remainingToFlush), "flush error : %s", ZBUFF_getErrorName(remainingToFlush)); + CHECK (enoughDstSize && remainingToFlush, "ZBUFF_compressEnd() not fully flushed (%u remaining), but enough space available", (U32)remainingToFlush); + cSize += dstBuffSize; + } } + crcOrig = XXH64_digest(&xxhState); + + /* multi - fragments decompression test */ + ZBUFF_decompressInitDictionary(zd, dict, dictSize); + errorCode = 1; + for (totalCSize = 0, totalGenSize = 0 ; errorCode ; ) { + size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); + errorCode = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize); + CHECK (ZBUFF_isError(errorCode), "decompression error : %s", ZBUFF_getErrorName(errorCode)); + totalGenSize += dstBuffSize; + totalCSize += readCSrcSize; + } + CHECK (errorCode != 0, "frame not fully decoded"); + CHECK (totalGenSize != totalTestSize, "decompressed data : wrong size") + CHECK (totalCSize != cSize, "compressed data should be fully read") + { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); + if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); + CHECK (crcDest!=crcOrig, "decompressed data corrupted"); } + + /*===== noisy/erroneous src decompression test =====*/ + + /* add some noise */ + { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; + U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) { + size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const noiseSize = MIN((cSize/3) , randomNoiseSize); + size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize); + size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize); + memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize); + } } + + /* try decompression on noisy data */ + ZBUFF_decompressInit(zd); + totalCSize = 0; + totalGenSize = 0; + while ( (totalCSize < cSize) && (totalGenSize < dstBufferSize) ) { + size_t readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); + size_t const decompressError = ZBUFF_decompressContinue(zd, dstBuffer+totalGenSize, &dstBuffSize, cBuffer+totalCSize, &readCSrcSize); + if (ZBUFF_isError(decompressError)) break; /* error correctly detected */ + totalGenSize += dstBuffSize; + totalCSize += readCSrcSize; + } } + DISPLAY("\r%u fuzzer tests completed \n", testNb); + +_cleanup: + ZBUFF_freeCCtx(zc); + ZBUFF_freeDCtx(zd); + free(cNoiseBuffer[0]); + free(cNoiseBuffer[1]); + free(cNoiseBuffer[2]); + free(cNoiseBuffer[3]); + free(cNoiseBuffer[4]); + free(copyBuffer); + free(cBuffer); + free(dstBuffer); + return result; + +_output_error: + result = 1; + goto _cleanup; +} + + +/*-******************************************************* +* Command line +*********************************************************/ +static int FUZ_usage(const char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -i# : Nb of tests (default:%u) \n", nbTestsDefault); + DISPLAY( " -s# : Select seed (default:prompt user)\n"); + DISPLAY( " -t# : Select starting test number (default:0)\n"); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -p : pause at the end\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + + +int main(int argc, const char** argv) +{ + U32 seed=0; + int seedset=0; + int argNb; + int nbTests = nbTestsDefault; + int testNb = 0; + int proba = FUZ_COMPRESSIBILITY_DEFAULT; + int result=0; + U32 mainPause = 0; + const char* programName = argv[0]; + ZSTD_customMem customMem = { ZBUFF_allocFunction, ZBUFF_freeFunction, NULL }; + ZSTD_customMem customNULL = { NULL, NULL, NULL }; + + /* Check command line */ + for(argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + if(!argument) continue; /* Protection if argument empty */ + + /* Parsing commands. Aggregated commands are allowed */ + if (argument[0]=='-') { + argument++; + + while (*argument!=0) { + switch(*argument) + { + case 'h': + return FUZ_usage(programName); + case 'v': + argument++; + g_displayLevel=4; + break; + case 'q': + argument++; + g_displayLevel--; + break; + case 'p': /* pause at the end */ + argument++; + mainPause = 1; + break; + + case 'i': + argument++; + nbTests=0; g_clockTime=0; + while ((*argument>='0') && (*argument<='9')) { + nbTests *= 10; + nbTests += *argument - '0'; + argument++; + } + break; + + case 'T': + argument++; + nbTests=0; g_clockTime=0; + while ((*argument>='0') && (*argument<='9')) { + g_clockTime *= 10; + g_clockTime += *argument - '0'; + argument++; + } + if (*argument=='m') g_clockTime *=60, argument++; + if (*argument=='n') argument++; + g_clockTime *= SEC_TO_MICRO; + break; + + case 's': + argument++; + seed=0; + seedset=1; + while ((*argument>='0') && (*argument<='9')) { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + + case 't': + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; + + case 'P': /* compressibility % */ + argument++; + proba=0; + while ((*argument>='0') && (*argument<='9')) { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba<0) proba=0; + if (proba>100) proba=100; + break; + + default: + return FUZ_usage(programName); + } + } } } /* for(argNb=1; argNb<argc; argNb++) */ + + /* Get Seed */ + DISPLAY("Starting zstd_buffered tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); + + if (!seedset) { + time_t const t = time(NULL); + U32 const h = XXH32(&t, sizeof(t), 1); + seed = h % 10000; + } + DISPLAY("Seed = %u\n", seed); + if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); + + if (nbTests<=0) nbTests=1; + + if (testNb==0) { + result = basicUnitTests(0, ((double)proba) / 100, customNULL); /* constant seed for predictability */ + if (!result) { + DISPLAYLEVEL(4, "Unit tests using customMem :\n") + result = basicUnitTests(0, ((double)proba) / 100, customMem); /* use custom memory allocation functions */ + } } + + if (!result) + result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100); + + if (mainPause) { + int unused; + DISPLAY("Press Enter \n"); + unused = getchar(); + (void)unused; + } + return result; +} diff --git a/src/zstd/tests/zstreamtest.c b/src/zstd/tests/zstreamtest.c new file mode 100644 index 000000000..79d5a8281 --- /dev/null +++ b/src/zstd/tests/zstreamtest.c @@ -0,0 +1,2654 @@ +/* + * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*-************************************ + * Compiler specific + **************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define _CRT_SECURE_NO_WARNINGS /* fgets */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4146) /* disable: C4146: minus unsigned expression */ +#endif + + +/*-************************************ + * Includes + **************************************/ +#include <stdlib.h> /* free */ +#include <stdio.h> /* fgets, sscanf */ +#include <string.h> /* strcmp */ +#include <assert.h> /* assert */ +#include "timefn.h" /* UTIL_time_t, UTIL_getTime */ +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_maxCLevel, ZSTD_customMem, ZSTD_getDictID_fromFrame */ +#include "zstd.h" /* ZSTD_compressBound */ +#include "zstd_errors.h" /* ZSTD_error_srcSize_wrong */ +#include "zstdmt_compress.h" +#include "zdict.h" /* ZDICT_trainFromBuffer */ +#include "datagen.h" /* RDG_genBuffer */ +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH64_* */ +#include "seqgen.h" +#include "util.h" +#include "timefn.h" /* UTIL_time_t, UTIL_clockSpanMicro, UTIL_getTime */ + + +/*-************************************ + * Constants + **************************************/ +#define KB *(1U<<10) +#define MB *(1U<<20) +#define GB *(1U<<30) + +static const int nbTestsDefault = 10000; +static const U32 g_cLevelMax_smallTests = 10; +#define COMPRESSIBLE_NOISE_LENGTH (10 MB) +#define FUZ_COMPRESSIBILITY_DEFAULT 50 +static const U32 prime32 = 2654435761U; + + +/*-************************************ + * Display Macros + **************************************/ +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { \ + DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stderr); } +static U32 g_displayLevel = 2; + +static const U64 g_refreshRate = SEC_TO_MICRO / 6; +static UTIL_time_t g_displayClock = UTIL_TIME_INITIALIZER; + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((UTIL_clockSpanMicro(g_displayClock) > g_refreshRate) || (g_displayLevel>=4)) \ + { g_displayClock = UTIL_getTime(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(stderr); } } + +static U64 g_clockTime = 0; + + +/*-******************************************************* + * Check macros + *********************************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b)?(a):(b)) +#define MAX(a,b) ((a)>(b)?(a):(b)) +/*! FUZ_rand() : + @return : a 27 bits random value, from a 32-bits `seed`. + `seed` is also modified */ +#define FUZ_rotl32(x,r) ((x << r) | (x >> (32 - r))) +static U32 FUZ_rand(U32* seedPtr) +{ + static const U32 prime2 = 2246822519U; + U32 rand32 = *seedPtr; + rand32 *= prime32; + rand32 += prime2; + rand32 = FUZ_rotl32(rand32, 13); + *seedPtr = rand32; + return rand32 >> 5; +} + +#define CHECK(cond, ...) { \ + if (cond) { \ + DISPLAY("Error => "); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY(" (seed %u, test nb %u, line %u) \n", \ + (unsigned)seed, testNb, __LINE__); \ + goto _output_error; \ +} } + +#define CHECK_Z(f) { \ + size_t const err = f; \ + CHECK(ZSTD_isError(err), "%s : %s ", \ + #f, ZSTD_getErrorName(err)); \ +} + +#define CHECK_RET(ret, cond, ...) { \ + if (cond) { \ + DISPLAY("Error %llu => ", (unsigned long long)ret); \ + DISPLAY(__VA_ARGS__); \ + DISPLAY(" (line %u)\n", __LINE__); \ + return ret; \ +} } + +#define CHECK_RET_Z(f) { \ + size_t const err = f; \ + CHECK_RET(err, ZSTD_isError(err), "%s : %s ", \ + #f, ZSTD_getErrorName(err)); \ +} + + +/*====================================================== + * Basic Unit tests + *======================================================*/ + +typedef struct { + void* start; + size_t size; + size_t filled; +} buffer_t; + +static const buffer_t kBuffNull = { NULL, 0 , 0 }; + +static void FUZ_freeDictionary(buffer_t dict) +{ + free(dict.start); +} + +static buffer_t FUZ_createDictionary(const void* src, size_t srcSize, size_t blockSize, size_t requestedDictSize) +{ + buffer_t dict = kBuffNull; + size_t const nbBlocks = (srcSize + (blockSize-1)) / blockSize; + size_t* const blockSizes = (size_t*)malloc(nbBlocks * sizeof(size_t)); + if (!blockSizes) return kBuffNull; + dict.start = malloc(requestedDictSize); + if (!dict.start) { free(blockSizes); return kBuffNull; } + { size_t nb; + for (nb=0; nb<nbBlocks-1; nb++) blockSizes[nb] = blockSize; + blockSizes[nbBlocks-1] = srcSize - (blockSize * (nbBlocks-1)); + } + { size_t const dictSize = ZDICT_trainFromBuffer(dict.start, requestedDictSize, src, blockSizes, (unsigned)nbBlocks); + free(blockSizes); + if (ZDICT_isError(dictSize)) { FUZ_freeDictionary(dict); return kBuffNull; } + dict.size = requestedDictSize; + dict.filled = dictSize; + return dict; + } +} + +/* Round trips data and updates xxh with the decompressed data produced */ +static size_t SEQ_roundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, + XXH64_state_t* xxh, void* data, size_t size, + ZSTD_EndDirective endOp) +{ + static BYTE compressed[1024]; + static BYTE uncompressed[1024]; + + ZSTD_inBuffer cin = {data, size, 0}; + size_t cret; + + do { + ZSTD_outBuffer cout = { compressed, sizeof(compressed), 0 }; + ZSTD_inBuffer din = { compressed, 0, 0 }; + ZSTD_outBuffer dout = { uncompressed, 0, 0 }; + + cret = ZSTD_compressStream2(cctx, &cout, &cin, endOp); + if (ZSTD_isError(cret)) + return cret; + + din.size = cout.pos; + while (din.pos < din.size || (endOp == ZSTD_e_end && cret == 0)) { + size_t dret; + + dout.pos = 0; + dout.size = sizeof(uncompressed); + dret = ZSTD_decompressStream(dctx, &dout, &din); + if (ZSTD_isError(dret)) + return dret; + XXH64_update(xxh, dout.dst, dout.pos); + if (dret == 0) + break; + } + } while (cin.pos < cin.size || (endOp != ZSTD_e_continue && cret != 0)); + return 0; +} + +/* Generates some data and round trips it */ +static size_t SEQ_generateRoundTrip(ZSTD_CCtx* cctx, ZSTD_DCtx* dctx, + XXH64_state_t* xxh, SEQ_stream* seq, + SEQ_gen_type type, unsigned value) +{ + static BYTE data[1024]; + size_t gen; + + do { + SEQ_outBuffer sout = {data, sizeof(data), 0}; + size_t ret; + gen = SEQ_gen(seq, type, value, &sout); + + ret = SEQ_roundTrip(cctx, dctx, xxh, sout.dst, sout.pos, ZSTD_e_continue); + if (ZSTD_isError(ret)) + return ret; + } while (gen != 0); + + return 0; +} + +static size_t getCCtxParams(ZSTD_CCtx* zc, ZSTD_parameters* savedParams) +{ + int value; + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_windowLog, (int*)&savedParams->cParams.windowLog)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_hashLog, (int*)&savedParams->cParams.hashLog)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_chainLog, (int*)&savedParams->cParams.chainLog)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_searchLog, (int*)&savedParams->cParams.searchLog)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_minMatch, (int*)&savedParams->cParams.minMatch)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_targetLength, (int*)&savedParams->cParams.targetLength)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_strategy, &value)); + savedParams->cParams.strategy = value; + + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_checksumFlag, &savedParams->fParams.checksumFlag)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_contentSizeFlag, &savedParams->fParams.contentSizeFlag)); + CHECK_RET_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_dictIDFlag, &value)); + savedParams->fParams.noDictIDFlag = !value; + return 0; +} + +static U32 badParameters(ZSTD_CCtx* zc, ZSTD_parameters const savedParams) +{ + ZSTD_parameters params; + if (ZSTD_isError(getCCtxParams(zc, ¶ms))) return 10; + CHECK_RET(1, params.cParams.windowLog != savedParams.cParams.windowLog, "windowLog"); + CHECK_RET(2, params.cParams.hashLog != savedParams.cParams.hashLog, "hashLog"); + CHECK_RET(3, params.cParams.chainLog != savedParams.cParams.chainLog, "chainLog"); + CHECK_RET(4, params.cParams.searchLog != savedParams.cParams.searchLog, "searchLog"); + CHECK_RET(5, params.cParams.minMatch != savedParams.cParams.minMatch, "minMatch"); + CHECK_RET(6, params.cParams.targetLength != savedParams.cParams.targetLength, "targetLength"); + + CHECK_RET(7, params.fParams.checksumFlag != savedParams.fParams.checksumFlag, "checksumFlag"); + CHECK_RET(8, params.fParams.contentSizeFlag != savedParams.fParams.contentSizeFlag, "contentSizeFlag"); + CHECK_RET(9, params.fParams.noDictIDFlag != savedParams.fParams.noDictIDFlag, "noDictIDFlag"); + return 0; +} + +static int basicUnitTests(U32 seed, double compressibility) +{ + size_t const CNBufferSize = COMPRESSIBLE_NOISE_LENGTH; + void* CNBuffer = malloc(CNBufferSize); + size_t const skippableFrameSize = 200 KB; + size_t const compressedBufferSize = (8 + skippableFrameSize) + ZSTD_compressBound(COMPRESSIBLE_NOISE_LENGTH); + void* compressedBuffer = malloc(compressedBufferSize); + size_t const decodedBufferSize = CNBufferSize; + void* decodedBuffer = malloc(decodedBufferSize); + size_t cSize; + int testResult = 0; + int testNb = 1; + U32 coreSeed = 0; /* this name to conform with CHECK_Z macro display */ + ZSTD_CStream* zc = ZSTD_createCStream(); + ZSTD_DStream* zd = ZSTD_createDStream(); + ZSTDMT_CCtx* mtctx = ZSTDMT_createCCtx(2); + + ZSTD_inBuffer inBuff, inBuff2; + ZSTD_outBuffer outBuff; + buffer_t dictionary = kBuffNull; + size_t const dictSize = 128 KB; + unsigned dictID = 0; + + /* Create compressible test buffer */ + if (!CNBuffer || !compressedBuffer || !decodedBuffer || !zc || !zd) { + DISPLAY("Not enough memory, aborting \n"); + goto _output_error; + } + RDG_genBuffer(CNBuffer, CNBufferSize, compressibility, 0., seed); + + /* Create dictionary */ + DISPLAYLEVEL(3, "creating dictionary for unit tests \n"); + dictionary = FUZ_createDictionary(CNBuffer, CNBufferSize / 3, 16 KB, 48 KB); + if (!dictionary.start) { + DISPLAY("Error creating dictionary, aborting \n"); + goto _output_error; + } + dictID = ZDICT_getDictID(dictionary.start, dictionary.filled); + + /* Basic compression test */ + DISPLAYLEVEL(3, "test%3i : compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + CHECK_Z( ZSTD_initCStream(zc, 1 /* cLevel */) ); + outBuff.dst = (char*)(compressedBuffer); + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + DISPLAYLEVEL(3, "OK (%u bytes)\n", (unsigned)outBuff.pos); + + /* generate skippable frame */ + MEM_writeLE32(compressedBuffer, ZSTD_MAGIC_SKIPPABLE_START); + MEM_writeLE32(((char*)compressedBuffer)+4, (U32)skippableFrameSize); + cSize = skippableFrameSize + 8; + + /* Basic compression test using dict */ + DISPLAYLEVEL(3, "test%3i : skipframe + compress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + CHECK_Z( ZSTD_initCStream_usingDict(zc, CNBuffer, dictSize, 1 /* cLevel */) ); + outBuff.dst = (char*)(compressedBuffer)+cSize; + assert(compressedBufferSize > cSize); + outBuff.size = compressedBufferSize - cSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + cSize += outBuff.pos; + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", + (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + /* context size functions */ + DISPLAYLEVEL(3, "test%3i : estimate CStream size : ", testNb++); + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictSize); + size_t const cstreamSize = ZSTD_estimateCStreamSize_usingCParams(cParams); + size_t const cdictSize = ZSTD_estimateCDictSize_advanced(dictSize, cParams, ZSTD_dlm_byCopy); /* uses ZSTD_initCStream_usingDict() */ + if (ZSTD_isError(cstreamSize)) goto _output_error; + if (ZSTD_isError(cdictSize)) goto _output_error; + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)(cstreamSize + cdictSize)); + } + + /* context size functions */ + DISPLAYLEVEL(3, "test%3i : estimate CStream size using CCtxParams : ", testNb++); + { ZSTD_CCtx_params* const params = ZSTD_createCCtxParams(); + size_t cstreamSize, cctxSize; + CHECK_Z( ZSTD_CCtxParams_setParameter(params, ZSTD_c_compressionLevel, 19) ); + cstreamSize = ZSTD_estimateCStreamSize_usingCCtxParams(params); + CHECK_Z(cstreamSize); + cctxSize = ZSTD_estimateCCtxSize_usingCCtxParams(params); + CHECK_Z(cctxSize); + if (cstreamSize <= cctxSize + 2 * ZSTD_BLOCKSIZE_MAX) goto _output_error; + ZSTD_freeCCtxParams(params); + DISPLAYLEVEL(3, "OK \n"); + } + + DISPLAYLEVEL(3, "test%3i : check actual CStream size : ", testNb++); + { size_t const s = ZSTD_sizeof_CStream(zc); + if (ZSTD_isError(s)) goto _output_error; + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s); + } + + /* Attempt bad compression parameters */ + DISPLAYLEVEL(3, "test%3i : use bad compression parameters : ", testNb++); + { size_t r; + ZSTD_parameters params = ZSTD_getParams(1, 0, 0); + params.cParams.minMatch = 2; + r = ZSTD_initCStream_advanced(zc, NULL, 0, params, 0); + if (!ZSTD_isError(r)) goto _output_error; + DISPLAYLEVEL(3, "init error : %s \n", ZSTD_getErrorName(r)); + } + + /* skippable frame test */ + DISPLAYLEVEL(3, "test%3i : decompress skippable frame : ", testNb++); + CHECK_Z( ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize) ); + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + DISPLAYLEVEL(5, " ( ZSTD_decompressStream => %u ) ", (unsigned)r); + if (r != 0) goto _output_error; + } + if (outBuff.pos != 0) goto _output_error; /* skippable frame output len is 0 */ + DISPLAYLEVEL(3, "OK \n"); + + /* Basic decompression test */ + inBuff2 = inBuff; + DISPLAYLEVEL(3, "test%3i : decompress %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); + CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, ZSTD_WINDOWLOG_LIMIT_DEFAULT+1) ); /* large limit */ + { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(3, "OK \n"); + + /* Re-use without init */ + DISPLAYLEVEL(3, "test%3i : decompress again without init (re-use previous settings): ", testNb++); + outBuff.pos = 0; + { size_t const remaining = ZSTD_decompressStream(zd, &outBuff, &inBuff2); + if (remaining != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(3, "OK \n"); + + /* check regenerated data is byte exact */ + DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); + { size_t i; + for (i=0; i<CNBufferSize; i++) { + if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error; + } } + DISPLAYLEVEL(3, "OK \n"); + + /* context size functions */ + DISPLAYLEVEL(3, "test%3i : estimate DStream size : ", testNb++); + { ZSTD_frameHeader fhi; + const void* cStart = (char*)compressedBuffer + (skippableFrameSize + 8); + size_t const gfhError = ZSTD_getFrameHeader(&fhi, cStart, cSize); + if (gfhError!=0) goto _output_error; + DISPLAYLEVEL(5, " (windowSize : %u) ", (unsigned)fhi.windowSize); + { size_t const s = ZSTD_estimateDStreamSize(fhi.windowSize) + /* uses ZSTD_initDStream_usingDict() */ + + ZSTD_estimateDDictSize(dictSize, ZSTD_dlm_byCopy); + if (ZSTD_isError(s)) goto _output_error; + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s); + } } + + DISPLAYLEVEL(3, "test%3i : check actual DStream size : ", testNb++); + { size_t const s = ZSTD_sizeof_DStream(zd); + if (ZSTD_isError(s)) goto _output_error; + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s); + } + + /* Decompression by small increment */ + DISPLAYLEVEL(3, "test%3i : decompress byte-by-byte : ", testNb++); + { /* skippable frame */ + size_t r = 1; + ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); + inBuff.src = compressedBuffer; + outBuff.dst = decodedBuffer; + inBuff.pos = 0; + outBuff.pos = 0; + while (r) { /* skippable frame */ + size_t const inSize = (FUZ_rand(&coreSeed) & 15) + 1; + size_t const outSize = (FUZ_rand(&coreSeed) & 15) + 1; + inBuff.size = inBuff.pos + inSize; + outBuff.size = outBuff.pos + outSize; + r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream on skippable frame error : %s \n", ZSTD_getErrorName(r)); + if (ZSTD_isError(r)) goto _output_error; + } + /* normal frame */ + ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); + r=1; + while (r) { + size_t const inSize = FUZ_rand(&coreSeed) & 15; + size_t const outSize = (FUZ_rand(&coreSeed) & 15) + (!inSize); /* avoid having both sizes at 0 => would trigger a no_forward_progress error */ + inBuff.size = inBuff.pos + inSize; + outBuff.size = outBuff.pos + outSize; + r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(r)); + if (ZSTD_isError(r)) goto _output_error; + } + } + if (outBuff.pos != CNBufferSize) DISPLAYLEVEL(4, "outBuff.pos != CNBufferSize : should have regenerated same amount ! \n"); + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != cSize) DISPLAYLEVEL(4, "inBuff.pos != cSize : should have real all input ! \n"); + if (inBuff.pos != cSize) goto _output_error; /* should have read the entire frame */ + DISPLAYLEVEL(3, "OK \n"); + + /* check regenerated data is byte exact */ + DISPLAYLEVEL(3, "test%3i : check decompressed result : ", testNb++); + { size_t i; + for (i=0; i<CNBufferSize; i++) { + if (((BYTE*)decodedBuffer)[i] != ((BYTE*)CNBuffer)[i]) goto _output_error; + } } + DISPLAYLEVEL(3, "OK \n"); + + /* Decompression forward progress */ + DISPLAYLEVEL(3, "test%3i : generate error when ZSTD_decompressStream() doesn't progress : ", testNb++); + { /* skippable frame */ + size_t r = 0; + int decNb = 0; + int const maxDec = 100; + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + + outBuff.dst = decodedBuffer; + outBuff.pos = 0; + outBuff.size = CNBufferSize-1; /* 1 byte missing */ + + for (decNb=0; decNb<maxDec; decNb++) { + if (r==0) ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); + r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (ZSTD_isError(r)) break; + } + if (!ZSTD_isError(r)) DISPLAYLEVEL(4, "ZSTD_decompressStream should have triggered a no_forward_progress error \n"); + if (!ZSTD_isError(r)) goto _output_error; /* should have triggered no_forward_progress error */ + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : NULL buffers : ", testNb++); + inBuff.src = NULL; + inBuff.size = 0; + inBuff.pos = 0; + outBuff.dst = NULL; + outBuff.size = 0; + outBuff.pos = 0; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed"); + CHECK_Z( ZSTD_endStream(zc, &outBuff) ); + outBuff.dst = (char*)(compressedBuffer); + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + { size_t const r = ZSTD_endStream(zc, &outBuff); + CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r); + } + inBuff.src = outBuff.dst; + inBuff.size = outBuff.pos; + inBuff.pos = 0; + outBuff.dst = NULL; + outBuff.size = 0; + outBuff.pos = 0; + CHECK_Z( ZSTD_initDStream(zd) ); + { size_t const ret = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (ret != 0) goto _output_error; + } + DISPLAYLEVEL(3, "OK\n"); + /* _srcSize compression test */ + DISPLAYLEVEL(3, "test%3i : compress_srcSize %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + CHECK_Z( ZSTD_initCStream_srcSize(zc, 1, CNBufferSize) ); + outBuff.dst = (char*)(compressedBuffer); + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + CHECK(inBuff.pos != inBuff.size, "Entire input should be consumed"); + { size_t const r = ZSTD_endStream(zc, &outBuff); + CHECK(r != 0, "Error or some data not flushed (ret=%zu)", r); + } + { unsigned long long origSize = ZSTD_findDecompressedSize(outBuff.dst, outBuff.pos); + CHECK(origSize == ZSTD_CONTENTSIZE_UNKNOWN, "Unknown!"); + CHECK((size_t)origSize != CNBufferSize, "Exact original size must be present (got %llu)", origSize); + } + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/COMPRESSIBLE_NOISE_LENGTH*100); + + /* wrong _srcSize compression test */ + DISPLAYLEVEL(3, "test%3i : too large srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); + ZSTD_initCStream_srcSize(zc, 1, CNBufferSize+1); + outBuff.dst = (char*)(compressedBuffer); + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */ + DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); } + + /* wrong _srcSize compression test */ + DISPLAYLEVEL(3, "test%3i : too small srcSize : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); + ZSTD_initCStream_srcSize(zc, 1, CNBufferSize-1); + outBuff.dst = (char*)(compressedBuffer); + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); + if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */ + DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); + } + + DISPLAYLEVEL(3, "test%3i : wrong srcSize !contentSizeFlag : %u bytes : ", testNb++, COMPRESSIBLE_NOISE_LENGTH-1); + { ZSTD_parameters params = ZSTD_getParams(1, CNBufferSize, 0); + params.fParams.contentSizeFlag = 0; + CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, CNBufferSize - MIN(CNBufferSize, 200 KB))); + outBuff.dst = (char*)compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const r = ZSTD_compressStream(zc, &outBuff, &inBuff); + if (ZSTD_getErrorCode(r) != ZSTD_error_srcSize_wrong) goto _output_error; /* must fail : wrong srcSize */ + DISPLAYLEVEL(3, "OK (error detected : %s) \n", ZSTD_getErrorName(r)); + } } + + /* Complex context re-use scenario */ + DISPLAYLEVEL(3, "test%3i : context re-use : ", testNb++); + ZSTD_freeCStream(zc); + zc = ZSTD_createCStream(); + if (zc==NULL) goto _output_error; /* memory allocation issue */ + /* use 1 */ + { size_t const inSize = 513; + DISPLAYLEVEL(5, "use1 "); + ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */ + inBuff.src = CNBuffer; + inBuff.size = inSize; + inBuff.pos = 0; + outBuff.dst = (char*)(compressedBuffer)+cSize; + outBuff.size = ZSTD_compressBound(inSize); + outBuff.pos = 0; + DISPLAYLEVEL(5, "compress1 "); + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + DISPLAYLEVEL(5, "end1 "); + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + } + /* use 2 */ + { size_t const inSize = 1025; /* will not continue, because tables auto-adjust and are therefore different size */ + DISPLAYLEVEL(5, "use2 "); + ZSTD_initCStream_advanced(zc, NULL, 0, ZSTD_getParams(19, inSize, 0), inSize); /* needs btopt + search3 to trigger hashLog3 */ + inBuff.src = CNBuffer; + inBuff.size = inSize; + inBuff.pos = 0; + outBuff.dst = (char*)(compressedBuffer)+cSize; + outBuff.size = ZSTD_compressBound(inSize); + outBuff.pos = 0; + DISPLAYLEVEL(5, "compress2 "); + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + DISPLAYLEVEL(5, "end2 "); + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + } + DISPLAYLEVEL(3, "OK \n"); + + /* Decompression single pass with empty frame */ + cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, NULL, 0, 1); + CHECK_Z(cSize); + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass on empty frame : ", testNb++); + { ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t const dctxSize = ZSTD_sizeof_DCtx(dctx); + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1)); + + outBuff.dst = decodedBuffer; + outBuff.pos = 0; + outBuff.size = CNBufferSize; + + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff); + CHECK_Z(r); + CHECK(r != 0, "Entire frame must be decompressed"); + CHECK(outBuff.pos != 0, "Wrong size!"); + CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!"); + } + CHECK(dctxSize != ZSTD_sizeof_DCtx(dctx), "No buffers allocated"); + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + /* Decompression with ZSTD_d_stableOutBuffer */ + cSize = ZSTD_compress(compressedBuffer, compressedBufferSize, CNBuffer, CNBufferSize, 1); + CHECK_Z(cSize); + { ZSTD_DCtx* dctx = ZSTD_createDCtx(); + size_t const dctxSize0 = ZSTD_sizeof_DCtx(dctx); + size_t dctxSize1; + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1)); + + outBuff.dst = decodedBuffer; + outBuff.pos = 0; + outBuff.size = CNBufferSize; + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() single pass : ", testNb++); + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff); + CHECK_Z(r); + CHECK(r != 0, "Entire frame must be decompressed"); + CHECK(outBuff.pos != CNBufferSize, "Wrong size!"); + CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!"); + } + CHECK(dctxSize0 != ZSTD_sizeof_DCtx(dctx), "No buffers allocated"); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer : ", testNb++); + outBuff.pos = 0; + inBuff.pos = 0; + inBuff.size = 0; + while (inBuff.pos < cSize) { + inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15)); + CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff)); + } + CHECK(outBuff.pos != CNBufferSize, "Wrong size!"); + CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!"); + dctxSize1 = ZSTD_sizeof_DCtx(dctx); + CHECK(!(dctxSize0 < dctxSize1), "Input buffer allocated"); + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer too small : ", testNb++); + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only); + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1)); + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + outBuff.pos = 0; + outBuff.size = CNBufferSize - 1; + { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff); + CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstSize_tooSmall, "Must error but got %s", ZSTD_getErrorName(r)); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() stable out buffer modified : ", testNb++); + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only); + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 1)); + inBuff.src = compressedBuffer; + inBuff.size = cSize - 1; + inBuff.pos = 0; + outBuff.pos = 0; + outBuff.size = CNBufferSize; + CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff)); + ++inBuff.size; + outBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(dctx, &outBuff, &inBuff); + CHECK(ZSTD_getErrorCode(r) != ZSTD_error_dstBuffer_wrong, "Must error but got %s", ZSTD_getErrorName(r)); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() buffered output : ", testNb++); + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only); + CHECK_Z(ZSTD_DCtx_setParameter(dctx, ZSTD_d_stableOutBuffer, 0)); + outBuff.pos = 0; + inBuff.pos = 0; + inBuff.size = 0; + while (inBuff.pos < cSize) { + inBuff.size += MIN(cSize - inBuff.pos, 1 + (FUZ_rand(&coreSeed) & 15)); + CHECK_Z(ZSTD_decompressStream(dctx, &outBuff, &inBuff)); + } + CHECK(outBuff.pos != CNBufferSize, "Wrong size!"); + CHECK(memcmp(CNBuffer, outBuff.dst, CNBufferSize) != 0, "Corruption!"); + CHECK(!(dctxSize1 < ZSTD_sizeof_DCtx(dctx)), "Output buffer allocated"); + DISPLAYLEVEL(3, "OK \n"); + + ZSTD_freeDCtx(dctx); + } + + /* CDict scenario */ + DISPLAYLEVEL(3, "test%3i : digested dictionary : ", testNb++); + { ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, 1 /*byRef*/ ); + size_t const initError = ZSTD_initCStream_usingCDict(zc, cdict); + DISPLAYLEVEL(5, "ZSTD_initCStream_usingCDict result : %u ", (unsigned)initError); + if (ZSTD_isError(initError)) goto _output_error; + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + DISPLAYLEVEL(5, "- starting ZSTD_compressStream "); + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + DISPLAYLEVEL(5, "- ZSTD_endStream result : %u ", (unsigned)r); + if (r != 0) goto _output_error; /* error, or some data not flushed */ + } + cSize = outBuff.pos; + ZSTD_freeCDict(cdict); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100); + } + + DISPLAYLEVEL(3, "test%3i : check CStream size : ", testNb++); + { size_t const s = ZSTD_sizeof_CStream(zc); + if (ZSTD_isError(s)) goto _output_error; + DISPLAYLEVEL(3, "OK (%u bytes) \n", (unsigned)s); + } + + DISPLAYLEVEL(4, "test%3i : check Dictionary ID : ", testNb++); + { unsigned const dID = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); + if (dID != dictID) goto _output_error; + DISPLAYLEVEL(4, "OK (%u) \n", dID); + } + + /* DDict scenario */ + DISPLAYLEVEL(3, "test%3i : decompress %u bytes with digested dictionary : ", testNb++, (unsigned)CNBufferSize); + { ZSTD_DDict* const ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); + size_t const initError = ZSTD_initDStream_usingDDict(zd, ddict); + if (ZSTD_isError(initError)) goto _output_error; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (r != 0) goto _output_error; } /* should reach end of frame == 0; otherwise, some data left, or an error */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* should regenerate the same amount */ + if (inBuff.pos != inBuff.size) goto _output_error; /* should have read the entire frame */ + ZSTD_freeDDict(ddict); + DISPLAYLEVEL(3, "OK \n"); + } + + /* Memory restriction */ + DISPLAYLEVEL(3, "test%3i : maxWindowSize < frame requirement : ", testNb++); + ZSTD_initDStream_usingDict(zd, CNBuffer, dictSize); + CHECK_Z( ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, 10) ); /* too small limit */ + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + { size_t const r = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (!ZSTD_isError(r)) goto _output_error; /* must fail : frame requires > 100 bytes */ + DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); } + ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters); /* leave zd in good shape for next tests */ + + DISPLAYLEVEL(3, "test%3i : dictionary source size and level : ", testNb++); + { ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + int const maxLevel = 16; /* first level with zstd_opt */ + int level; + assert(maxLevel < ZSTD_maxCLevel()); + CHECK_Z( ZSTD_DCtx_loadDictionary_byReference(dctx, dictionary.start, dictionary.filled) ); + for (level = 1; level <= maxLevel; ++level) { + ZSTD_CDict* const cdict = ZSTD_createCDict(dictionary.start, dictionary.filled, level); + size_t const maxSize = MIN(1 MB, CNBufferSize); + size_t size; + for (size = 512; size <= maxSize; size <<= 1) { + U64 const crcOrig = XXH64(CNBuffer, size, 0); + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + ZSTD_parameters savedParams; + getCCtxParams(cctx, &savedParams); + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = size; + inBuff.pos = 0; + CHECK_Z(ZSTD_CCtx_refCDict(cctx, cdict)); + CHECK_Z(ZSTD_compressStream2(cctx, &outBuff, &inBuff, ZSTD_e_end)); + CHECK(badParameters(cctx, savedParams), "Bad CCtx params"); + if (inBuff.pos != inBuff.size) goto _output_error; + { ZSTD_outBuffer decOut = {decodedBuffer, size, 0}; + ZSTD_inBuffer decIn = {outBuff.dst, outBuff.pos, 0}; + CHECK_Z( ZSTD_decompressStream(dctx, &decOut, &decIn) ); + if (decIn.pos != decIn.size) goto _output_error; + if (decOut.pos != size) goto _output_error; + { U64 const crcDec = XXH64(decOut.dst, decOut.pos, 0); + if (crcDec != crcOrig) goto _output_error; + } } + ZSTD_freeCCtx(cctx); + } + ZSTD_freeCDict(cdict); + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK\n"); + + ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dictionary.start, dictionary.filled) ); + cSize = ZSTD_compress2(zc, compressedBuffer, compressedBufferSize, CNBuffer, MIN(CNBufferSize, 100 KB)); + CHECK_Z(cSize); + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with dictionary : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + /* We should fail to decompress without a dictionary. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + /* We should succeed to decompress with the dictionary. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The dictionary should presist across calls. */ + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The dictionary should not be cleared by ZSTD_reset_session_only. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* When we reset the context the dictionary is cleared. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_resetDStream() with dictionary : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + /* We should succeed to decompress with the dictionary. */ + ZSTD_resetDStream(dctx); + CHECK_Z( ZSTD_DCtx_loadDictionary(dctx, dictionary.start, dictionary.filled) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The dictionary should not be cleared by ZSTD_resetDStream(). */ + ZSTD_resetDStream(dctx); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The dictionary should be cleared by ZSTD_initDStream(). */ + CHECK_Z( ZSTD_initDStream(dctx) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressStream() with ddict : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); + /* We should succeed to decompress with the ddict. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_refDDict(dctx, ddict) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The ddict should presist across calls. */ + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* When we reset the context the ddict is cleared. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + ZSTD_freeDCtx(dctx); + ZSTD_freeDDict(ddict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_decompressDCtx() with prefix : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + /* We should succeed to decompress with the prefix. */ + ZSTD_DCtx_reset(dctx, ZSTD_reset_session_and_parameters); + CHECK_Z( ZSTD_DCtx_refPrefix_advanced(dctx, dictionary.start, dictionary.filled, ZSTD_dct_auto) ); + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + if (ZSTD_decompressStream(dctx, &out, &in) != 0) goto _output_error; + if (in.pos != in.size) goto _output_error; + } + /* The prefix should be cleared after the first compression. */ + { ZSTD_outBuffer out = {decodedBuffer, decodedBufferSize, 0}; + ZSTD_inBuffer in = {compressedBuffer, cSize, 0}; + size_t const ret = ZSTD_decompressStream(dctx, &out, &in); + if (!ZSTD_isError(ret)) goto _output_error; + } + ZSTD_freeDCtx(dctx); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_initDStream*() with dictionary : ", testNb++); + { + ZSTD_DCtx* dctx = ZSTD_createDCtx(); + ZSTD_DDict* ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); + size_t ret; + /* We should succeed to decompress with the dictionary. */ + CHECK_Z( ZSTD_initDStream_usingDict(dctx, dictionary.start, dictionary.filled) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) ); + /* The dictionary should presist across calls. */ + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) ); + /* We should succeed to decompress with the ddict. */ + CHECK_Z( ZSTD_initDStream_usingDDict(dctx, ddict) ); + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) ); + /* The ddict should presist across calls. */ + CHECK_Z( ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize) ); + /* When we reset the context the ddict is cleared. */ + CHECK_Z( ZSTD_initDStream(dctx) ); + ret = ZSTD_decompressDCtx(dctx, decodedBuffer, decodedBufferSize, compressedBuffer, cSize); + if (!ZSTD_isError(ret)) goto _output_error; + ZSTD_freeDCtx(dctx); + ZSTD_freeDDict(ddict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_usingCDict_advanced with masked dictID : ", testNb++); + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, CNBufferSize, dictionary.filled); + ZSTD_frameParameters const fParams = { 1 /* contentSize */, 1 /* checksum */, 1 /* noDictID */}; + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_auto, cParams, ZSTD_defaultCMem); + size_t const initError = ZSTD_initCStream_usingCDict_advanced(zc, cdict, fParams, CNBufferSize); + if (ZSTD_isError(initError)) goto _output_error; + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const r = ZSTD_endStream(zc, &outBuff); + if (r != 0) goto _output_error; } /* error, or some data not flushed */ + cSize = outBuff.pos; + ZSTD_freeCDict(cdict); + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100); + } + + DISPLAYLEVEL(3, "test%3i : try retrieving dictID from frame : ", testNb++); + { U32 const did = ZSTD_getDictID_fromFrame(compressedBuffer, cSize); + if (did != 0) goto _output_error; + } + DISPLAYLEVEL(3, "OK (not detected) \n"); + + DISPLAYLEVEL(3, "test%3i : decompress without dictionary : ", testNb++); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize); + if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */ + DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); + } + + DISPLAYLEVEL(3, "test%3i : compress with ZSTD_CCtx_refPrefix : ", testNb++); + CHECK_Z( ZSTD_CCtx_refPrefix(zc, dictionary.start, dictionary.filled) ); + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + cSize = outBuff.pos; + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100); + + DISPLAYLEVEL(3, "test%3i : decompress with ZSTD_DCtx_refPrefix : ", testNb++); + CHECK_Z( ZSTD_DCtx_refPrefix(zd, dictionary.start, dictionary.filled) ); + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + inBuff.src = compressedBuffer; + inBuff.size = cSize; + inBuff.pos = 0; + CHECK_Z( ZSTD_decompressStream(zd, &outBuff, &inBuff) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + if (outBuff.pos != CNBufferSize) goto _output_error; /* must regenerate whole input */ + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should fail): ", testNb++); + { size_t const r = ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize); + if (!ZSTD_isError(r)) goto _output_error; /* must fail : dictionary not used */ + DISPLAYLEVEL(3, "OK (%s)\n", ZSTD_getErrorName(r)); + } + + DISPLAYLEVEL(3, "test%3i : compress again with ZSTD_compressStream2 : ", testNb++); + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + cSize = outBuff.pos; + DISPLAYLEVEL(3, "OK (%u bytes : %.2f%%)\n", (unsigned)cSize, (double)cSize/CNBufferSize*100); + + DISPLAYLEVEL(3, "test%3i : decompress without dictionary (should work): ", testNb++); + CHECK_Z( ZSTD_decompress(decodedBuffer, CNBufferSize, compressedBuffer, cSize) ); + DISPLAYLEVEL(3, "OK \n"); + + /* Empty srcSize */ + DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced with pledgedSrcSize=0 and dict : ", testNb++); + { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); + params.fParams.contentSizeFlag = 1; + CHECK_Z( ZSTD_initCStream_advanced(zc, dictionary.start, dictionary.filled, params, 0 /* pledgedSrcSize==0 means "empty" when params.fParams.contentSizeFlag is set */) ); + } /* cstream advanced shall write content size = 0 */ + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = 0; + inBuff.pos = 0; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; + cSize = outBuff.pos; + if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : pledgedSrcSize == 0 behaves properly : ", testNb++); + { ZSTD_parameters params = ZSTD_getParams(5, 0, 0); + params.fParams.contentSizeFlag = 1; + CHECK_Z( ZSTD_initCStream_advanced(zc, NULL, 0, params, 0) ); + } /* cstream advanced shall write content size = 0 */ + inBuff.src = CNBuffer; + inBuff.size = 0; + inBuff.pos = 0; + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; + cSize = outBuff.pos; + if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != 0) goto _output_error; + + ZSTD_resetCStream(zc, 0); /* resetCStream should treat 0 as unknown */ + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = 0; + inBuff.pos = 0; + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + if (ZSTD_endStream(zc, &outBuff) != 0) goto _output_error; + cSize = outBuff.pos; + if (ZSTD_findDecompressedSize(compressedBuffer, cSize) != ZSTD_CONTENTSIZE_UNKNOWN) goto _output_error; + DISPLAYLEVEL(3, "OK \n"); + + /* Basic multithreading compression test */ + DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads : ", testNb++, COMPRESSIBLE_NOISE_LENGTH); + { ZSTD_parameters const params = ZSTD_getParams(1, 0, 0); + int jobSize; + CHECK_Z( ZSTDMT_getMTCtxParameter(mtctx, ZSTDMT_p_jobSize, &jobSize)); + CHECK(jobSize != 0, "job size non-zero"); + CHECK_Z( ZSTDMT_initCStream_advanced(mtctx, CNBuffer, dictSize, params, CNBufferSize) ); + CHECK_Z( ZSTDMT_getMTCtxParameter(mtctx, ZSTDMT_p_jobSize, &jobSize)); + CHECK(jobSize != 0, "job size non-zero"); + } + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = CNBufferSize; + inBuff.pos = 0; + { size_t const compressResult = ZSTDMT_compressStream_generic(mtctx, &outBuff, &inBuff, ZSTD_e_end); + if (compressResult != 0) goto _output_error; /* compression must be completed in a single round */ + } + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + { size_t const compressedSize = ZSTD_findFrameCompressedSize(compressedBuffer, outBuff.pos); + if (compressedSize != outBuff.pos) goto _output_error; /* must be a full valid frame */ + } + DISPLAYLEVEL(3, "OK \n"); + + /* Complex multithreading + dictionary test */ + { U32 const nbWorkers = 2; + size_t const jobSize = 4 * 1 MB; + size_t const srcSize = jobSize * nbWorkers; /* we want each job to have predictable size */ + size_t const segLength = 2 KB; + size_t const offset = 600 KB; /* must be larger than window defined in cdict */ + size_t const start = jobSize + (offset-1); + const BYTE* const srcToCopy = (const BYTE*)CNBuffer + start; + BYTE* const dst = (BYTE*)CNBuffer + start - offset; + DISPLAYLEVEL(3, "test%3i : compress %u bytes with multiple threads + dictionary : ", testNb++, (unsigned)srcSize); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, 3) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_nbWorkers, nbWorkers) ); + CHECK_Z( ZSTD_CCtx_setParameter(zc, ZSTD_c_jobSize, jobSize) ); + assert(start > offset); + assert(start + segLength < COMPRESSIBLE_NOISE_LENGTH); + memcpy(dst, srcToCopy, segLength); /* create a long repetition at long distance for job 2 */ + outBuff.dst = compressedBuffer; + outBuff.size = compressedBufferSize; + outBuff.pos = 0; + inBuff.src = CNBuffer; + inBuff.size = srcSize; assert(srcSize < COMPRESSIBLE_NOISE_LENGTH); + inBuff.pos = 0; + } + { ZSTD_compressionParameters const cParams = ZSTD_getCParams(1, 4 KB, dictionary.filled); /* intentionally lies on estimatedSrcSize, to push cdict into targeting a small window size */ + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); + DISPLAYLEVEL(5, "cParams.windowLog = %u : ", cParams.windowLog); + CHECK_Z( ZSTD_CCtx_refCDict(zc, cdict) ); + CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end) ); + CHECK_Z( ZSTD_CCtx_refCDict(zc, NULL) ); /* do not keep a reference to cdict, as its lifetime ends */ + ZSTD_freeCDict(cdict); + } + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + cSize = outBuff.pos; + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : decompress large frame created from multiple threads + dictionary : ", testNb++); + { ZSTD_DStream* const dstream = ZSTD_createDCtx(); + ZSTD_frameHeader zfh; + ZSTD_getFrameHeader(&zfh, compressedBuffer, cSize); + DISPLAYLEVEL(5, "frame windowsize = %u : ", (unsigned)zfh.windowSize); + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + inBuff.src = compressedBuffer; + inBuff.pos = 0; + CHECK_Z( ZSTD_initDStream_usingDict(dstream, dictionary.start, dictionary.filled) ); + inBuff.size = 1; /* avoid shortcut to single-pass mode */ + CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) ); + inBuff.size = cSize; + CHECK_Z( ZSTD_decompressStream(dstream, &outBuff, &inBuff) ); + if (inBuff.pos != inBuff.size) goto _output_error; /* entire input should be consumed */ + ZSTD_freeDStream(dstream); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : check dictionary FSE tables can represent every code : ", testNb++); + { unsigned const kMaxWindowLog = 24; + unsigned value; + ZSTD_compressionParameters cParams = ZSTD_getCParams(3, 1U << kMaxWindowLog, 1024); + ZSTD_CDict* cdict; + ZSTD_DDict* ddict; + SEQ_stream seq = SEQ_initStream(0x87654321); + SEQ_gen_type type; + XXH64_state_t xxh; + + XXH64_reset(&xxh, 0); + cParams.windowLog = kMaxWindowLog; + cdict = ZSTD_createCDict_advanced(dictionary.start, dictionary.filled, ZSTD_dlm_byRef, ZSTD_dct_fullDict, cParams, ZSTD_defaultCMem); + ddict = ZSTD_createDDict(dictionary.start, dictionary.filled); + + if (!cdict || !ddict) goto _output_error; + + ZSTD_CCtx_reset(zc, ZSTD_reset_session_only); + ZSTD_resetDStream(zd); + CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict)); + CHECK_Z(ZSTD_initDStream_usingDDict(zd, ddict)); + CHECK_Z(ZSTD_DCtx_setParameter(zd, ZSTD_d_windowLogMax, kMaxWindowLog)); + /* Test all values < 300 */ + for (value = 0; value < 300; ++value) { + for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) { + CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value)); + } + } + /* Test values 2^8 to 2^17 */ + for (value = (1 << 8); value < (1 << 17); value <<= 1) { + for (type = (SEQ_gen_type)0; type < SEQ_gen_max; ++type) { + CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value)); + CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, type, value + (value >> 2))); + } + } + /* Test offset values up to the max window log */ + for (value = 8; value <= kMaxWindowLog; ++value) { + CHECK_Z(SEQ_generateRoundTrip(zc, zd, &xxh, &seq, SEQ_gen_of, (1U << value) - 1)); + } + + CHECK_Z(SEQ_roundTrip(zc, zd, &xxh, NULL, 0, ZSTD_e_end)); + CHECK(SEQ_digest(&seq) != XXH64_digest(&xxh), "SEQ XXH64 does not match"); + + ZSTD_freeCDict(cdict); + ZSTD_freeDDict(ddict); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_srcSize sets requestedParams : ", testNb++); + { int level; + CHECK_Z(ZSTD_initCStream_srcSize(zc, 11, ZSTD_CONTENTSIZE_UNKNOWN)); + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level)); + CHECK(level != 11, "Compression level does not match"); + ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN); + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_compressionLevel, &level)); + CHECK(level != 11, "Compression level does not match"); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_initCStream_advanced sets requestedParams : ", testNb++); + { ZSTD_parameters const params = ZSTD_getParams(9, 0, 0); + CHECK_Z(ZSTD_initCStream_advanced(zc, NULL, 0, params, ZSTD_CONTENTSIZE_UNKNOWN)); + CHECK(badParameters(zc, params), "Compression parameters do not match"); + ZSTD_resetCStream(zc, ZSTD_CONTENTSIZE_UNKNOWN); + CHECK(badParameters(zc, params), "Compression parameters do not match"); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : ZSTD_c_srcSizeHint bounds : ", testNb++); + ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters); + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, INT_MAX)); + { int srcSizeHint; + CHECK_Z(ZSTD_CCtx_getParameter(zc, ZSTD_c_srcSizeHint, &srcSizeHint)); + CHECK(!(srcSizeHint == INT_MAX), "srcSizeHint doesn't match"); + } + CHECK(!ZSTD_isError(ZSTD_CCtx_setParameter(zc, ZSTD_c_srcSizeHint, -1)), "Out of range doesn't error"); + DISPLAYLEVEL(3, "OK \n"); + + /* Overlen overwriting window data bug */ + DISPLAYLEVEL(3, "test%3i : wildcopy doesn't overwrite potential match data : ", testNb++); + { /* This test has a window size of 1024 bytes and consists of 3 blocks: + 1. 'a' repeated 517 times + 2. 'b' repeated 516 times + 3. a compressed block with no literals and 3 sequence commands: + litlength = 0, offset = 24, match length = 24 + litlength = 0, offset = 24, match length = 3 (this one creates an overlength write of length 2*WILDCOPY_OVERLENGTH - 3) + litlength = 0, offset = 1021, match length = 3 (this one will try to read from overwritten data if the buffer is too small) */ + + const char* testCase = + "\x28\xB5\x2F\xFD\x04\x00\x4C\x00\x00\x10\x61\x61\x01\x00\x00\x2A" + "\x80\x05\x44\x00\x00\x08\x62\x01\x00\x00\x2A\x20\x04\x5D\x00\x00" + "\x00\x03\x40\x00\x00\x64\x60\x27\xB0\xE0\x0C\x67\x62\xCE\xE0"; + ZSTD_DStream* const zds = ZSTD_createDStream(); + if (zds==NULL) goto _output_error; + + CHECK_Z( ZSTD_initDStream(zds) ); + inBuff.src = testCase; + inBuff.size = 47; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + + while (inBuff.pos < inBuff.size) { + CHECK_Z( ZSTD_decompressStream(zds, &outBuff, &inBuff) ); + } + + ZSTD_freeDStream(zds); + } + DISPLAYLEVEL(3, "OK \n"); + + /* Small Sequence Section bug */ + DISPLAYLEVEL(3, "test%3i : decompress blocks with small sequences section : ", testNb++); + { /* This test consists of 3 blocks. Each block has one sequence. + The sequence has literal length of 10, match length of 10 and offset of 10. + The sequence value and compression mode for the blocks are following: + The order of values are ll, ml, of. + - First block : (10, 7, 13) (rle, rle, rle) + - size of sequences section: 6 bytes (1 byte for nbSeq, 1 byte for encoding mode, 3 bytes for rle, 1 byte bitstream) + - Second block : (10, 7, 1) (repeat, repeat, rle) + - size of sequences section: 4 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 bytes for rle, 1 byte bitstream) + - Third block : (10, 7, 1) (repeat, repeat, repeat) + - size of sequences section: 3 bytes (1 byte for nbSeq, 1 byte for encoding mode, 1 byte bitstream) */ + + unsigned char compressed[] = { + 0x28, 0xb5, 0x2f, 0xfd, 0x24, 0x3c, 0x35, 0x01, 0x00, 0xf0, 0x85, 0x08, + 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b, + 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, + 0x69, 0x94, 0x89, 0x1c, 0x03, 0x44, 0x0a, 0x07, 0x00, 0xb4, 0x04, 0x80, + 0x40, 0x0a, 0xa4 + }; + unsigned int compressedSize = 51; + unsigned char decompressed[] = { + 0x85, 0x08, 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x85, 0x08, + 0xc2, 0xc4, 0x70, 0xcf, 0xd7, 0xc0, 0x96, 0x7e, 0x4c, 0x6b, 0xa9, 0x8b, + 0xbc, 0xc5, 0xb6, 0xd9, 0x7f, 0x4c, 0x4c, 0x6b, 0xa9, 0x8b, 0xbc, 0xc5, + 0xb6, 0xd9, 0x7f, 0x4c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, + 0x89, 0x1c, 0xf1, 0x05, 0xa6, 0x54, 0xef, 0xac, 0x69, 0x94, 0x89, 0x1c + }; + unsigned int decompressedSize = 60; + + ZSTD_DStream* const zds = ZSTD_createDStream(); + if (zds==NULL) goto _output_error; + + CHECK_Z( ZSTD_initDStream(zds) ); + inBuff.src = compressed; + inBuff.size = compressedSize; + inBuff.pos = 0; + outBuff.dst = decodedBuffer; + outBuff.size = CNBufferSize; + outBuff.pos = 0; + + CHECK(ZSTD_decompressStream(zds, &outBuff, &inBuff) != 0, + "Decompress did not reach the end of frame"); + CHECK(inBuff.pos != inBuff.size, "Decompress did not fully consume input"); + CHECK(outBuff.pos != decompressedSize, "Decompressed size does not match"); + CHECK(memcmp(outBuff.dst, decompressed, decompressedSize) != 0, + "Decompressed data does not match"); + + ZSTD_freeDStream(zds); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : raw block can be streamed: ", testNb++); + { size_t const inputSize = 10000; + size_t const compCapacity = ZSTD_compressBound(inputSize); + BYTE* const input = (BYTE*)malloc(inputSize); + BYTE* const comp = (BYTE*)malloc(compCapacity); + BYTE* const decomp = (BYTE*)malloc(inputSize); + + CHECK(input == NULL || comp == NULL || decomp == NULL, "failed to alloc buffers"); + + RDG_genBuffer(input, inputSize, 0.0, 0.0, seed); + { size_t const compSize = ZSTD_compress(comp, compCapacity, input, inputSize, -(int)inputSize); + ZSTD_inBuffer in = { comp, 0, 0 }; + ZSTD_outBuffer out = { decomp, 0, 0 }; + CHECK_Z(compSize); + CHECK_Z( ZSTD_DCtx_reset(zd, ZSTD_reset_session_and_parameters) ); + while (in.size < compSize) { + in.size = MIN(in.size + 100, compSize); + while (in.pos < in.size) { + size_t const outPos = out.pos; + if (out.pos == out.size) { + out.size = MIN(out.size + 10, inputSize); + } + CHECK_Z( ZSTD_decompressStream(zd, &out, &in) ); + CHECK(!(out.pos > outPos), "We are not streaming (no output generated)"); + } + } + CHECK(in.pos != compSize, "Not all input consumed!"); + CHECK(out.pos != inputSize, "Not all output produced!"); + } + CHECK(memcmp(input, decomp, inputSize), "round trip failed!"); + + free(input); + free(comp); + free(decomp); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : dictionary + uncompressible block + reusing tables checks offset table validity: ", testNb++); + { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced( + dictionary.start, dictionary.filled, + ZSTD_dlm_byRef, ZSTD_dct_fullDict, + ZSTD_getCParams(3, 0, dictionary.filled), + ZSTD_defaultCMem); + const size_t inbufsize = 2 * 128 * 1024; /* 2 blocks */ + const size_t outbufsize = ZSTD_compressBound(inbufsize); + size_t inbufpos = 0; + size_t cursegmentlen; + BYTE *inbuf = (BYTE *)malloc(inbufsize); + BYTE *outbuf = (BYTE *)malloc(outbufsize); + BYTE *checkbuf = (BYTE *)malloc(inbufsize); + size_t ret; + + CHECK(cdict == NULL, "failed to alloc cdict"); + CHECK(inbuf == NULL, "failed to alloc input buffer"); + + /* first block is uncompressible */ + cursegmentlen = 128 * 1024; + RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0., 0., seed); + inbufpos += cursegmentlen; + + /* second block is compressible */ + cursegmentlen = 128 * 1024 - 256; + RDG_genBuffer(inbuf + inbufpos, cursegmentlen, 0.05, 0., seed); + inbufpos += cursegmentlen; + + /* and includes a very long backref */ + cursegmentlen = 128; + memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 256, cursegmentlen); + inbufpos += cursegmentlen; + + /* and includes a very long backref */ + cursegmentlen = 128; + memcpy(inbuf + inbufpos, (BYTE*)dictionary.start + 128, cursegmentlen); + inbufpos += cursegmentlen; + + ret = ZSTD_compress_usingCDict(zc, outbuf, outbufsize, inbuf, inbufpos, cdict); + CHECK_Z(ret); + + ret = ZSTD_decompress_usingDict(zd, checkbuf, inbufsize, outbuf, ret, dictionary.start, dictionary.filled); + CHECK_Z(ret); + + CHECK(memcmp(inbuf, checkbuf, inbufpos), "start and finish buffers don't match"); + + ZSTD_freeCDict(cdict); + free(inbuf); + free(outbuf); + free(checkbuf); + } + DISPLAYLEVEL(3, "OK \n"); + + DISPLAYLEVEL(3, "test%3i : dictionary + small blocks + reusing tables checks offset table validity: ", testNb++); + { ZSTD_CDict* const cdict = ZSTD_createCDict_advanced( + dictionary.start, dictionary.filled, + ZSTD_dlm_byRef, ZSTD_dct_fullDict, + ZSTD_getCParams(3, 0, dictionary.filled), + ZSTD_defaultCMem); + ZSTD_outBuffer out = {compressedBuffer, compressedBufferSize, 0}; + int remainingInput = 256 * 1024; + int offset; + + CHECK_Z(ZSTD_CCtx_reset(zc, ZSTD_reset_session_and_parameters)); + CHECK_Z(ZSTD_CCtx_refCDict(zc, cdict)); + CHECK_Z(ZSTD_CCtx_setParameter(zc, ZSTD_c_checksumFlag, 1)); + /* Write a bunch of 6 byte blocks */ + while (remainingInput > 0) { + char testBuffer[6] = "\xAA\xAA\xAA\xAA\xAA\xAA"; + const size_t kSmallBlockSize = sizeof(testBuffer); + ZSTD_inBuffer in = {testBuffer, kSmallBlockSize, 0}; + + CHECK_Z(ZSTD_compressStream2(zc, &out, &in, ZSTD_e_flush)); + CHECK(in.pos != in.size, "input not fully consumed"); + remainingInput -= kSmallBlockSize; + } + /* Write several very long offset matches into the dictionary */ + for (offset = 1024; offset >= 0; offset -= 128) { + ZSTD_inBuffer in = {(BYTE*)dictionary.start + offset, 128, 0}; + ZSTD_EndDirective flush = offset > 0 ? ZSTD_e_continue : ZSTD_e_end; + CHECK_Z(ZSTD_compressStream2(zc, &out, &in, flush)); + CHECK(in.pos != in.size, "input not fully consumed"); + } + /* Ensure decompression works */ + CHECK_Z(ZSTD_decompress_usingDict(zd, decodedBuffer, CNBufferSize, out.dst, out.pos, dictionary.start, dictionary.filled)); + + ZSTD_freeCDict(cdict); + } + DISPLAYLEVEL(3, "OK \n"); + +_end: + FUZ_freeDictionary(dictionary); + ZSTD_freeCStream(zc); + ZSTD_freeDStream(zd); + ZSTDMT_freeCCtx(mtctx); + free(CNBuffer); + free(compressedBuffer); + free(decodedBuffer); + return testResult; + +_output_error: + testResult = 1; + DISPLAY("Error detected in Unit tests ! \n"); + goto _end; +} + + +/* ====== Fuzzer tests ====== */ + +static size_t findDiff(const void* buf1, const void* buf2, size_t max) +{ + const BYTE* b1 = (const BYTE*)buf1; + const BYTE* b2 = (const BYTE*)buf2; + size_t u; + for (u=0; u<max; u++) { + if (b1[u] != b2[u]) break; + } + if (u==max) { + DISPLAY("=> No difference detected within %u bytes \n", (unsigned)max); + return u; + } + DISPLAY("Error at position %u / %u \n", (unsigned)u, (unsigned)max); + if (u>=3) + DISPLAY(" %02X %02X %02X ", + b1[u-3], b1[u-2], b1[u-1]); + DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n", + b1[u], b1[u+1], b1[u+2], b1[u+3], b1[u+4], b1[u+5]); + if (u>=3) + DISPLAY(" %02X %02X %02X ", + b2[u-3], b2[u-2], b2[u-1]); + DISPLAY(" :%02X: %02X %02X %02X %02X %02X \n", + b2[u], b2[u+1], b2[u+2], b2[u+3], b2[u+4], b2[u+5]); + return u; +} + +static size_t FUZ_rLogLength(U32* seed, U32 logLength) +{ + size_t const lengthMask = ((size_t)1 << logLength) - 1; + return (lengthMask+1) + (FUZ_rand(seed) & lengthMask); +} + +static size_t FUZ_randomLength(U32* seed, U32 maxLog) +{ + U32 const logLength = FUZ_rand(seed) % maxLog; + return FUZ_rLogLength(seed, logLength); +} + +/* Return value in range minVal <= v <= maxVal */ +static U32 FUZ_randomClampedLength(U32* seed, U32 minVal, U32 maxVal) +{ + U32 const mod = maxVal < minVal ? 1 : (maxVal + 1) - minVal; + return (U32)((FUZ_rand(seed) % mod) + minVal); +} + +static int fuzzerTests(U32 seed, unsigned nbTests, unsigned startTest, double compressibility, int bigTests) +{ + U32 const maxSrcLog = bigTests ? 24 : 22; + static const U32 maxSampleLog = 19; + size_t const srcBufferSize = (size_t)1<<maxSrcLog; + BYTE* cNoiseBuffer[5]; + size_t const copyBufferSize = srcBufferSize + (1<<maxSampleLog); + BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize); + size_t const cBufferSize = ZSTD_compressBound(srcBufferSize); + BYTE* const cBuffer = (BYTE*)malloc (cBufferSize); + size_t const dstBufferSize = srcBufferSize; + BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize); + U32 result = 0; + unsigned testNb = 0; + U32 coreSeed = seed; + ZSTD_CStream* zc = ZSTD_createCStream(); /* will be re-created sometimes */ + ZSTD_DStream* zd = ZSTD_createDStream(); /* will be re-created sometimes */ + ZSTD_DStream* const zd_noise = ZSTD_createDStream(); + UTIL_time_t const startClock = UTIL_getTime(); + const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */ + size_t dictSize = 0; + U32 oldTestLog = 0; + U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel() : g_cLevelMax_smallTests; + + /* allocations */ + cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); + CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || + !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise , + "Not enough memory, fuzzer tests cancelled"); + + /* Create initial samples */ + RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ + RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ + RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); + RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ + RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ + memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */ + ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ + + /* catch up testNb */ + for (testNb=1; testNb < startTest; testNb++) + FUZ_rand(&coreSeed); + + /* test loop */ + for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) { + U32 lseed; + const BYTE* srcBuffer; + size_t totalTestSize, totalGenSize, cSize; + XXH64_state_t xxhState; + U64 crcOrig; + U32 resetAllowed = 1; + size_t maxTestSize; + + /* init */ + FUZ_rand(&coreSeed); + lseed = coreSeed ^ prime32; + if (nbTests >= testNb) { + DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); + } else { + DISPLAYUPDATE(2, "\r%6u ", testNb); + } + + /* states full reset (deliberately not synchronized) */ + /* some issues can only happen when reusing states */ + if ((FUZ_rand(&lseed) & 0xFF) == 131) { + ZSTD_freeCStream(zc); + zc = ZSTD_createCStream(); + CHECK(zc==NULL, "ZSTD_createCStream : allocation error"); + resetAllowed=0; + } + if ((FUZ_rand(&lseed) & 0xFF) == 132) { + ZSTD_freeDStream(zd); + zd = ZSTD_createDStream(); + CHECK(zd==NULL, "ZSTD_createDStream : allocation error"); + CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */ + } + + /* srcBuffer selection [0-4] */ + { U32 buffNb = FUZ_rand(&lseed) & 0x7F; + if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ + else { + buffNb >>= 3; + if (buffNb & 7) { + const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ + buffNb = tnb[buffNb >> 3]; + } else { + const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ + buffNb = tnb[buffNb >> 3]; + } } + srcBuffer = cNoiseBuffer[buffNb]; + } + + /* compression init */ + if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ + && oldTestLog /* at least one test happened */ && resetAllowed) { + maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); + maxTestSize = MIN(maxTestSize, srcBufferSize-16); + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? 0 : maxTestSize; + CHECK_Z( ZSTD_resetCStream(zc, pledgedSrcSize) ); + } + } else { + U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const cLevelCandidate = ( FUZ_rand(&lseed) % + (ZSTD_maxCLevel() - + (MAX(testLog, dictLog) / 3))) + + 1; + U32 const cLevel = MIN(cLevelCandidate, cLevelMax); + maxTestSize = FUZ_rLogLength(&lseed, testLog); + oldTestLog = testLog; + /* random dictionary selection */ + dictSize = ((FUZ_rand(&lseed)&7)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; + } + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; + ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); + params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; + params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; + params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; + CHECK_Z ( ZSTD_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); + } } + + /* multi-segments compression test */ + XXH64_reset(&xxhState, 0); + { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; + U32 n; + for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) { + /* compress random chunks into randomly sized dst buffers */ + { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize); + size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); + ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; + outBuff.size = outBuff.pos + dstBuffSize; + + CHECK_Z( ZSTD_compressStream(zc, &outBuff, &inBuff) ); + + XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); + memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); + totalTestSize += inBuff.pos; + } + + /* random flush operation, to mess around */ + if ((FUZ_rand(&lseed) & 15) == 0) { + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); + outBuff.size = outBuff.pos + adjustedDstSize; + CHECK_Z( ZSTD_flushStream(zc, &outBuff) ); + } } + + /* final frame epilogue */ + { size_t remainingToFlush = (size_t)(-1); + while (remainingToFlush) { + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); + outBuff.size = outBuff.pos + adjustedDstSize; + remainingToFlush = ZSTD_endStream(zc, &outBuff); + CHECK (ZSTD_isError(remainingToFlush), "end error : %s", ZSTD_getErrorName(remainingToFlush)); + } } + crcOrig = XXH64_digest(&xxhState); + cSize = outBuff.pos; + } + + /* multi - fragments decompression test */ + if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { + CHECK_Z ( ZSTD_resetDStream(zd) ); + } else { + CHECK_Z ( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); + } + { size_t decompressionResult = 1; + ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + for (totalGenSize = 0 ; decompressionResult ; ) { + size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); + inBuff.size = inBuff.pos + readCSrcSize; + outBuff.size = outBuff.pos + dstBuffSize; + decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (ZSTD_getErrorCode(decompressionResult) == ZSTD_error_checksum_wrong) { + DISPLAY("checksum error : \n"); + findDiff(copyBuffer, dstBuffer, totalTestSize); + } + CHECK( ZSTD_isError(decompressionResult), "decompression error : %s", + ZSTD_getErrorName(decompressionResult) ); + } + CHECK (decompressionResult != 0, "frame not fully decoded"); + CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", + (unsigned)outBuff.pos, (unsigned)totalTestSize); + CHECK (inBuff.pos != cSize, "compressed data should be fully read") + { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); + if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); + CHECK (crcDest!=crcOrig, "decompressed data corrupted"); + } } + + /*===== noisy/erroneous src decompression test =====*/ + + /* add some noise */ + { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; + U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) { + size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const noiseSize = MIN((cSize/3) , randomNoiseSize); + size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize); + size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize); + memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize); + } } + + /* try decompression on noisy data */ + CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */ + { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + while (outBuff.pos < dstBufferSize) { + size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize); + size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize); + outBuff.size = outBuff.pos + adjustedDstSize; + inBuff.size = inBuff.pos + adjustedCSrcSize; + { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (ZSTD_isError(decompressError)) break; /* error correctly detected */ + /* No forward progress possible */ + if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break; + } } } } + DISPLAY("\r%u fuzzer tests completed \n", testNb); + +_cleanup: + ZSTD_freeCStream(zc); + ZSTD_freeDStream(zd); + ZSTD_freeDStream(zd_noise); + free(cNoiseBuffer[0]); + free(cNoiseBuffer[1]); + free(cNoiseBuffer[2]); + free(cNoiseBuffer[3]); + free(cNoiseBuffer[4]); + free(copyBuffer); + free(cBuffer); + free(dstBuffer); + return result; + +_output_error: + result = 1; + goto _cleanup; +} + + +/* fuzzing ZSTDMT_* interface */ +static int fuzzerTests_MT(U32 seed, int nbTests, int startTest, + double compressibility, int bigTests) +{ + const U32 maxSrcLog = bigTests ? 24 : 22; + static const U32 maxSampleLog = 19; + size_t const srcBufferSize = (size_t)1<<maxSrcLog; + BYTE* cNoiseBuffer[5]; + size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog); + BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize); + size_t const cBufferSize = ZSTD_compressBound(srcBufferSize); + BYTE* const cBuffer = (BYTE*)malloc (cBufferSize); + size_t const dstBufferSize = srcBufferSize; + BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize); + U32 result = 0; + int testNb = 0; + U32 coreSeed = seed; + int nbThreads = 2; + ZSTDMT_CCtx* zc = ZSTDMT_createCCtx(nbThreads); /* will be reset sometimes */ + ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ + ZSTD_DStream* const zd_noise = ZSTD_createDStream(); + UTIL_time_t const startClock = UTIL_getTime(); + const BYTE* dict=NULL; /* can keep same dict on 2 consecutive tests */ + size_t dictSize = 0; + int const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests; + U32 const nbThreadsMax = bigTests ? 4 : 2; + + /* allocations */ + cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); + CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || + !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise , + "Not enough memory, fuzzer tests cancelled"); + + /* Create initial samples */ + RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ + RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ + RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); + RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ + RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ + memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */ + ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ + DISPLAYLEVEL(6, "Creating initial context with %i threads \n", nbThreads); + + /* catch up testNb */ + for (testNb=1; testNb < startTest; testNb++) + FUZ_rand(&coreSeed); + + /* test loop */ + for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) { + U32 lseed; + const BYTE* srcBuffer; + size_t totalTestSize, totalGenSize, cSize; + XXH64_state_t xxhState; + U64 crcOrig; + size_t maxTestSize; + + FUZ_rand(&coreSeed); + if (nbTests >= testNb) { + DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); + } else { + DISPLAYUPDATE(2, "\r%6u ", testNb); + } + lseed = coreSeed ^ prime32; + + /* states full reset (deliberately not synchronized) */ + /* some issues can only happen when reusing states */ + if ((FUZ_rand(&lseed) & 0xFF) == 131) { + nbThreads = (FUZ_rand(&lseed) % nbThreadsMax) + 1; + DISPLAYLEVEL(5, "Creating new context with %u threads \n", nbThreads); + ZSTDMT_freeCCtx(zc); + zc = ZSTDMT_createCCtx(nbThreads); + CHECK(zc==NULL, "ZSTDMT_createCCtx allocation error") + } + if ((FUZ_rand(&lseed) & 0xFF) == 132) { + ZSTD_freeDStream(zd); + zd = ZSTD_createDStream(); + CHECK(zd==NULL, "ZSTDMT_createCCtx allocation error") + ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ + } + + /* srcBuffer selection [0-4] */ + { U32 buffNb = FUZ_rand(&lseed) & 0x7F; + if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ + else { + buffNb >>= 3; + if (buffNb & 7) { + const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ + buffNb = tnb[buffNb >> 3]; + } else { + const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ + buffNb = tnb[buffNb >> 3]; + } } + srcBuffer = cNoiseBuffer[buffNb]; + } + + /* compression init */ + { U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; + int const cLevelCandidate = ( FUZ_rand(&lseed) + % (ZSTD_maxCLevel() - (MAX(testLog, dictLog) / 2)) ) + + 1; + int const cLevelThreadAdjusted = cLevelCandidate - (nbThreads * 2) + 2; /* reduce cLevel when multiple threads to reduce memory consumption */ + int const cLevelMin = MAX(cLevelThreadAdjusted, 1); /* no negative cLevel yet */ + int const cLevel = MIN(cLevelMin, cLevelMax); + maxTestSize = FUZ_rLogLength(&lseed, testLog); + + if (FUZ_rand(&lseed)&1) { /* simple init */ + int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; + DISPLAYLEVEL(5, "Init with compression level = %i \n", compressionLevel); + CHECK_Z( ZSTDMT_initCStream(zc, compressionLevel) ); + } else { /* advanced init */ + /* random dictionary selection */ + dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; + } + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; + ZSTD_parameters params = ZSTD_getParams(cLevel, pledgedSrcSize, dictSize); + DISPLAYLEVEL(5, "Init with windowLog = %u, pledgedSrcSize = %u, dictSize = %u \n", + params.cParams.windowLog, (unsigned)pledgedSrcSize, (unsigned)dictSize); + params.fParams.checksumFlag = FUZ_rand(&lseed) & 1; + params.fParams.noDictIDFlag = FUZ_rand(&lseed) & 1; + params.fParams.contentSizeFlag = FUZ_rand(&lseed) & 1; + DISPLAYLEVEL(5, "checksumFlag : %u \n", params.fParams.checksumFlag); + CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_overlapLog, FUZ_rand(&lseed) % 12) ); + CHECK_Z( ZSTDMT_setMTCtxParameter(zc, ZSTDMT_p_jobSize, FUZ_rand(&lseed) % (2*maxTestSize+1)) ); /* custom job size */ + CHECK_Z( ZSTDMT_initCStream_advanced(zc, dict, dictSize, params, pledgedSrcSize) ); + } } } + + /* multi-segments compression test */ + XXH64_reset(&xxhState, 0); + { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; + U32 n; + for (n=0, cSize=0, totalTestSize=0 ; totalTestSize < maxTestSize ; n++) { + /* compress random chunks into randomly sized dst buffers */ + { size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const srcSize = MIN (maxTestSize-totalTestSize, randomSrcSize); + size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); + ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; + outBuff.size = outBuff.pos + dstBuffSize; + + DISPLAYLEVEL(6, "Sending %u bytes to compress \n", (unsigned)srcSize); + CHECK_Z( ZSTDMT_compressStream(zc, &outBuff, &inBuff) ); + DISPLAYLEVEL(6, "%u bytes read by ZSTDMT_compressStream \n", (unsigned)inBuff.pos); + + XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); + memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); + totalTestSize += inBuff.pos; + } + + /* random flush operation, to mess around */ + if ((FUZ_rand(&lseed) & 15) == 0) { + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); + size_t const previousPos = outBuff.pos; + outBuff.size = outBuff.pos + adjustedDstSize; + DISPLAYLEVEL(5, "Flushing into dst buffer of size %u \n", (unsigned)adjustedDstSize); + CHECK_Z( ZSTDMT_flushStream(zc, &outBuff) ); + assert(outBuff.pos >= previousPos); + DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_flushStream \n", (unsigned)(outBuff.pos-previousPos)); + } } + + /* final frame epilogue */ + { size_t remainingToFlush = (size_t)(-1); + while (remainingToFlush) { + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); + size_t const previousPos = outBuff.pos; + outBuff.size = outBuff.pos + adjustedDstSize; + DISPLAYLEVEL(5, "Ending into dst buffer of size %u \n", (unsigned)adjustedDstSize); + remainingToFlush = ZSTDMT_endStream(zc, &outBuff); + CHECK (ZSTD_isError(remainingToFlush), "ZSTDMT_endStream error : %s", ZSTD_getErrorName(remainingToFlush)); + assert(outBuff.pos >= previousPos); + DISPLAYLEVEL(6, "%u bytes flushed by ZSTDMT_endStream \n", (unsigned)(outBuff.pos-previousPos)); + DISPLAYLEVEL(5, "endStream : remainingToFlush : %u \n", (unsigned)remainingToFlush); + } } + crcOrig = XXH64_digest(&xxhState); + cSize = outBuff.pos; + DISPLAYLEVEL(5, "Frame completed : %u bytes compressed into %u bytes \n", + (unsigned)totalTestSize, (unsigned)cSize); + } + + /* multi - fragments decompression test */ + assert(totalTestSize < dstBufferSize); + memset(dstBuffer, 170, totalTestSize); /* init dest area */ + if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { + CHECK_Z( ZSTD_resetDStream(zd) ); + } else { + CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); + } + { size_t decompressionResult = 1; + ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + for (totalGenSize = 0 ; decompressionResult ; ) { + size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); + inBuff.size = inBuff.pos + readCSrcSize; + outBuff.size = outBuff.pos + dstBuffSize; + DISPLAYLEVEL(6, "ZSTD_decompressStream input %u bytes into outBuff %u bytes \n", + (unsigned)readCSrcSize, (unsigned)dstBuffSize); + decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (ZSTD_isError(decompressionResult)) { + DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult)); + findDiff(copyBuffer, dstBuffer, totalTestSize); + } + CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); + DISPLAYLEVEL(6, "total ingested (inBuff.pos) = %u and produced (outBuff.pos) = %u \n", + (unsigned)inBuff.pos, (unsigned)outBuff.pos); + } + CHECK (outBuff.pos != totalTestSize, + "decompressed data : wrong size (%u != %u)", + (unsigned)outBuff.pos, (unsigned)totalTestSize ); + CHECK (inBuff.pos != cSize, + "compressed data should be fully read (%u != %u)", + (unsigned)inBuff.pos, (unsigned)cSize ); + { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); + if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); + CHECK (crcDest!=crcOrig, "decompressed data corrupted"); + } } + + /*===== noisy/erroneous src decompression test =====*/ + + /* add some noise */ + { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; + U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) { + size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const noiseSize = MIN((cSize/3) , randomNoiseSize); + size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize); + size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize); + memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize); + } } + + /* try decompression on noisy data */ + CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */ + { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + while (outBuff.pos < dstBufferSize) { + size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize); + size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize); + outBuff.size = outBuff.pos + adjustedDstSize; + inBuff.size = inBuff.pos + adjustedCSrcSize; + { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (ZSTD_isError(decompressError)) break; /* error correctly detected */ + /* No forward progress possible */ + if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break; + } } } } + DISPLAY("\r%u fuzzer tests completed \n", testNb); + +_cleanup: + ZSTDMT_freeCCtx(zc); + ZSTD_freeDStream(zd); + ZSTD_freeDStream(zd_noise); + free(cNoiseBuffer[0]); + free(cNoiseBuffer[1]); + free(cNoiseBuffer[2]); + free(cNoiseBuffer[3]); + free(cNoiseBuffer[4]); + free(copyBuffer); + free(cBuffer); + free(dstBuffer); + return result; + +_output_error: + result = 1; + goto _cleanup; +} + +/** If useOpaqueAPI, sets param in cctxParams. + * Otherwise, sets the param in zc. */ +static size_t setCCtxParameter(ZSTD_CCtx* zc, ZSTD_CCtx_params* cctxParams, + ZSTD_cParameter param, unsigned value, + int useOpaqueAPI) +{ + if (useOpaqueAPI) { + return ZSTD_CCtxParams_setParameter(cctxParams, param, value); + } else { + return ZSTD_CCtx_setParameter(zc, param, value); + } +} + +/* Tests for ZSTD_compress_generic() API */ +static int fuzzerTests_newAPI(U32 seed, int nbTests, int startTest, + double compressibility, int bigTests) +{ + U32 const maxSrcLog = bigTests ? 24 : 22; + static const U32 maxSampleLog = 19; + size_t const srcBufferSize = (size_t)1<<maxSrcLog; + BYTE* cNoiseBuffer[5]; + size_t const copyBufferSize= srcBufferSize + (1<<maxSampleLog); + BYTE* const copyBuffer = (BYTE*)malloc (copyBufferSize); + size_t const cBufferSize = ZSTD_compressBound(srcBufferSize); + BYTE* const cBuffer = (BYTE*)malloc (cBufferSize); + size_t const dstBufferSize = srcBufferSize; + BYTE* const dstBuffer = (BYTE*)malloc (dstBufferSize); + U32 result = 0; + int testNb = 0; + U32 coreSeed = seed; + ZSTD_CCtx* zc = ZSTD_createCCtx(); /* will be reset sometimes */ + ZSTD_DStream* zd = ZSTD_createDStream(); /* will be reset sometimes */ + ZSTD_DStream* const zd_noise = ZSTD_createDStream(); + UTIL_time_t const startClock = UTIL_getTime(); + const BYTE* dict = NULL; /* can keep same dict on 2 consecutive tests */ + size_t dictSize = 0; + U32 oldTestLog = 0; + U32 windowLogMalus = 0; /* can survive between 2 loops */ + U32 const cLevelMax = bigTests ? (U32)ZSTD_maxCLevel()-1 : g_cLevelMax_smallTests; + U32 const nbThreadsMax = bigTests ? 4 : 2; + ZSTD_CCtx_params* cctxParams = ZSTD_createCCtxParams(); + + /* allocations */ + cNoiseBuffer[0] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[1] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[2] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[3] = (BYTE*)malloc (srcBufferSize); + cNoiseBuffer[4] = (BYTE*)malloc (srcBufferSize); + CHECK (!cNoiseBuffer[0] || !cNoiseBuffer[1] || !cNoiseBuffer[2] || !cNoiseBuffer[3] || !cNoiseBuffer[4] || + !copyBuffer || !dstBuffer || !cBuffer || !zc || !zd || !zd_noise , + "Not enough memory, fuzzer tests cancelled"); + + /* Create initial samples */ + RDG_genBuffer(cNoiseBuffer[0], srcBufferSize, 0.00, 0., coreSeed); /* pure noise */ + RDG_genBuffer(cNoiseBuffer[1], srcBufferSize, 0.05, 0., coreSeed); /* barely compressible */ + RDG_genBuffer(cNoiseBuffer[2], srcBufferSize, compressibility, 0., coreSeed); + RDG_genBuffer(cNoiseBuffer[3], srcBufferSize, 0.95, 0., coreSeed); /* highly compressible */ + RDG_genBuffer(cNoiseBuffer[4], srcBufferSize, 1.00, 0., coreSeed); /* sparse content */ + memset(copyBuffer, 0x65, copyBufferSize); /* make copyBuffer considered initialized */ + CHECK_Z( ZSTD_initDStream_usingDict(zd, NULL, 0) ); /* ensure at least one init */ + + /* catch up testNb */ + for (testNb=1; testNb < startTest; testNb++) + FUZ_rand(&coreSeed); + + /* test loop */ + for ( ; (testNb <= nbTests) || (UTIL_clockSpanMicro(startClock) < g_clockTime) ; testNb++ ) { + U32 lseed; + int opaqueAPI; + const BYTE* srcBuffer; + size_t totalTestSize, totalGenSize, cSize; + XXH64_state_t xxhState; + U64 crcOrig; + U32 resetAllowed = 1; + size_t maxTestSize; + ZSTD_parameters savedParams; + + /* init */ + if (nbTests >= testNb) { DISPLAYUPDATE(2, "\r%6u/%6u ", testNb, nbTests); } + else { DISPLAYUPDATE(2, "\r%6u ", testNb); } + FUZ_rand(&coreSeed); + lseed = coreSeed ^ prime32; + DISPLAYLEVEL(5, " *** Test %u *** \n", testNb); + opaqueAPI = FUZ_rand(&lseed) & 1; + + /* states full reset (deliberately not synchronized) */ + /* some issues can only happen when reusing states */ + if ((FUZ_rand(&lseed) & 0xFF) == 131) { + DISPLAYLEVEL(5, "Creating new context \n"); + ZSTD_freeCCtx(zc); + zc = ZSTD_createCCtx(); + CHECK(zc == NULL, "ZSTD_createCCtx allocation error"); + resetAllowed = 0; + } + if ((FUZ_rand(&lseed) & 0xFF) == 132) { + ZSTD_freeDStream(zd); + zd = ZSTD_createDStream(); + CHECK(zd == NULL, "ZSTD_createDStream allocation error"); + ZSTD_initDStream_usingDict(zd, NULL, 0); /* ensure at least one init */ + } + + /* srcBuffer selection [0-4] */ + { U32 buffNb = FUZ_rand(&lseed) & 0x7F; + if (buffNb & 7) buffNb=2; /* most common : compressible (P) */ + else { + buffNb >>= 3; + if (buffNb & 7) { + const U32 tnb[2] = { 1, 3 }; /* barely/highly compressible */ + buffNb = tnb[buffNb >> 3]; + } else { + const U32 tnb[2] = { 0, 4 }; /* not compressible / sparse */ + buffNb = tnb[buffNb >> 3]; + } } + srcBuffer = cNoiseBuffer[buffNb]; + } + + /* compression init */ + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, NULL, 0) ); /* cancel previous dict /*/ + if ((FUZ_rand(&lseed)&1) /* at beginning, to keep same nb of rand */ + && oldTestLog /* at least one test happened */ + && resetAllowed) { + /* just set a compression level */ + maxTestSize = FUZ_randomLength(&lseed, oldTestLog+2); + if (maxTestSize >= srcBufferSize) maxTestSize = srcBufferSize-1; + { int const compressionLevel = (FUZ_rand(&lseed) % 5) + 1; + DISPLAYLEVEL(5, "t%u : compression level : %i \n", testNb, compressionLevel); + CHECK_Z (setCCtxParameter(zc, cctxParams, ZSTD_c_compressionLevel, compressionLevel, opaqueAPI) ); + } + } else { + U32 const testLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const dictLog = FUZ_rand(&lseed) % maxSrcLog; + U32 const cLevelCandidate = (FUZ_rand(&lseed) % + (ZSTD_maxCLevel() - + (MAX(testLog, dictLog) / 2))) + + 1; + int const cLevel = MIN(cLevelCandidate, cLevelMax); + DISPLAYLEVEL(5, "t%i: base cLevel : %u \n", testNb, cLevel); + maxTestSize = FUZ_rLogLength(&lseed, testLog); + DISPLAYLEVEL(5, "t%i: maxTestSize : %u \n", testNb, (unsigned)maxTestSize); + oldTestLog = testLog; + /* random dictionary selection */ + dictSize = ((FUZ_rand(&lseed)&63)==1) ? FUZ_rLogLength(&lseed, dictLog) : 0; + { size_t const dictStart = FUZ_rand(&lseed) % (srcBufferSize - dictSize); + dict = srcBuffer + dictStart; + if (!dictSize) dict=NULL; + } + { U64 const pledgedSrcSize = (FUZ_rand(&lseed) & 3) ? ZSTD_CONTENTSIZE_UNKNOWN : maxTestSize; + ZSTD_compressionParameters cParams = ZSTD_getCParams(cLevel, pledgedSrcSize, dictSize); + const U32 windowLogMax = bigTests ? 24 : 20; + const U32 searchLogMax = bigTests ? 15 : 13; + if (dictSize) + DISPLAYLEVEL(5, "t%u: with dictionary of size : %zu \n", testNb, dictSize); + + /* mess with compression parameters */ + cParams.windowLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.windowLog = MIN(windowLogMax, cParams.windowLog); + cParams.hashLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.chainLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.searchLog += (FUZ_rand(&lseed) & 3) - 1; + cParams.searchLog = MIN(searchLogMax, cParams.searchLog); + cParams.minMatch += (FUZ_rand(&lseed) & 3) - 1; + cParams.targetLength = (U32)((cParams.targetLength + 1 ) * (0.5 + ((double)(FUZ_rand(&lseed) & 127) / 128))); + cParams = ZSTD_adjustCParams(cParams, pledgedSrcSize, dictSize); + + if (FUZ_rand(&lseed) & 1) { + DISPLAYLEVEL(5, "t%u: windowLog : %u \n", testNb, cParams.windowLog); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_windowLog, cParams.windowLog, opaqueAPI) ); + assert(cParams.windowLog >= ZSTD_WINDOWLOG_MIN); /* guaranteed by ZSTD_adjustCParams() */ + windowLogMalus = (cParams.windowLog - ZSTD_WINDOWLOG_MIN) / 5; + } + if (FUZ_rand(&lseed) & 1) { + DISPLAYLEVEL(5, "t%u: hashLog : %u \n", testNb, cParams.hashLog); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_hashLog, cParams.hashLog, opaqueAPI) ); + } + if (FUZ_rand(&lseed) & 1) { + DISPLAYLEVEL(5, "t%u: chainLog : %u \n", testNb, cParams.chainLog); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_chainLog, cParams.chainLog, opaqueAPI) ); + } + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_searchLog, cParams.searchLog, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_minMatch, cParams.minMatch, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_targetLength, cParams.targetLength, opaqueAPI) ); + + /* mess with long distance matching parameters */ + if (bigTests) { + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_enableLongDistanceMatching, FUZ_rand(&lseed) & 63, opaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashLog, FUZ_randomClampedLength(&lseed, ZSTD_HASHLOG_MIN, 23), opaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmMinMatch, FUZ_randomClampedLength(&lseed, ZSTD_LDM_MINMATCH_MIN, ZSTD_LDM_MINMATCH_MAX), opaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmBucketSizeLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_BUCKETSIZELOG_MIN, ZSTD_LDM_BUCKETSIZELOG_MAX), opaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_ldmHashRateLog, FUZ_randomClampedLength(&lseed, ZSTD_LDM_HASHRATELOG_MIN, ZSTD_LDM_HASHRATELOG_MAX), opaqueAPI) ); + if (FUZ_rand(&lseed) & 3) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_srcSizeHint, FUZ_randomClampedLength(&lseed, ZSTD_SRCSIZEHINT_MIN, ZSTD_SRCSIZEHINT_MAX), opaqueAPI) ); + } + + /* mess with frame parameters */ + if (FUZ_rand(&lseed) & 1) { + int const checksumFlag = FUZ_rand(&lseed) & 1; + DISPLAYLEVEL(5, "t%u: frame checksum : %u \n", testNb, checksumFlag); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_checksumFlag, checksumFlag, opaqueAPI) ); + } + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_dictIDFlag, FUZ_rand(&lseed) & 1, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_contentSizeFlag, FUZ_rand(&lseed) & 1, opaqueAPI) ); + if (FUZ_rand(&lseed) & 1) { + DISPLAYLEVEL(5, "t%u: pledgedSrcSize : %u \n", testNb, (unsigned)pledgedSrcSize); + CHECK_Z( ZSTD_CCtx_setPledgedSrcSize(zc, pledgedSrcSize) ); + } + + /* multi-threading parameters. Only adjust occasionally for small tests. */ + if (bigTests || (FUZ_rand(&lseed) & 0xF) == 0xF) { + U32 const nbThreadsCandidate = (FUZ_rand(&lseed) & 4) + 1; + U32 const nbThreadsAdjusted = (windowLogMalus < nbThreadsCandidate) ? nbThreadsCandidate - windowLogMalus : 1; + int const nbThreads = MIN(nbThreadsAdjusted, nbThreadsMax); + DISPLAYLEVEL(5, "t%i: nbThreads : %u \n", testNb, nbThreads); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_nbWorkers, nbThreads, opaqueAPI) ); + if (nbThreads > 1) { + U32 const jobLog = FUZ_rand(&lseed) % (testLog+1); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_overlapLog, FUZ_rand(&lseed) % 10, opaqueAPI) ); + CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_jobSize, (U32)FUZ_rLogLength(&lseed, jobLog), opaqueAPI) ); + } + } + /* Enable rsyncable mode 1 in 4 times. */ + setCCtxParameter(zc, cctxParams, ZSTD_c_rsyncable, (FUZ_rand(&lseed) % 4 == 0), opaqueAPI); + + if (FUZ_rand(&lseed) & 1) CHECK_Z( setCCtxParameter(zc, cctxParams, ZSTD_c_forceMaxWindow, FUZ_rand(&lseed) & 1, opaqueAPI) ); + + /* Apply parameters */ + if (opaqueAPI) { + DISPLAYLEVEL(5, "t%u: applying CCtxParams \n", testNb); + CHECK_Z (ZSTD_CCtx_setParametersUsingCCtxParams(zc, cctxParams) ); + } + + if (FUZ_rand(&lseed) & 1) { + if (FUZ_rand(&lseed) & 1) { + CHECK_Z( ZSTD_CCtx_loadDictionary(zc, dict, dictSize) ); + } else { + CHECK_Z( ZSTD_CCtx_loadDictionary_byReference(zc, dict, dictSize) ); + } + } else { + CHECK_Z( ZSTD_CCtx_refPrefix(zc, dict, dictSize) ); + } + } } + + CHECK_Z(getCCtxParams(zc, &savedParams)); + + /* multi-segments compression test */ + XXH64_reset(&xxhState, 0); + { ZSTD_outBuffer outBuff = { cBuffer, cBufferSize, 0 } ; + for (cSize=0, totalTestSize=0 ; (totalTestSize < maxTestSize) ; ) { + /* compress random chunks into randomly sized dst buffers */ + size_t const randomSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const srcSize = MIN(maxTestSize-totalTestSize, randomSrcSize); + size_t const srcStart = FUZ_rand(&lseed) % (srcBufferSize - srcSize); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); + size_t const dstBuffSize = MIN(cBufferSize - cSize, randomDstSize); + ZSTD_EndDirective const flush = (FUZ_rand(&lseed) & 15) ? ZSTD_e_continue : ZSTD_e_flush; + ZSTD_inBuffer inBuff = { srcBuffer+srcStart, srcSize, 0 }; + outBuff.size = outBuff.pos + dstBuffSize; + + CHECK_Z( ZSTD_compressStream2(zc, &outBuff, &inBuff, flush) ); + DISPLAYLEVEL(6, "t%u: compress consumed %u bytes (total : %u) ; flush: %u (total : %u) \n", + testNb, (unsigned)inBuff.pos, (unsigned)(totalTestSize + inBuff.pos), (unsigned)flush, (unsigned)outBuff.pos); + + XXH64_update(&xxhState, srcBuffer+srcStart, inBuff.pos); + memcpy(copyBuffer+totalTestSize, srcBuffer+srcStart, inBuff.pos); + totalTestSize += inBuff.pos; + } + + /* final frame epilogue */ + { size_t remainingToFlush = 1; + while (remainingToFlush) { + ZSTD_inBuffer inBuff = { NULL, 0, 0 }; + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog+1); + size_t const adjustedDstSize = MIN(cBufferSize - cSize, randomDstSize); + outBuff.size = outBuff.pos + adjustedDstSize; + DISPLAYLEVEL(6, "t%u: End-flush into dst buffer of size %u \n", testNb, (unsigned)adjustedDstSize); + remainingToFlush = ZSTD_compressStream2(zc, &outBuff, &inBuff, ZSTD_e_end); + DISPLAYLEVEL(6, "t%u: Total flushed so far : %u bytes \n", testNb, (unsigned)outBuff.pos); + CHECK( ZSTD_isError(remainingToFlush), + "ZSTD_compressStream2 w/ ZSTD_e_end error : %s", + ZSTD_getErrorName(remainingToFlush) ); + } } + crcOrig = XXH64_digest(&xxhState); + cSize = outBuff.pos; + DISPLAYLEVEL(5, "Frame completed : %zu bytes \n", cSize); + } + + CHECK(badParameters(zc, savedParams), "CCtx params are wrong"); + + /* multi - fragments decompression test */ + if (!dictSize /* don't reset if dictionary : could be different */ && (FUZ_rand(&lseed) & 1)) { + DISPLAYLEVEL(5, "resetting DCtx (dict:%p) \n", dict); + CHECK_Z( ZSTD_resetDStream(zd) ); + } else { + if (dictSize) + DISPLAYLEVEL(5, "using dictionary of size %zu \n", dictSize); + CHECK_Z( ZSTD_initDStream_usingDict(zd, dict, dictSize) ); + } + { size_t decompressionResult = 1; + ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + for (totalGenSize = 0 ; decompressionResult ; ) { + size_t const readCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const dstBuffSize = MIN(dstBufferSize - totalGenSize, randomDstSize); + inBuff.size = inBuff.pos + readCSrcSize; + outBuff.size = outBuff.pos + dstBuffSize; + DISPLAYLEVEL(6, "decompression presented %u new bytes (pos:%u/%u)\n", + (unsigned)readCSrcSize, (unsigned)inBuff.pos, (unsigned)cSize); + decompressionResult = ZSTD_decompressStream(zd, &outBuff, &inBuff); + DISPLAYLEVEL(6, "so far: consumed = %u, produced = %u \n", + (unsigned)inBuff.pos, (unsigned)outBuff.pos); + if (ZSTD_isError(decompressionResult)) { + DISPLAY("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(decompressionResult)); + findDiff(copyBuffer, dstBuffer, totalTestSize); + } + CHECK (ZSTD_isError(decompressionResult), "decompression error : %s", ZSTD_getErrorName(decompressionResult)); + CHECK (inBuff.pos > cSize, "ZSTD_decompressStream consumes too much input : %u > %u ", (unsigned)inBuff.pos, (unsigned)cSize); + } + CHECK (inBuff.pos != cSize, "compressed data should be fully read (%u != %u)", (unsigned)inBuff.pos, (unsigned)cSize); + CHECK (outBuff.pos != totalTestSize, "decompressed data : wrong size (%u != %u)", (unsigned)outBuff.pos, (unsigned)totalTestSize); + { U64 const crcDest = XXH64(dstBuffer, totalTestSize, 0); + if (crcDest!=crcOrig) findDiff(copyBuffer, dstBuffer, totalTestSize); + CHECK (crcDest!=crcOrig, "decompressed data corrupted"); + } } + + /*===== noisy/erroneous src decompression test =====*/ + + /* add some noise */ + { U32 const nbNoiseChunks = (FUZ_rand(&lseed) & 7) + 2; + U32 nn; for (nn=0; nn<nbNoiseChunks; nn++) { + size_t const randomNoiseSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const noiseSize = MIN((cSize/3) , randomNoiseSize); + size_t const noiseStart = FUZ_rand(&lseed) % (srcBufferSize - noiseSize); + size_t const cStart = FUZ_rand(&lseed) % (cSize - noiseSize); + memcpy(cBuffer+cStart, srcBuffer+noiseStart, noiseSize); + } } + + /* try decompression on noisy data */ + CHECK_Z( ZSTD_initDStream(zd_noise) ); /* note : no dictionary */ + { ZSTD_inBuffer inBuff = { cBuffer, cSize, 0 }; + ZSTD_outBuffer outBuff= { dstBuffer, dstBufferSize, 0 }; + while (outBuff.pos < dstBufferSize) { + size_t const randomCSrcSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const randomDstSize = FUZ_randomLength(&lseed, maxSampleLog); + size_t const adjustedDstSize = MIN(dstBufferSize - outBuff.pos, randomDstSize); + size_t const adjustedCSrcSize = MIN(cSize - inBuff.pos, randomCSrcSize); + outBuff.size = outBuff.pos + adjustedDstSize; + inBuff.size = inBuff.pos + adjustedCSrcSize; + { size_t const decompressError = ZSTD_decompressStream(zd, &outBuff, &inBuff); + if (ZSTD_isError(decompressError)) break; /* error correctly detected */ + /* Good so far, but no more progress possible */ + if (outBuff.pos < outBuff.size && inBuff.pos == cSize) break; + } } } } + DISPLAY("\r%u fuzzer tests completed \n", testNb-1); + +_cleanup: + ZSTD_freeCCtx(zc); + ZSTD_freeDStream(zd); + ZSTD_freeDStream(zd_noise); + ZSTD_freeCCtxParams(cctxParams); + free(cNoiseBuffer[0]); + free(cNoiseBuffer[1]); + free(cNoiseBuffer[2]); + free(cNoiseBuffer[3]); + free(cNoiseBuffer[4]); + free(copyBuffer); + free(cBuffer); + free(dstBuffer); + return result; + +_output_error: + result = 1; + goto _cleanup; +} + +/*-******************************************************* +* Command line +*********************************************************/ +static int FUZ_usage(const char* programName) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args]\n", programName); + DISPLAY( "\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -i# : Number of tests (default:%u)\n", nbTestsDefault); + DISPLAY( " -T# : Max duration to run for. Overrides number of tests. (e.g. -T1m or -T60s for one minute)\n"); + DISPLAY( " -s# : Select seed (default:prompt user)\n"); + DISPLAY( " -t# : Select starting test number (default:0)\n"); + DISPLAY( " -P# : Select compressibility in %% (default:%i%%)\n", FUZ_COMPRESSIBILITY_DEFAULT); + DISPLAY( " -v : verbose\n"); + DISPLAY( " -p : pause at the end\n"); + DISPLAY( " -h : display help and exit\n"); + return 0; +} + +typedef enum { simple_api, mt_api, advanced_api } e_api; + +int main(int argc, const char** argv) +{ + U32 seed = 0; + int seedset = 0; + int nbTests = nbTestsDefault; + int testNb = 0; + int proba = FUZ_COMPRESSIBILITY_DEFAULT; + int result = 0; + int mainPause = 0; + int bigTests = (sizeof(size_t) == 8); + e_api selected_api = simple_api; + const char* const programName = argv[0]; + int argNb; + + /* Check command line */ + for(argNb=1; argNb<argc; argNb++) { + const char* argument = argv[argNb]; + assert(argument != NULL); + + /* Parsing commands. Aggregated commands are allowed */ + if (argument[0]=='-') { + + if (!strcmp(argument, "--mt")) { selected_api=mt_api; testNb += !testNb; continue; } + if (!strcmp(argument, "--newapi")) { selected_api=advanced_api; testNb += !testNb; continue; } + if (!strcmp(argument, "--no-big-tests")) { bigTests=0; continue; } + + argument++; + while (*argument!=0) { + switch(*argument) + { + case 'h': + return FUZ_usage(programName); + + case 'v': + argument++; + g_displayLevel++; + break; + + case 'q': + argument++; + g_displayLevel--; + break; + + case 'p': /* pause at the end */ + argument++; + mainPause = 1; + break; + + case 'i': /* limit tests by nb of iterations (default) */ + argument++; + nbTests=0; g_clockTime=0; + while ((*argument>='0') && (*argument<='9')) { + nbTests *= 10; + nbTests += *argument - '0'; + argument++; + } + break; + + case 'T': /* limit tests by time */ + argument++; + nbTests=0; g_clockTime=0; + while ((*argument>='0') && (*argument<='9')) { + g_clockTime *= 10; + g_clockTime += *argument - '0'; + argument++; + } + if (*argument=='m') { /* -T1m == -T60 */ + g_clockTime *=60, argument++; + if (*argument=='n') argument++; /* -T1mn == -T60 */ + } else if (*argument=='s') argument++; /* -T10s == -T10 */ + g_clockTime *= SEC_TO_MICRO; + break; + + case 's': /* manually select seed */ + argument++; + seedset=1; + seed=0; + while ((*argument>='0') && (*argument<='9')) { + seed *= 10; + seed += *argument - '0'; + argument++; + } + break; + + case 't': /* select starting test number */ + argument++; + testNb=0; + while ((*argument>='0') && (*argument<='9')) { + testNb *= 10; + testNb += *argument - '0'; + argument++; + } + break; + + case 'P': /* compressibility % */ + argument++; + proba=0; + while ((*argument>='0') && (*argument<='9')) { + proba *= 10; + proba += *argument - '0'; + argument++; + } + if (proba<0) proba=0; + if (proba>100) proba=100; + break; + + default: + return FUZ_usage(programName); + } + } } } /* for(argNb=1; argNb<argc; argNb++) */ + + /* Get Seed */ + DISPLAY("Starting zstream tester (%i-bits, %s)\n", (int)(sizeof(size_t)*8), ZSTD_VERSION_STRING); + + if (!seedset) { + time_t const t = time(NULL); + U32 const h = XXH32(&t, sizeof(t), 1); + seed = h % 10000; + } + + DISPLAY("Seed = %u\n", (unsigned)seed); + if (proba!=FUZ_COMPRESSIBILITY_DEFAULT) DISPLAY("Compressibility : %i%%\n", proba); + + if (nbTests<=0) nbTests=1; + + if (testNb==0) { + result = basicUnitTests(0, ((double)proba) / 100); /* constant seed for predictability */ + } + + if (!result) { + switch(selected_api) + { + case simple_api : + result = fuzzerTests(seed, nbTests, testNb, ((double)proba) / 100, bigTests); + break; + case mt_api : + result = fuzzerTests_MT(seed, nbTests, testNb, ((double)proba) / 100, bigTests); + break; + case advanced_api : + result = fuzzerTests_newAPI(seed, nbTests, testNb, ((double)proba) / 100, bigTests); + break; + default : + assert(0); /* impossible */ + } + } + + if (mainPause) { + int unused; + DISPLAY("Press Enter \n"); + unused = getchar(); + (void)unused; + } + return result; +} |