summaryrefslogtreecommitdiffstats
path: root/src/zstd/tests/test-zstd-versions.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rwxr-xr-xsrc/zstd/tests/test-zstd-versions.py277
1 files changed, 277 insertions, 0 deletions
diff --git a/src/zstd/tests/test-zstd-versions.py b/src/zstd/tests/test-zstd-versions.py
new file mode 100755
index 000000000..fa2171752
--- /dev/null
+++ b/src/zstd/tests/test-zstd-versions.py
@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+"""Test zstd interoperability between versions"""
+
+# ################################################################
+# Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
+# ################################################################
+
+import filecmp
+import glob
+import hashlib
+import os
+import shutil
+import sys
+import subprocess
+from subprocess import Popen, PIPE
+
+repo_url = 'https://github.com/facebook/zstd.git'
+tmp_dir_name = 'tests/versionsTest'
+make_cmd = 'make'
+git_cmd = 'git'
+test_dat_src = 'README.md'
+test_dat = 'test_dat'
+head = 'vdevel'
+dict_source = 'dict_source'
+dict_files = './zstd/programs/*.c ./zstd/lib/common/*.c ./zstd/lib/compress/*.c ./zstd/lib/decompress/*.c ./zstd/lib/dictBuilder/*.c ./zstd/lib/legacy/*.c '
+dict_files += './zstd/programs/*.h ./zstd/lib/common/*.h ./zstd/lib/compress/*.h ./zstd/lib/dictBuilder/*.h ./zstd/lib/legacy/*.h'
+
+
+def execute(command, print_output=False, print_error=True, param_shell=False):
+ popen = Popen(command, stdout=PIPE, stderr=PIPE, shell=param_shell)
+ stdout_lines, stderr_lines = popen.communicate()
+ stderr_lines = stderr_lines.decode("utf-8")
+ stdout_lines = stdout_lines.decode("utf-8")
+ if print_output:
+ print(stdout_lines)
+ print(stderr_lines)
+ if popen.returncode is not None and popen.returncode != 0:
+ if not print_output and print_error:
+ print(stderr_lines)
+ return popen.returncode
+
+
+def proc(cmd_args, pipe=True, dummy=False):
+ if dummy:
+ return
+ if pipe:
+ subproc = Popen(cmd_args, stdout=PIPE, stderr=PIPE)
+ else:
+ subproc = Popen(cmd_args)
+ return subproc.communicate()
+
+
+def make(args, pipe=True):
+ return proc([make_cmd] + args, pipe)
+
+
+def git(args, pipe=True):
+ return proc([git_cmd] + args, pipe)
+
+
+def get_git_tags():
+ stdout, stderr = git(['tag', '-l', 'v[0-9].[0-9].[0-9]'])
+ tags = stdout.decode('utf-8').split()
+ return tags
+
+
+def create_dict(tag, dict_source_path):
+ dict_name = 'dict.' + tag
+ if not os.path.isfile(dict_name):
+ cFiles = glob.glob(dict_source_path + "/*.c")
+ hFiles = glob.glob(dict_source_path + "/*.h")
+ if tag == 'v0.5.0':
+ result = execute('./dictBuilder.' + tag + ' ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+ else:
+ result = execute('./zstd.' + tag + ' -f --train ' + ' '.join(cFiles) + ' ' + ' '.join(hFiles) + ' -o ' + dict_name, print_output=False, param_shell=True)
+ if result == 0:
+ print(dict_name + ' created')
+ else:
+ print('ERROR: creating of ' + dict_name + ' failed')
+ else:
+ print(dict_name + ' already exists')
+
+
+def dict_compress_sample(tag, sample):
+ dict_name = 'dict.' + tag
+ DEVNULL = open(os.devnull, 'wb')
+ if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_01_64_' + tag + '_dictio.zst')
+ if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-5f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_05_64_' + tag + '_dictio.zst')
+ if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-9f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_09_64_' + tag + '_dictio.zst')
+ if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-15f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_15_64_' + tag + '_dictio.zst')
+ if subprocess.call(['./zstd.' + tag, '-D', dict_name, '-18f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_18_64_' + tag + '_dictio.zst')
+ # zstdFiles = glob.glob("*.zst*")
+ # print(zstdFiles)
+ print(tag + " : dict compression completed")
+
+
+def compress_sample(tag, sample):
+ DEVNULL = open(os.devnull, 'wb')
+ if subprocess.call(['./zstd.' + tag, '-f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_01_64_' + tag + '_nodict.zst')
+ if subprocess.call(['./zstd.' + tag, '-5f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_05_64_' + tag + '_nodict.zst')
+ if subprocess.call(['./zstd.' + tag, '-9f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_09_64_' + tag + '_nodict.zst')
+ if subprocess.call(['./zstd.' + tag, '-15f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_15_64_' + tag + '_nodict.zst')
+ if subprocess.call(['./zstd.' + tag, '-18f', sample], stderr=DEVNULL) == 0:
+ os.rename(sample + '.zst', sample + '_18_64_' + tag + '_nodict.zst')
+ # zstdFiles = glob.glob("*.zst*")
+ # print(zstdFiles)
+ print(tag + " : compression completed")
+
+
+# http://stackoverflow.com/a/19711609/2132223
+def sha1_of_file(filepath):
+ with open(filepath, 'rb') as f:
+ return hashlib.sha1(f.read()).hexdigest()
+
+
+def remove_duplicates():
+ list_of_zst = sorted(glob.glob('*.zst'))
+ for i, ref_zst in enumerate(list_of_zst):
+ if not os.path.isfile(ref_zst):
+ continue
+ for j in range(i + 1, len(list_of_zst)):
+ compared_zst = list_of_zst[j]
+ if not os.path.isfile(compared_zst):
+ continue
+ if filecmp.cmp(ref_zst, compared_zst):
+ os.remove(compared_zst)
+ print('duplicated : {} == {}'.format(ref_zst, compared_zst))
+
+
+def decompress_zst(tag):
+ dec_error = 0
+ list_zst = sorted(glob.glob('*_nodict.zst'))
+ for file_zst in list_zst:
+ print(file_zst, end=' ')
+ print(tag, end=' ')
+ file_dec = file_zst + '_d64_' + tag + '.dec'
+ if tag <= 'v0.5.0':
+ params = ['./zstd.' + tag, '-df', file_zst, file_dec]
+ else:
+ params = ['./zstd.' + tag, '-df', file_zst, '-o', file_dec]
+ if execute(params) == 0:
+ if not filecmp.cmp(file_dec, test_dat):
+ print('ERR !! ')
+ dec_error = 1
+ else:
+ print('OK ')
+ else:
+ print('command does not work')
+ dec_error = 1
+ return dec_error
+
+
+def decompress_dict(tag):
+ dec_error = 0
+ list_zst = sorted(glob.glob('*_dictio.zst'))
+ for file_zst in list_zst:
+ dict_tag = file_zst[0:len(file_zst)-11] # remove "_dictio.zst"
+ if head in dict_tag: # find vdevel
+ dict_tag = head
+ else:
+ dict_tag = dict_tag[dict_tag.rfind('v'):]
+ if tag == 'v0.6.0' and dict_tag < 'v0.6.0':
+ continue
+ dict_name = 'dict.' + dict_tag
+ print(file_zst + ' ' + tag + ' dict=' + dict_tag, end=' ')
+ file_dec = file_zst + '_d64_' + tag + '.dec'
+ if tag <= 'v0.5.0':
+ params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, file_dec]
+ else:
+ params = ['./zstd.' + tag, '-D', dict_name, '-df', file_zst, '-o', file_dec]
+ if execute(params) == 0:
+ if not filecmp.cmp(file_dec, test_dat):
+ print('ERR !! ')
+ dec_error = 1
+ else:
+ print('OK ')
+ else:
+ print('command does not work')
+ dec_error = 1
+ return dec_error
+
+
+if __name__ == '__main__':
+ error_code = 0
+ base_dir = os.getcwd() + '/..' # /path/to/zstd
+ tmp_dir = base_dir + '/' + tmp_dir_name # /path/to/zstd/tests/versionsTest
+ clone_dir = tmp_dir + '/' + 'zstd' # /path/to/zstd/tests/versionsTest/zstd
+ dict_source_path = tmp_dir + '/' + dict_source # /path/to/zstd/tests/versionsTest/dict_source
+ programs_dir = base_dir + '/programs' # /path/to/zstd/programs
+ os.makedirs(tmp_dir, exist_ok=True)
+
+ # since Travis clones limited depth, we should clone full repository
+ if not os.path.isdir(clone_dir):
+ git(['clone', repo_url, clone_dir])
+
+ shutil.copy2(base_dir + '/' + test_dat_src, tmp_dir + '/' + test_dat)
+
+ # Retrieve all release tags
+ print('Retrieve all release tags :')
+ os.chdir(clone_dir)
+ alltags = get_git_tags() + [head]
+ tags = [t for t in alltags if t >= 'v0.5.0']
+ print(tags)
+
+ # Build all release zstd
+ for tag in tags:
+ os.chdir(base_dir)
+ dst_zstd = '{}/zstd.{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/zstd.<TAG>
+ if not os.path.isfile(dst_zstd) or tag == head:
+ if tag != head:
+ r_dir = '{}/{}'.format(tmp_dir, tag) # /path/to/zstd/tests/versionsTest/<TAG>
+ os.makedirs(r_dir, exist_ok=True)
+ os.chdir(clone_dir)
+ git(['--work-tree=' + r_dir, 'checkout', tag, '--', '.'], False)
+ if tag == 'v0.5.0':
+ os.chdir(r_dir + '/dictBuilder') # /path/to/zstd/tests/versionsTest/v0.5.0/dictBuilder
+ make(['clean', 'dictBuilder'], False)
+ shutil.copy2('dictBuilder', '{}/dictBuilder.{}'.format(tmp_dir, tag))
+ os.chdir(r_dir + '/programs') # /path/to/zstd/tests/versionsTest/<TAG>/programs
+ make(['clean', 'zstd'], False)
+ else:
+ os.chdir(programs_dir)
+ make(['zstd'], False)
+ shutil.copy2('zstd', dst_zstd)
+
+ # remove any remaining *.zst and *.dec from previous test
+ os.chdir(tmp_dir)
+ for compressed in glob.glob("*.zst"):
+ os.remove(compressed)
+ for dec in glob.glob("*.dec"):
+ os.remove(dec)
+
+ # copy *.c and *.h to a temporary directory ("dict_source")
+ if not os.path.isdir(dict_source_path):
+ os.mkdir(dict_source_path)
+ print('cp ' + dict_files + ' ' + dict_source_path)
+ execute('cp ' + dict_files + ' ' + dict_source_path, param_shell=True)
+
+ print('Compress test.dat by all released zstd')
+
+ error_code = 0
+ for tag in tags:
+ print(tag)
+ if tag >= 'v0.5.0':
+ create_dict(tag, dict_source_path)
+ dict_compress_sample(tag, test_dat)
+ remove_duplicates()
+ error_code += decompress_dict(tag)
+ compress_sample(tag, test_dat)
+ remove_duplicates()
+ error_code += decompress_zst(tag)
+
+ print('')
+ print('Enumerate different compressed files')
+ zstds = sorted(glob.glob('*.zst'))
+ for zstd in zstds:
+ print(zstd + ' : ' + repr(os.path.getsize(zstd)) + ', ' + sha1_of_file(zstd))
+
+ if error_code != 0:
+ print('====== ERROR !!! =======')
+
+ sys.exit(error_code)