diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 18:45:59 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 18:45:59 +0000 |
commit | 19fcec84d8d7d21e796c7624e521b60d28ee21ed (patch) | |
tree | 42d26aa27d1e3f7c0b8bd3fd14e7d7082f5008dc /src/zstd/contrib/pzstd/test | |
parent | Initial commit. (diff) | |
download | ceph-upstream/16.2.11+ds.tar.xz ceph-upstream/16.2.11+ds.zip |
Adding upstream version 16.2.11+ds.upstream/16.2.11+dsupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r-- | src/zstd/contrib/pzstd/test/BUCK | 37 | ||||
-rw-r--r-- | src/zstd/contrib/pzstd/test/OptionsTest.cpp | 536 | ||||
-rw-r--r-- | src/zstd/contrib/pzstd/test/PzstdTest.cpp | 149 | ||||
-rw-r--r-- | src/zstd/contrib/pzstd/test/RoundTrip.h | 86 | ||||
-rw-r--r-- | src/zstd/contrib/pzstd/test/RoundTripTest.cpp | 86 |
5 files changed, 894 insertions, 0 deletions
diff --git a/src/zstd/contrib/pzstd/test/BUCK b/src/zstd/contrib/pzstd/test/BUCK new file mode 100644 index 000000000..6d3fdd3c2 --- /dev/null +++ b/src/zstd/contrib/pzstd/test/BUCK @@ -0,0 +1,37 @@ +cxx_test( + name='options_test', + srcs=['OptionsTest.cpp'], + deps=['//contrib/pzstd:options'], +) + +cxx_test( + name='pzstd_test', + srcs=['PzstdTest.cpp'], + deps=[ + ':round_trip', + '//contrib/pzstd:libpzstd', + '//contrib/pzstd/utils:scope_guard', + '//programs:datagen', + ], +) + +cxx_binary( + name='round_trip_test', + srcs=['RoundTripTest.cpp'], + deps=[ + ':round_trip', + '//contrib/pzstd/utils:scope_guard', + '//programs:datagen', + ] +) + +cxx_library( + name='round_trip', + header_namespace='test', + exported_headers=['RoundTrip.h'], + deps=[ + '//contrib/pzstd:libpzstd', + '//contrib/pzstd:options', + '//contrib/pzstd/utils:scope_guard', + ] +) diff --git a/src/zstd/contrib/pzstd/test/OptionsTest.cpp b/src/zstd/contrib/pzstd/test/OptionsTest.cpp new file mode 100644 index 000000000..e60114825 --- /dev/null +++ b/src/zstd/contrib/pzstd/test/OptionsTest.cpp @@ -0,0 +1,536 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ +#include "Options.h" + +#include <array> +#include <gtest/gtest.h> + +using namespace pzstd; + +namespace pzstd { +bool operator==(const Options &lhs, const Options &rhs) { + return lhs.numThreads == rhs.numThreads && + lhs.maxWindowLog == rhs.maxWindowLog && + lhs.compressionLevel == rhs.compressionLevel && + lhs.decompress == rhs.decompress && lhs.inputFiles == rhs.inputFiles && + lhs.outputFile == rhs.outputFile && lhs.overwrite == rhs.overwrite && + lhs.keepSource == rhs.keepSource && lhs.writeMode == rhs.writeMode && + lhs.checksum == rhs.checksum && lhs.verbosity == rhs.verbosity; +} + +std::ostream &operator<<(std::ostream &out, const Options &opt) { + out << "{"; + { + out << "\n\t" + << "numThreads: " << opt.numThreads; + out << ",\n\t" + << "maxWindowLog: " << opt.maxWindowLog; + out << ",\n\t" + << "compressionLevel: " << opt.compressionLevel; + out << ",\n\t" + << "decompress: " << opt.decompress; + out << ",\n\t" + << "inputFiles: {"; + { + bool first = true; + for (const auto &file : opt.inputFiles) { + if (!first) { + out << ","; + } + first = false; + out << "\n\t\t" << file; + } + } + out << "\n\t}"; + out << ",\n\t" + << "outputFile: " << opt.outputFile; + out << ",\n\t" + << "overwrite: " << opt.overwrite; + out << ",\n\t" + << "keepSource: " << opt.keepSource; + out << ",\n\t" + << "writeMode: " << static_cast<int>(opt.writeMode); + out << ",\n\t" + << "checksum: " << opt.checksum; + out << ",\n\t" + << "verbosity: " << opt.verbosity; + } + out << "\n}"; + return out; +} +} + +namespace { +#ifdef _WIN32 +const char nullOutput[] = "nul"; +#else +const char nullOutput[] = "/dev/null"; +#endif + +constexpr auto autoMode = Options::WriteMode::Auto; +} // anonymous namespace + +#define EXPECT_SUCCESS(...) EXPECT_EQ(Options::Status::Success, __VA_ARGS__) +#define EXPECT_FAILURE(...) EXPECT_EQ(Options::Status::Failure, __VA_ARGS__) +#define EXPECT_MESSAGE(...) EXPECT_EQ(Options::Status::Message, __VA_ARGS__) + +template <typename... Args> +std::array<const char *, sizeof...(Args) + 1> makeArray(Args... args) { + return {{nullptr, args...}}; +} + +TEST(Options, ValidInputs) { + { + Options options; + auto args = makeArray("--processes", "5", "-o", "x", "y", "-f"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {5, 23, 3, false, {"y"}, "x", + true, true, autoMode, true, 2}; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("-p", "1", "input", "-19"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 23, 19, false, {"input"}, "", + false, true, autoMode, true, 2}; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = + makeArray("--ultra", "-22", "-p", "1", "-o", "x", "-d", "x.zst", "-f"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 0, 22, true, {"x.zst"}, "x", + true, true, autoMode, true, 2}; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("--processes", "100", "hello.zst", "--decompress", + "--force"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {100, 23, 3, true, {"hello.zst"}, "", true, + true, autoMode, true, 2}; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("x", "-dp", "1", "-c"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 23, 3, true, {"x"}, "-", + false, true, autoMode, true, 2}; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("x", "-dp", "1", "--stdout"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 23, 3, true, {"x"}, "-", + false, true, autoMode, true, 2}; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("-p", "1", "x", "-5", "-fo", "-", "--ultra", "-d"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {1, 0, 5, true, {"x"}, "-", + true, true, autoMode, true, 2}; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("silesia.tar", "-o", "silesia.tar.pzstd", "-p", "2"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {2, + 23, + 3, + false, + {"silesia.tar"}, + "silesia.tar.pzstd", + false, + true, + autoMode, + true, + 2}; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("x", "-p", "1"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-p", "1"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + } +} + +TEST(Options, GetOutputFile) { + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("x.zst", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("x", "y", "-o", nullOutput); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("x.zst", "-do", nullOutput); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(nullOutput, options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("x.zst", "-d"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("x", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("xzst", "-d"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("", options.getOutputFile(options.inputFiles[0])); + } + { + Options options; + auto args = makeArray("xzst", "-doxx"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("xx", options.getOutputFile(options.inputFiles[0])); + } +} + +TEST(Options, MultipleFiles) { + { + Options options; + auto args = makeArray("x", "y", "z"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected; + expected.inputFiles = {"x", "y", "z"}; + expected.verbosity = 1; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("x", "y", "z", "-o", nullOutput); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected; + expected.inputFiles = {"x", "y", "z"}; + expected.outputFile = nullOutput; + expected.verbosity = 1; + EXPECT_EQ(expected, options); + } + { + Options options; + auto args = makeArray("x", "y", "-o-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "y", "-o", "file"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-qqvd12qp4", "-f", "x", "--", "--rm", "-c"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + Options expected = {4, 23, 12, true, {"x", "--rm", "-c"}, + "", true, true, autoMode, true, + 0}; + EXPECT_EQ(expected, options); + } +} + +TEST(Options, NumThreads) { + { + Options options; + auto args = makeArray("x", "-dfo", "-"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-p", "0", "-fo", "-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-f", "-p", "-o", "-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, BadCompressionLevel) { + { + Options options; + auto args = makeArray("x", "-20"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "--ultra", "-23"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "--1"); // negative 1? + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, InvalidOption) { + { + Options options; + auto args = makeArray("x", "-x"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, BadOutputFile) { + { + Options options; + auto args = makeArray("notzst", "-d", "-p", "1"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ("", options.getOutputFile(options.inputFiles.front())); + } +} + +TEST(Options, BadOptionsWithArguments) { + { + Options options; + auto args = makeArray("x", "-pf"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-p", "10f"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-p"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-o"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("x", "-o"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, KeepSource) { + { + Options options; + auto args = makeArray("x", "--rm", "-k"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + } + { + Options options; + auto args = makeArray("x", "--rm", "--keep"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + } + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + } + { + Options options; + auto args = makeArray("x", "--rm"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(false, options.keepSource); + } +} + +TEST(Options, Verbosity) { + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(2, options.verbosity); + } + { + Options options; + auto args = makeArray("--quiet", "-qq", "x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(-1, options.verbosity); + } + { + Options options; + auto args = makeArray("x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(1, options.verbosity); + } + { + Options options; + auto args = makeArray("--", "x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(1, options.verbosity); + } + { + Options options; + auto args = makeArray("-qv", "x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(1, options.verbosity); + } + { + Options options; + auto args = makeArray("-v", "x", "y"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(3, options.verbosity); + } + { + Options options; + auto args = makeArray("-v", "x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(3, options.verbosity); + } +} + +TEST(Options, TestMode) { + { + Options options; + auto args = makeArray("x", "-t"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + EXPECT_EQ(true, options.decompress); + EXPECT_EQ(nullOutput, options.outputFile); + } + { + Options options; + auto args = makeArray("x", "--test", "--rm", "-ohello"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.keepSource); + EXPECT_EQ(true, options.decompress); + EXPECT_EQ(nullOutput, options.outputFile); + } +} + +TEST(Options, Checksum) { + { + Options options; + auto args = makeArray("x.zst", "--no-check", "-Cd"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.checksum); + } + { + Options options; + auto args = makeArray("x"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.checksum); + } + { + Options options; + auto args = makeArray("x", "--no-check", "--check"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(true, options.checksum); + } + { + Options options; + auto args = makeArray("x", "--no-check"); + EXPECT_SUCCESS(options.parse(args.size(), args.data())); + EXPECT_EQ(false, options.checksum); + } +} + +TEST(Options, InputFiles) { + { + Options options; + auto args = makeArray("-cd"); + options.parse(args.size(), args.data()); + EXPECT_EQ(1, options.inputFiles.size()); + EXPECT_EQ("-", options.inputFiles[0]); + EXPECT_EQ("-", options.outputFile); + } + { + Options options; + auto args = makeArray(); + options.parse(args.size(), args.data()); + EXPECT_EQ(1, options.inputFiles.size()); + EXPECT_EQ("-", options.inputFiles[0]); + EXPECT_EQ("-", options.outputFile); + } + { + Options options; + auto args = makeArray("-d"); + options.parse(args.size(), args.data()); + EXPECT_EQ(1, options.inputFiles.size()); + EXPECT_EQ("-", options.inputFiles[0]); + EXPECT_EQ("-", options.outputFile); + } + { + Options options; + auto args = makeArray("x", "-"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, InvalidOptions) { + { + Options options; + auto args = makeArray("-ibasdf"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("- "); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-n15"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-0", "x"); + EXPECT_FAILURE(options.parse(args.size(), args.data())); + } +} + +TEST(Options, Extras) { + { + Options options; + auto args = makeArray("-h"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-H"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("-V"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("--help"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } + { + Options options; + auto args = makeArray("--version"); + EXPECT_MESSAGE(options.parse(args.size(), args.data())); + } +} diff --git a/src/zstd/contrib/pzstd/test/PzstdTest.cpp b/src/zstd/contrib/pzstd/test/PzstdTest.cpp new file mode 100644 index 000000000..5c7d66310 --- /dev/null +++ b/src/zstd/contrib/pzstd/test/PzstdTest.cpp @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ +#include "Pzstd.h" +extern "C" { +#include "datagen.h" +} +#include "test/RoundTrip.h" +#include "utils/ScopeGuard.h" + +#include <cstddef> +#include <cstdio> +#include <gtest/gtest.h> +#include <memory> +#include <random> + +using namespace std; +using namespace pzstd; + +TEST(Pzstd, SmallSizes) { + unsigned seed = std::random_device{}(); + std::fprintf(stderr, "Pzstd.SmallSizes seed: %u\n", seed); + std::mt19937 gen(seed); + + for (unsigned len = 1; len < 256; ++len) { + if (len % 16 == 0) { + std::fprintf(stderr, "%u / 16\n", len / 16); + } + std::string inputFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + { + static uint8_t buf[256]; + RDG_genBuffer(buf, len, 0.5, 0.0, gen()); + auto fd = std::fopen(inputFile.c_str(), "wb"); + auto written = std::fwrite(buf, 1, len, fd); + std::fclose(fd); + ASSERT_EQ(written, len); + } + for (unsigned numThreads = 1; numThreads <= 2; ++numThreads) { + for (unsigned level = 1; level <= 4; level *= 4) { + auto errorGuard = makeScopeGuard([&] { + std::fprintf(stderr, "# threads: %u\n", numThreads); + std::fprintf(stderr, "compression level: %u\n", level); + }); + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.numThreads = numThreads; + options.compressionLevel = level; + options.verbosity = 1; + ASSERT_TRUE(roundTrip(options)); + errorGuard.dismiss(); + } + } + } +} + +TEST(Pzstd, LargeSizes) { + unsigned seed = std::random_device{}(); + std::fprintf(stderr, "Pzstd.LargeSizes seed: %u\n", seed); + std::mt19937 gen(seed); + + for (unsigned len = 1 << 20; len <= (1 << 24); len *= 2) { + std::string inputFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + { + std::unique_ptr<uint8_t[]> buf(new uint8_t[len]); + RDG_genBuffer(buf.get(), len, 0.5, 0.0, gen()); + auto fd = std::fopen(inputFile.c_str(), "wb"); + auto written = std::fwrite(buf.get(), 1, len, fd); + std::fclose(fd); + ASSERT_EQ(written, len); + } + for (unsigned numThreads = 1; numThreads <= 16; numThreads *= 4) { + for (unsigned level = 1; level <= 4; level *= 4) { + auto errorGuard = makeScopeGuard([&] { + std::fprintf(stderr, "# threads: %u\n", numThreads); + std::fprintf(stderr, "compression level: %u\n", level); + }); + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.numThreads = std::min(numThreads, options.numThreads); + options.compressionLevel = level; + options.verbosity = 1; + ASSERT_TRUE(roundTrip(options)); + errorGuard.dismiss(); + } + } + } +} + +TEST(Pzstd, DISABLED_ExtremelyLargeSize) { + unsigned seed = std::random_device{}(); + std::fprintf(stderr, "Pzstd.ExtremelyLargeSize seed: %u\n", seed); + std::mt19937 gen(seed); + + std::string inputFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + + { + // Write 4GB + 64 MB + constexpr size_t kLength = 1 << 26; + std::unique_ptr<uint8_t[]> buf(new uint8_t[kLength]); + auto fd = std::fopen(inputFile.c_str(), "wb"); + auto closeGuard = makeScopeGuard([&] { std::fclose(fd); }); + for (size_t i = 0; i < (1 << 6) + 1; ++i) { + RDG_genBuffer(buf.get(), kLength, 0.5, 0.0, gen()); + auto written = std::fwrite(buf.get(), 1, kLength, fd); + if (written != kLength) { + std::fprintf(stderr, "Failed to write file, skipping test\n"); + return; + } + } + } + + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.compressionLevel = 1; + if (options.numThreads == 0) { + options.numThreads = 1; + } + ASSERT_TRUE(roundTrip(options)); +} + +TEST(Pzstd, ExtremelyCompressible) { + std::string inputFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + { + std::unique_ptr<uint8_t[]> buf(new uint8_t[10000]); + std::memset(buf.get(), 'a', 10000); + auto fd = std::fopen(inputFile.c_str(), "wb"); + auto written = std::fwrite(buf.get(), 1, 10000, fd); + std::fclose(fd); + ASSERT_EQ(written, 10000); + } + Options options; + options.overwrite = true; + options.inputFiles = {inputFile}; + options.numThreads = 1; + options.compressionLevel = 1; + ASSERT_TRUE(roundTrip(options)); +} diff --git a/src/zstd/contrib/pzstd/test/RoundTrip.h b/src/zstd/contrib/pzstd/test/RoundTrip.h new file mode 100644 index 000000000..c6364ecb4 --- /dev/null +++ b/src/zstd/contrib/pzstd/test/RoundTrip.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ +#pragma once + +#include "Options.h" +#include "Pzstd.h" +#include "utils/ScopeGuard.h" + +#include <cstdio> +#include <string> +#include <cstdint> +#include <memory> + +namespace pzstd { + +inline bool check(std::string source, std::string decompressed) { + std::unique_ptr<std::uint8_t[]> sBuf(new std::uint8_t[1024]); + std::unique_ptr<std::uint8_t[]> dBuf(new std::uint8_t[1024]); + + auto sFd = std::fopen(source.c_str(), "rb"); + auto dFd = std::fopen(decompressed.c_str(), "rb"); + auto guard = makeScopeGuard([&] { + std::fclose(sFd); + std::fclose(dFd); + }); + + size_t sRead, dRead; + + do { + sRead = std::fread(sBuf.get(), 1, 1024, sFd); + dRead = std::fread(dBuf.get(), 1, 1024, dFd); + if (std::ferror(sFd) || std::ferror(dFd)) { + return false; + } + if (sRead != dRead) { + return false; + } + + for (size_t i = 0; i < sRead; ++i) { + if (sBuf.get()[i] != dBuf.get()[i]) { + return false; + } + } + } while (sRead == 1024); + if (!std::feof(sFd) || !std::feof(dFd)) { + return false; + } + return true; +} + +inline bool roundTrip(Options& options) { + if (options.inputFiles.size() != 1) { + return false; + } + std::string source = options.inputFiles.front(); + std::string compressedFile = std::tmpnam(nullptr); + std::string decompressedFile = std::tmpnam(nullptr); + auto guard = makeScopeGuard([&] { + std::remove(compressedFile.c_str()); + std::remove(decompressedFile.c_str()); + }); + + { + options.outputFile = compressedFile; + options.decompress = false; + if (pzstdMain(options) != 0) { + return false; + } + } + { + options.decompress = true; + options.inputFiles.front() = compressedFile; + options.outputFile = decompressedFile; + if (pzstdMain(options) != 0) { + return false; + } + } + return check(source, decompressedFile); +} +} diff --git a/src/zstd/contrib/pzstd/test/RoundTripTest.cpp b/src/zstd/contrib/pzstd/test/RoundTripTest.cpp new file mode 100644 index 000000000..36af0673a --- /dev/null +++ b/src/zstd/contrib/pzstd/test/RoundTripTest.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2016-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ +extern "C" { +#include "datagen.h" +} +#include "Options.h" +#include "test/RoundTrip.h" +#include "utils/ScopeGuard.h" + +#include <cstddef> +#include <cstdio> +#include <cstdlib> +#include <memory> +#include <random> + +using namespace std; +using namespace pzstd; + +namespace { +string +writeData(size_t size, double matchProba, double litProba, unsigned seed) { + std::unique_ptr<uint8_t[]> buf(new uint8_t[size]); + RDG_genBuffer(buf.get(), size, matchProba, litProba, seed); + string file = tmpnam(nullptr); + auto fd = std::fopen(file.c_str(), "wb"); + auto guard = makeScopeGuard([&] { std::fclose(fd); }); + auto bytesWritten = std::fwrite(buf.get(), 1, size, fd); + if (bytesWritten != size) { + std::abort(); + } + return file; +} + +template <typename Generator> +string generateInputFile(Generator& gen) { + // Use inputs ranging from 1 Byte to 2^16 Bytes + std::uniform_int_distribution<size_t> size{1, 1 << 16}; + std::uniform_real_distribution<> prob{0, 1}; + return writeData(size(gen), prob(gen), prob(gen), gen()); +} + +template <typename Generator> +Options generateOptions(Generator& gen, const string& inputFile) { + Options options; + options.inputFiles = {inputFile}; + options.overwrite = true; + + std::uniform_int_distribution<unsigned> numThreads{1, 32}; + std::uniform_int_distribution<unsigned> compressionLevel{1, 10}; + + options.numThreads = numThreads(gen); + options.compressionLevel = compressionLevel(gen); + + return options; +} +} + +int main() { + std::mt19937 gen(std::random_device{}()); + + auto newlineGuard = makeScopeGuard([] { std::fprintf(stderr, "\n"); }); + for (unsigned i = 0; i < 10000; ++i) { + if (i % 100 == 0) { + std::fprintf(stderr, "Progress: %u%%\r", i / 100); + } + auto inputFile = generateInputFile(gen); + auto inputGuard = makeScopeGuard([&] { std::remove(inputFile.c_str()); }); + for (unsigned i = 0; i < 10; ++i) { + auto options = generateOptions(gen, inputFile); + if (!roundTrip(options)) { + std::fprintf(stderr, "numThreads: %u\n", options.numThreads); + std::fprintf(stderr, "level: %u\n", options.compressionLevel); + std::fprintf(stderr, "decompress? %u\n", (unsigned)options.decompress); + std::fprintf(stderr, "file: %s\n", inputFile.c_str()); + return 1; + } + } + } + return 0; +} |