summaryrefslogtreecommitdiffstats
path: root/src/zstd/contrib/pzstd/Pzstd.h
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/zstd/contrib/pzstd/Pzstd.h150
1 files changed, 150 insertions, 0 deletions
diff --git a/src/zstd/contrib/pzstd/Pzstd.h b/src/zstd/contrib/pzstd/Pzstd.h
new file mode 100644
index 000000000..79d1fcca2
--- /dev/null
+++ b/src/zstd/contrib/pzstd/Pzstd.h
@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ */
+#pragma once
+
+#include "ErrorHolder.h"
+#include "Logging.h"
+#include "Options.h"
+#include "utils/Buffer.h"
+#include "utils/Range.h"
+#include "utils/ResourcePool.h"
+#include "utils/ThreadPool.h"
+#include "utils/WorkQueue.h"
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+#undef ZSTD_STATIC_LINKING_ONLY
+
+#include <cstddef>
+#include <cstdint>
+#include <memory>
+
+namespace pzstd {
+/**
+ * Runs pzstd with `options` and returns the number of bytes written.
+ * An error occurred if `errorHandler.hasError()`.
+ *
+ * @param options The pzstd options to use for (de)compression
+ * @returns 0 upon success and non-zero on failure.
+ */
+int pzstdMain(const Options& options);
+
+class SharedState {
+ public:
+ SharedState(const Options& options) : log(options.verbosity) {
+ if (!options.decompress) {
+ auto parameters = options.determineParameters();
+ cStreamPool.reset(new ResourcePool<ZSTD_CStream>{
+ [this, parameters]() -> ZSTD_CStream* {
+ this->log(VERBOSE, "%s\n", "Creating new ZSTD_CStream");
+ auto zcs = ZSTD_createCStream();
+ if (zcs) {
+ auto err = ZSTD_initCStream_advanced(
+ zcs, nullptr, 0, parameters, 0);
+ if (ZSTD_isError(err)) {
+ ZSTD_freeCStream(zcs);
+ return nullptr;
+ }
+ }
+ return zcs;
+ },
+ [](ZSTD_CStream *zcs) {
+ ZSTD_freeCStream(zcs);
+ }});
+ } else {
+ dStreamPool.reset(new ResourcePool<ZSTD_DStream>{
+ [this]() -> ZSTD_DStream* {
+ this->log(VERBOSE, "%s\n", "Creating new ZSTD_DStream");
+ auto zds = ZSTD_createDStream();
+ if (zds) {
+ auto err = ZSTD_initDStream(zds);
+ if (ZSTD_isError(err)) {
+ ZSTD_freeDStream(zds);
+ return nullptr;
+ }
+ }
+ return zds;
+ },
+ [](ZSTD_DStream *zds) {
+ ZSTD_freeDStream(zds);
+ }});
+ }
+ }
+
+ ~SharedState() {
+ // The resource pools have references to this, so destroy them first.
+ cStreamPool.reset();
+ dStreamPool.reset();
+ }
+
+ Logger log;
+ ErrorHolder errorHolder;
+ std::unique_ptr<ResourcePool<ZSTD_CStream>> cStreamPool;
+ std::unique_ptr<ResourcePool<ZSTD_DStream>> dStreamPool;
+};
+
+/**
+ * Streams input from `fd`, breaks input up into chunks, and compresses each
+ * chunk independently. Output of each chunk gets streamed to a queue, and
+ * the output queues get put into `chunks` in order.
+ *
+ * @param state The shared state
+ * @param chunks Each compression jobs output queue gets `pushed()` here
+ * as soon as it is available
+ * @param executor The thread pool to run compression jobs in
+ * @param fd The input file descriptor
+ * @param size The size of the input file if known, 0 otherwise
+ * @param numThreads The number of threads in the thread pool
+ * @param parameters The zstd parameters to use for compression
+ * @returns The number of bytes read from the file
+ */
+std::uint64_t asyncCompressChunks(
+ SharedState& state,
+ WorkQueue<std::shared_ptr<BufferWorkQueue>>& chunks,
+ ThreadPool& executor,
+ FILE* fd,
+ std::uintmax_t size,
+ std::size_t numThreads,
+ ZSTD_parameters parameters);
+
+/**
+ * Streams input from `fd`. If pzstd headers are available it breaks the input
+ * up into independent frames. It sends each frame to an independent
+ * decompression job. Output of each frame gets streamed to a queue, and
+ * the output queues get put into `frames` in order.
+ *
+ * @param state The shared state
+ * @param frames Each decompression jobs output queue gets `pushed()` here
+ * as soon as it is available
+ * @param executor The thread pool to run compression jobs in
+ * @param fd The input file descriptor
+ * @returns The number of bytes read from the file
+ */
+std::uint64_t asyncDecompressFrames(
+ SharedState& state,
+ WorkQueue<std::shared_ptr<BufferWorkQueue>>& frames,
+ ThreadPool& executor,
+ FILE* fd);
+
+/**
+ * Streams input in from each queue in `outs` in order, and writes the data to
+ * `outputFd`.
+ *
+ * @param state The shared state
+ * @param outs A queue of output queues, one for each
+ * (de)compression job.
+ * @param outputFd The file descriptor to write to
+ * @param decompress Are we decompressing?
+ * @returns The number of bytes written
+ */
+std::uint64_t writeFile(
+ SharedState& state,
+ WorkQueue<std::shared_ptr<BufferWorkQueue>>& outs,
+ FILE* outputFd,
+ bool decompress);
+}