Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

2 files changed, 113 insertions, 0 deletions

diff --git a/src/arrow/cpp/build-support/fuzzing/generate_corpuses.sh b/src/arrow/cpp/build-support/fuzzing/generate_corpuses.sh
new file mode 100755
index 000000000..e3f00e647
--- /dev/null
+++ b/src/arrow/cpp/build-support/fuzzing/generate_corpuses.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Generate and pack seed corpus files, for OSS-Fuzz
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 <build output dir>"
+    exit 1
+fi
+
+set -ex
+
+CORPUS_DIR=/tmp/corpus
+ARROW_ROOT=$(cd $(dirname $BASH_SOURCE)/../../..; pwd)
+ARROW_CPP=$ARROW_ROOT/cpp
+OUT=$1
+
+# NOTE: name of seed corpus output file should be "<FUZZ TARGET>-seed_corpus.zip"
+# where "<FUZZ TARGET>" is the exact name of the fuzz target executable the
+# seed corpus is generated for.
+
+IPC_INTEGRATION_FILES=$(find ${ARROW_ROOT}/testing/data/arrow-ipc-stream/integration -name "*.stream")
+
+rm -rf ${CORPUS_DIR}
+${OUT}/arrow-ipc-generate-fuzz-corpus -stream ${CORPUS_DIR}
+# Several IPC integration files can have the same name, make sure
+# they all appear in the corpus by numbering the duplicates.
+cp --backup=numbered ${IPC_INTEGRATION_FILES} ${CORPUS_DIR}
+${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-ipc-stream-fuzz_seed_corpus.zip
+
+rm -rf ${CORPUS_DIR}
+${OUT}/arrow-ipc-generate-fuzz-corpus -file ${CORPUS_DIR}
+${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-ipc-file-fuzz_seed_corpus.zip
+
+rm -rf ${CORPUS_DIR}
+${OUT}/arrow-ipc-generate-tensor-fuzz-corpus -stream ${CORPUS_DIR}
+${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/arrow-ipc-tensor-stream-fuzz_seed_corpus.zip
+
+rm -rf ${CORPUS_DIR}
+${OUT}/parquet-arrow-generate-fuzz-corpus ${CORPUS_DIR}
+# Add Parquet testing examples
+cp ${ARROW_CPP}/submodules/parquet-testing/data/*.parquet ${CORPUS_DIR}
+${ARROW_CPP}/build-support/fuzzing/pack_corpus.py ${CORPUS_DIR} ${OUT}/parquet-arrow-fuzz_seed_corpus.zip
diff --git a/src/arrow/cpp/build-support/fuzzing/pack_corpus.py b/src/arrow/cpp/build-support/fuzzing/pack_corpus.py
new file mode 100755
index 000000000..2064fed60
--- /dev/null
+++ b/src/arrow/cpp/build-support/fuzzing/pack_corpus.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Rename a bunch of corpus files to their SHA1 hashes, and
+# pack them into a ZIP archive.
+
+import hashlib
+from pathlib import Path
+import sys
+import zipfile
+
+
+def process_dir(corpus_dir, zip_output):
+    seen = set()
+
+    for child in corpus_dir.iterdir():
+        if not child.is_file():
+            raise IOError("Not a file: {0}".format(child))
+        with child.open('rb') as f:
+            data = f.read()
+        arcname = hashlib.sha1(data).hexdigest()
+        if arcname in seen:
+            raise ValueError("Duplicate hash: {0} (in file {1})"
+                             .format(arcname, child))
+        zip_output.writestr(str(arcname), data)
+        seen.add(arcname)
+
+
+def main(corpus_dir, zip_output_name):
+    with zipfile.ZipFile(zip_output_name, 'w') as zip_output:
+        process_dir(Path(corpus_dir), zip_output)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        print("Usage: {0} <corpus dir> <output zip file>".format(sys.argv[0]))
+        sys.exit(1)
+    main(sys.argv[1], sys.argv[2])