Adding upstream version 18.2.2.upstream/18.2.2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

5 files changed, 454 insertions, 0 deletions

diff --git a/src/arrow/dev/tasks/python-wheels/github.linux.amd64.yml b/src/arrow/dev/tasks/python-wheels/github.linux.amd64.yml
new file mode 100644
index 000000000..dc2386482
--- /dev/null
+++ b/src/arrow/dev/tasks/python-wheels/github.linux.amd64.yml
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+
+{{ macros.github_header() }}
+
+jobs:
+  build:
+    name: "Build wheel for Manylinux {{ manylinux_version }}"
+    runs-on: ubuntu-latest
+    env:
+      # archery uses these environment variables
+      ARCH: amd64
+      PYTHON: "{{ python_version }}"
+
+    steps:
+      {{ macros.github_checkout_arrow()|indent }}
+      {{ macros.github_install_archery()|indent }}
+      {{ macros.github_login_dockerhub()|indent }}
+
+      - name: Build wheel
+        shell: bash
+        run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
+
+      # TODO(kszucs): auditwheel show
+      - name: Test wheel
+        shell: bash
+        run: |
+          archery docker run python-wheel-manylinux-test-imports
+          archery docker run python-wheel-manylinux-test-unittests
+
+      {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
+      {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
+
+      {% if arrow.branch == 'master' %}
+      - name: Push Docker Image
+        shell: bash
+        run: |
+          archery docker push python-wheel-manylinux-{{ manylinux_version }}
+          archery docker push python-wheel-manylinux-test-unittests
+      {% endif %}
diff --git a/src/arrow/dev/tasks/python-wheels/github.osx.amd64.yml b/src/arrow/dev/tasks/python-wheels/github.osx.amd64.yml
new file mode 100644
index 000000000..8078abfd5
--- /dev/null
+++ b/src/arrow/dev/tasks/python-wheels/github.osx.amd64.yml
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{% import 'macros.jinja' as macros with context %}
+
+{{ macros.github_header() }}
+
+env:
+  ARROW_S3: {{ arrow_s3 }}
+  CC: "clang"
+  CXX: "clang++"
+  MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}"
+  PYARROW_BUILD_VERBOSE: 1
+  PYARROW_VERSION: "{{ arrow.no_rc_version }}"
+  PYTHON_VERSION: "{{ python_version }}"
+  PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}"
+  VCPKG_DEFAULT_TRIPLET: x64-osx-static-release
+  VCPKG_FEATURE_FLAGS: "-manifests"
+  VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }}
+  VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }}
+  VCPKG_VERSION: "{{ vcpkg_version }}"
+
+jobs:
+  build:
+    name: Build wheel for OS X
+    runs-on: macos-10.15
+    steps:
+      {{ macros.github_checkout_arrow()|indent }}
+
+      - name: Install System Dependencies
+        run: brew install bash bison coreutils ninja cmake
+
+      - uses: actions/cache@v2
+        id: vcpkg-cache
+        with:
+          path: vcpkg
+          key: vcpkg-{{ macos_deployment_target }}-{{ vcpkg_version }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }}
+
+      - name: Install Vcpkg
+        if: steps.vcpkg-cache.outputs.cache-hit != 'true'
+        shell: bash
+        env:
+          MACOSX_DEPLOYMENT_TARGET: "10.15"
+        run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT
+
+      - name: Install Packages
+        run: |
+          $VCPKG_ROOT/vcpkg install \
+            abseil \
+            boost-filesystem \
+            brotli \
+            bzip2 \
+            c-ares \
+            curl \
+            flatbuffers \
+            gflags \
+            glog \
+            grpc \
+            lz4 \
+            openssl \
+            orc \
+            protobuf \
+            rapidjson \
+            re2 \
+            snappy \
+            thrift \
+            utf8proc \
+            zlib \
+            zstd
+
+      {% if arrow_s3 == "ON" %}
+      - name: Install AWS SDK C++
+        run: $VCPKG_ROOT/vcpkg install aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer]
+      {% endif %}
+
+      - name: Install Python {{ python_version }}
+        shell: bash
+        run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version }}
+
+      - name: Build Wheel
+        shell: bash
+        run: |
+          $PYTHON -m virtualenv build-env
+          source build-env/bin/activate
+          pip install --upgrade pip wheel
+          arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build
+
+      - name: Test Wheel
+        shell: bash
+        run: |
+          $PYTHON -m virtualenv test-env
+          source test-env/bin/activate
+          pip install --upgrade pip wheel
+          arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
+
+      {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
+      {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
diff --git a/src/arrow/dev/tasks/python-wheels/github.osx.arm64.yml b/src/arrow/dev/tasks/python-wheels/github.osx.arm64.yml
new file mode 100644
index 000000000..e5456dbfc
--- /dev/null
+++ b/src/arrow/dev/tasks/python-wheels/github.osx.arm64.yml
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Prerequisits on the host:
+# - brew install bash bison coreutils ninja cmake
+# - sudo arrow/ci/scripts/install_python.sh macos 3.9
+
+{% import 'macros.jinja' as macros with context %}
+
+{{ macros.github_header() }}
+
+env:
+  ARROW_FLIGHT: OFF
+  ARROW_JEMALLOC: OFF
+  ARROW_SIMD_LEVEL: "{{ arrow_simd_level }}"
+  CC: "clang"
+  CMAKE_BUILD_TYPE: release
+  CMAKE_CXX_COMPILER_LAUNCHER: "ccache"
+  CXX: "clang++"
+  MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}"
+  PYARROW_BUILD_VERBOSE: 1
+  PYARROW_VERSION: "{{ arrow.no_rc_version }}"
+  PYTHON_VERSION: "{{ python_version }}"
+  PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}"
+  VCPKG_DEFAULT_TRIPLET: {{ arch }}-osx-static-release
+  VCPKG_FEATURE_FLAGS: "-manifests"
+  VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }}
+  VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }}
+  VCPKG_VERSION: "{{ vcpkg_version }}"
+
+jobs:
+  build:
+    name: Build wheel for OS X
+    runs-on: self-hosted
+    steps:
+      - name: Cleanup
+        shell: bash
+        run: rm -rf arrow vcpkg build crossbow-env build-env test-*-env
+
+      {{ macros.github_checkout_arrow()|indent }}
+
+      - name: Add Brew's Bison to PATH
+        shell: bash
+        run: echo "/opt/homebrew/opt/bison/bin" >> $GITHUB_PATH
+
+      - name: Install Vcpkg
+        shell: bash
+        env:
+          MACOSX_DEPLOYMENT_TARGET: "11.0"
+        run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT
+
+      - name: Install OpenSSL
+        shell: bash
+        run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl
+
+      {% if arch == "universal2" %}
+      # OpenSSL doesn't provide an universal2 configuration yet, so vcpkg is
+      # unable to propagate the list of architectures from VCPKG_OSX_ARCHIETCTURES.
+      # In order to prevent link time warnings (which may turn out to be errors)
+      # we compile OpenSSL separately for the two architectures and merge the
+      # binaries into universal2 ones using `lipo`.
+      - name: Create universal binaries for OpenSSL
+        shell: bash
+        run: |
+          for arch in arm64 x64; do
+            VCPKG_DEFAULT_TRIPLET=${arch}-osx-static-release arch -arm64 $VCPKG_ROOT/vcpkg install openssl
+          done
+          for lib in libcrypto libssl; do
+            lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \
+                         $VCPKG_ROOT/installed/x64-osx-static-release/lib/${lib}.a \
+                 -output $VCPKG_ROOT/installed/universal2-osx-static-release/lib/${lib}.a
+          done
+      {% endif %}
+
+      - name: Install Packages
+        run: |
+          arch -arm64 $VCPKG_ROOT/vcpkg install \
+            aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
+            boost-filesystem \
+            brotli \
+            bzip2 \
+            c-ares \
+            curl \
+            flatbuffers \
+            gflags \
+            glog \
+            lz4 \
+            orc \
+            protobuf \
+            rapidjson \
+            re2 \
+            snappy \
+            thrift \
+            utf8proc \
+            zlib \
+            zstd
+
+      - name: Build Wheel
+        shell: bash
+        run: |
+          $PYTHON -m virtualenv build-env
+          source build-env/bin/activate
+          pip install --upgrade pip wheel
+          arch -arm64 arrow/ci/scripts/python_wheel_macos_build.sh {{ arch }} $(pwd)/arrow $(pwd)/build
+
+      - name: Test Wheel on ARM64
+        shell: bash
+        env:
+          PYTEST_ADDOPTS: "-k 'not test_cancellation'"
+        run: |
+          $PYTHON -m virtualenv test-arm64-env
+          source test-arm64-env/bin/activate
+          pip install --upgrade pip wheel
+          arch -arm64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
+
+      {% if arch == "universal2" %}
+      - name: Test Wheel on AMD64
+        shell: bash
+        env:
+          PYTEST_ADDOPTS: "-k 'not test_cancellation'"
+        run: |
+          $PYTHON -m virtualenv test-amd64-env
+          source test-amd64-env/bin/activate
+          pip install --upgrade pip wheel
+          arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
+      {% endif %}
+
+      - name: Upload artifacts
+        shell: bash
+        run: |
+          $PYTHON -m virtualenv crossbow-env
+          source crossbow-env/bin/activate
+          arch -x86_64 pip install -e arrow/dev/archery[crossbow-upload]
+          arch -x86_64 archery crossbow \
+          --queue-path $(pwd) \
+          --queue-remote {{ queue_remote_url }} \
+          upload-artifacts \
+          --sha {{ task.branch }} \
+          --tag {{ task.tag }} \
+          "arrow/python/repaired_wheels/*.whl"
+        env:
+          CROSSBOW_GITHUB_TOKEN: {{ "${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" }}
+
+      {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
diff --git a/src/arrow/dev/tasks/python-wheels/github.windows.yml b/src/arrow/dev/tasks/python-wheels/github.windows.yml
new file mode 100644
index 000000000..f9989aed0
--- /dev/null
+++ b/src/arrow/dev/tasks/python-wheels/github.windows.yml
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+
+{{ macros.github_header() }}
+
+jobs:
+  build:
+    name: "Build wheel for Windows"
+    runs-on: windows-2019
+    env:
+      # archery uses this environment variable
+      PYTHON: "{{ python_version }}"
+      # this is a private repository at the moment (mostly because of licensing
+      # consideration of windows images with visual studio), but anyone can
+      # recreate the image by manually building it via:
+      # `archery build python-wheel-windows-vs2017`
+      # note that we don't run docker build since there wouldn't be a cache hit
+      # and rebuilding the dependencies takes a fair amount of time
+      REPO: ghcr.io/ursacomputing/arrow
+      # prefer the docker cli over docker-compose
+      ARCHERY_USE_DOCKER_CLI: 1
+
+    steps:
+      {{ macros.github_checkout_arrow()|indent }}
+      {{ macros.github_login_ghcr()|indent }}
+      {{ macros.github_install_archery()|indent }}
+
+      - name: Build wheel
+        shell: cmd
+        run: archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017
+
+      # Two layers of the official python 3.6 windows image are not available for download.
+      # Docker pull returns with unexpected status resolving reader: 403 Forbidden.
+      # See https://issues.apache.org/jira/browse/ARROW-14424
+      {% if python_version != "3.6" %}
+      - name: Test wheel
+        shell: cmd
+        run: archery docker run python-wheel-windows-test
+      {% endif %}
+
+      {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }}
+      {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }}
diff --git a/src/arrow/dev/tasks/python-wheels/travis.linux.arm64.yml b/src/arrow/dev/tasks/python-wheels/travis.linux.arm64.yml
new file mode 100644
index 000000000..d32d89d83
--- /dev/null
+++ b/src/arrow/dev/tasks/python-wheels/travis.linux.arm64.yml
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+
+arch: arm64-graviton2
+virt: vm
+os: linux
+dist: focal
+group: edge
+language: minimal
+
+addons:
+  apt:
+    packages:
+      - libgit2-dev
+      - python3-pip
+
+services:
+  - docker
+
+# don't build twice
+if: tag IS blank
+
+env:
+  global:
+    - BUILD_REF={{ arrow.head }}
+    - TRAVIS_TAG={{ task.tag }}
+    # archery uses these environment variables
+    - ARCH=arm64v8
+    - PYTHON="{{ python_version }}"
+
+before_script:
+  - set -e
+  {{ macros.travis_checkout_arrow() }}
+  {{ macros.travis_docker_login() }}
+
+script:
+  # Install Archery and Crossbow dependencies
+  {{ macros.travis_install_archery() }}
+
+  # Build and Test packages
+  # output something every minutes to prevent travis from killing the build
+  - while sleep 1m; do echo "=====[ $SECONDS seconds still running ]====="; done &
+  - archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
+  - archery docker run python-wheel-manylinux-test-imports
+  - archery docker run python-wheel-manylinux-test-unittests
+  - kill %1
+
+after_success:
+  # Upload wheel as github artifact
+  {{ macros.travis_upload_releases("arrow/python/repaired_wheels/*.whl") }}
+  {{ macros.travis_upload_gemfury("arrow/python/repaired_wheels/*.whl") }}
+
+  {% if arrow.branch == 'master' %}
+  # Push the docker image to dockerhub
+  - archery docker push python-wheel-manylinux-{{ manylinux_version }}
+  - archery docker push python-wheel-manylinux-test-unittests
+  {% endif %}