diff options
Diffstat (limited to 'src/arrow/dev/tasks/python-wheels')
5 files changed, 454 insertions, 0 deletions
diff --git a/src/arrow/dev/tasks/python-wheels/github.linux.amd64.yml b/src/arrow/dev/tasks/python-wheels/github.linux.amd64.yml new file mode 100644 index 000000000..dc2386482 --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/github.linux.amd64.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + build: + name: "Build wheel for Manylinux {{ manylinux_version }}" + runs-on: ubuntu-latest + env: + # archery uses these environment variables + ARCH: amd64 + PYTHON: "{{ python_version }}" + + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + {{ macros.github_login_dockerhub()|indent }} + + - name: Build wheel + shell: bash + run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} + + # TODO(kszucs): auditwheel show + - name: Test wheel + shell: bash + run: | + archery docker run python-wheel-manylinux-test-imports + archery docker run python-wheel-manylinux-test-unittests + + {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} + + {% if arrow.branch == 'master' %} + - name: Push Docker Image + shell: bash + run: | + archery docker push python-wheel-manylinux-{{ manylinux_version }} + archery docker push python-wheel-manylinux-test-unittests + {% endif %} diff --git a/src/arrow/dev/tasks/python-wheels/github.osx.amd64.yml b/src/arrow/dev/tasks/python-wheels/github.osx.amd64.yml new file mode 100644 index 000000000..8078abfd5 --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/github.osx.amd64.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +env: + ARROW_S3: {{ arrow_s3 }} + CC: "clang" + CXX: "clang++" + MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}" + PYARROW_BUILD_VERBOSE: 1 + PYARROW_VERSION: "{{ arrow.no_rc_version }}" + PYTHON_VERSION: "{{ python_version }}" + PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}" + VCPKG_DEFAULT_TRIPLET: x64-osx-static-release + VCPKG_FEATURE_FLAGS: "-manifests" + VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} + VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }} + VCPKG_VERSION: "{{ vcpkg_version }}" + +jobs: + build: + name: Build wheel for OS X + runs-on: macos-10.15 + steps: + {{ macros.github_checkout_arrow()|indent }} + + - name: Install System Dependencies + run: brew install bash bison coreutils ninja cmake + + - uses: actions/cache@v2 + id: vcpkg-cache + with: + path: vcpkg + key: vcpkg-{{ macos_deployment_target }}-{{ vcpkg_version }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }} + + - name: Install Vcpkg + if: steps.vcpkg-cache.outputs.cache-hit != 'true' + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: "10.15" + run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT + + - name: Install Packages + run: | + $VCPKG_ROOT/vcpkg install \ + abseil \ + boost-filesystem \ + brotli \ + bzip2 \ + c-ares \ + curl \ + flatbuffers \ + gflags \ + glog \ + grpc \ + lz4 \ + openssl \ + orc \ + protobuf \ + rapidjson \ + re2 \ + snappy \ + thrift \ + utf8proc \ + zlib \ + zstd + + {% if arrow_s3 == "ON" %} + - name: Install AWS SDK C++ + run: $VCPKG_ROOT/vcpkg install aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] + {% endif %} + + - name: Install Python {{ python_version }} + shell: bash + run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version }} + + - name: Build Wheel + shell: bash + run: | + $PYTHON -m virtualenv build-env + source build-env/bin/activate + pip install --upgrade pip wheel + arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build + + - name: Test Wheel + shell: bash + run: | + $PYTHON -m virtualenv test-env + source test-env/bin/activate + pip install --upgrade pip wheel + arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + + {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/src/arrow/dev/tasks/python-wheels/github.osx.arm64.yml b/src/arrow/dev/tasks/python-wheels/github.osx.arm64.yml new file mode 100644 index 000000000..e5456dbfc --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/github.osx.arm64.yml @@ -0,0 +1,157 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prerequisits on the host: +# - brew install bash bison coreutils ninja cmake +# - sudo arrow/ci/scripts/install_python.sh macos 3.9 + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +env: + ARROW_FLIGHT: OFF + ARROW_JEMALLOC: OFF + ARROW_SIMD_LEVEL: "{{ arrow_simd_level }}" + CC: "clang" + CMAKE_BUILD_TYPE: release + CMAKE_CXX_COMPILER_LAUNCHER: "ccache" + CXX: "clang++" + MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}" + PYARROW_BUILD_VERBOSE: 1 + PYARROW_VERSION: "{{ arrow.no_rc_version }}" + PYTHON_VERSION: "{{ python_version }}" + PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}" + VCPKG_DEFAULT_TRIPLET: {{ arch }}-osx-static-release + VCPKG_FEATURE_FLAGS: "-manifests" + VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} + VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }} + VCPKG_VERSION: "{{ vcpkg_version }}" + +jobs: + build: + name: Build wheel for OS X + runs-on: self-hosted + steps: + - name: Cleanup + shell: bash + run: rm -rf arrow vcpkg build crossbow-env build-env test-*-env + + {{ macros.github_checkout_arrow()|indent }} + + - name: Add Brew's Bison to PATH + shell: bash + run: echo "/opt/homebrew/opt/bison/bin" >> $GITHUB_PATH + + - name: Install Vcpkg + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: "11.0" + run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT + + - name: Install OpenSSL + shell: bash + run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl + + {% if arch == "universal2" %} + # OpenSSL doesn't provide an universal2 configuration yet, so vcpkg is + # unable to propagate the list of architectures from VCPKG_OSX_ARCHIETCTURES. + # In order to prevent link time warnings (which may turn out to be errors) + # we compile OpenSSL separately for the two architectures and merge the + # binaries into universal2 ones using `lipo`. + - name: Create universal binaries for OpenSSL + shell: bash + run: | + for arch in arm64 x64; do + VCPKG_DEFAULT_TRIPLET=${arch}-osx-static-release arch -arm64 $VCPKG_ROOT/vcpkg install openssl + done + for lib in libcrypto libssl; do + lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \ + $VCPKG_ROOT/installed/x64-osx-static-release/lib/${lib}.a \ + -output $VCPKG_ROOT/installed/universal2-osx-static-release/lib/${lib}.a + done + {% endif %} + + - name: Install Packages + run: | + arch -arm64 $VCPKG_ROOT/vcpkg install \ + aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ + boost-filesystem \ + brotli \ + bzip2 \ + c-ares \ + curl \ + flatbuffers \ + gflags \ + glog \ + lz4 \ + orc \ + protobuf \ + rapidjson \ + re2 \ + snappy \ + thrift \ + utf8proc \ + zlib \ + zstd + + - name: Build Wheel + shell: bash + run: | + $PYTHON -m virtualenv build-env + source build-env/bin/activate + pip install --upgrade pip wheel + arch -arm64 arrow/ci/scripts/python_wheel_macos_build.sh {{ arch }} $(pwd)/arrow $(pwd)/build + + - name: Test Wheel on ARM64 + shell: bash + env: + PYTEST_ADDOPTS: "-k 'not test_cancellation'" + run: | + $PYTHON -m virtualenv test-arm64-env + source test-arm64-env/bin/activate + pip install --upgrade pip wheel + arch -arm64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + + {% if arch == "universal2" %} + - name: Test Wheel on AMD64 + shell: bash + env: + PYTEST_ADDOPTS: "-k 'not test_cancellation'" + run: | + $PYTHON -m virtualenv test-amd64-env + source test-amd64-env/bin/activate + pip install --upgrade pip wheel + arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + {% endif %} + + - name: Upload artifacts + shell: bash + run: | + $PYTHON -m virtualenv crossbow-env + source crossbow-env/bin/activate + arch -x86_64 pip install -e arrow/dev/archery[crossbow-upload] + arch -x86_64 archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + "arrow/python/repaired_wheels/*.whl" + env: + CROSSBOW_GITHUB_TOKEN: {{ "${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" }} + + {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/src/arrow/dev/tasks/python-wheels/github.windows.yml b/src/arrow/dev/tasks/python-wheels/github.windows.yml new file mode 100644 index 000000000..f9989aed0 --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/github.windows.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + build: + name: "Build wheel for Windows" + runs-on: windows-2019 + env: + # archery uses this environment variable + PYTHON: "{{ python_version }}" + # this is a private repository at the moment (mostly because of licensing + # consideration of windows images with visual studio), but anyone can + # recreate the image by manually building it via: + # `archery build python-wheel-windows-vs2017` + # note that we don't run docker build since there wouldn't be a cache hit + # and rebuilding the dependencies takes a fair amount of time + REPO: ghcr.io/ursacomputing/arrow + # prefer the docker cli over docker-compose + ARCHERY_USE_DOCKER_CLI: 1 + + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_login_ghcr()|indent }} + {{ macros.github_install_archery()|indent }} + + - name: Build wheel + shell: cmd + run: archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017 + + # Two layers of the official python 3.6 windows image are not available for download. + # Docker pull returns with unexpected status resolving reader: 403 Forbidden. + # See https://issues.apache.org/jira/browse/ARROW-14424 + {% if python_version != "3.6" %} + - name: Test wheel + shell: cmd + run: archery docker run python-wheel-windows-test + {% endif %} + + {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} + {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} diff --git a/src/arrow/dev/tasks/python-wheels/travis.linux.arm64.yml b/src/arrow/dev/tasks/python-wheels/travis.linux.arm64.yml new file mode 100644 index 000000000..d32d89d83 --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/travis.linux.arm64.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +arch: arm64-graviton2 +virt: vm +os: linux +dist: focal +group: edge +language: minimal + +addons: + apt: + packages: + - libgit2-dev + - python3-pip + +services: + - docker + +# don't build twice +if: tag IS blank + +env: + global: + - BUILD_REF={{ arrow.head }} + - TRAVIS_TAG={{ task.tag }} + # archery uses these environment variables + - ARCH=arm64v8 + - PYTHON="{{ python_version }}" + +before_script: + - set -e + {{ macros.travis_checkout_arrow() }} + {{ macros.travis_docker_login() }} + +script: + # Install Archery and Crossbow dependencies + {{ macros.travis_install_archery() }} + + # Build and Test packages + # output something every minutes to prevent travis from killing the build + - while sleep 1m; do echo "=====[ $SECONDS seconds still running ]====="; done & + - archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} + - archery docker run python-wheel-manylinux-test-imports + - archery docker run python-wheel-manylinux-test-unittests + - kill %1 + +after_success: + # Upload wheel as github artifact + {{ macros.travis_upload_releases("arrow/python/repaired_wheels/*.whl") }} + {{ macros.travis_upload_gemfury("arrow/python/repaired_wheels/*.whl") }} + + {% if arrow.branch == 'master' %} + # Push the docker image to dockerhub + - archery docker push python-wheel-manylinux-{{ manylinux_version }} + - archery docker push python-wheel-manylinux-test-unittests + {% endif %} |