diff options
Diffstat (limited to 'src/arrow/.github/workflows')
24 files changed, 3104 insertions, 0 deletions
diff --git a/src/arrow/.github/workflows/archery.yml b/src/arrow/.github/workflows/archery.yml new file mode 100644 index 000000000..fa84f4dc3 --- /dev/null +++ b/src/arrow/.github/workflows/archery.yml @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Archery & Crossbow + +on: + push: + paths: + - '.github/workflows/archery.yml' + - 'dev/archery/**' + - 'dev/tasks/**' + - 'docker-compose.yml' + pull_request: + paths: + - '.github/workflows/archery.yml' + - 'dev/archery/**' + - 'dev/tasks/**' + - 'docker-compose.yml' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + + test: + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + name: Archery Unittests and Crossbow Check Config + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Git Fixup + shell: bash + run: git branch master origin/master || true + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: '3.6' + - name: Install pygit2 binary wheel + run: pip install pygit2 --only-binary pygit2 + - name: Install Archery, Crossbow- and Test Dependencies + run: pip install pytest responses -e dev/archery[all] + - name: Archery Unittests + working-directory: dev/archery + run: pytest -v archery + - name: Archery Docker Validation + run: archery docker check-config + - name: Crossbow Check Config + working-directory: dev/tasks + run: archery crossbow check-config diff --git a/src/arrow/.github/workflows/comment_bot.yml b/src/arrow/.github/workflows/comment_bot.yml new file mode 100644 index 000000000..b78ae80fb --- /dev/null +++ b/src/arrow/.github/workflows/comment_bot.yml @@ -0,0 +1,170 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Comment Bot + +on: + # TODO(kszucs): support pull_request_review_comment + issue_comment: + types: + - created + - edited + +jobs: + crossbow: + name: Listen! + if: startsWith(github.event.comment.body, '@github-actions crossbow') + runs-on: ubuntu-latest + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + path: arrow + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Archery and Crossbow dependencies + run: pip install -e arrow/dev/archery[bot] + - name: Handle Github comment event + env: + ARROW_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CROSSBOW_GITHUB_TOKEN: ${{ secrets.CROSSBOW_GITHUB_TOKEN }} + run: | + archery trigger-bot \ + --event-name ${{ github.event_name }} \ + --event-payload ${{ github.event_path }} + + autotune: + name: "Fix all the things" + if: startsWith(github.event.comment.body, '@github-actions autotune') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: r-lib/actions/pr-fetch@master + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: See what is different + run: | + set -ex + git remote add upstream https://github.com/apache/arrow + git fetch upstream + changed() { + git diff --name-only HEAD..upstream/master | grep -e "$1" >/dev/null 2>&1 + } + if changed '^r/.*\.R$'; then + echo "R_DOCS=true" >> $GITHUB_ENV + echo "R_CODE=true" >> $GITHUB_ENV + fi + if changed 'cmake' || changed 'CMake'; then + echo "CMAKE_FORMAT=true" >> $GITHUB_ENV + fi + if changed '^cpp/src'; then + echo "CLANG_FORMAT_CPP=true" >> $GITHUB_ENV + fi + if changed '^r/src'; then + echo "CLANG_FORMAT_R=true" >> $GITHUB_ENV + fi + - name: Ensure clang-format has the appropriate versoin + if: env.CMAKE_FORMAT == 'true' || + env.CLANG_FORMAT_CPP == 'true' || + env.CLANG_FORMAT_R == 'true' || + endsWith(github.event.comment.body, 'everything') + run: | + set -e + . .env # To get the clang version we use + sudo apt update + sudo apt install -y clang-format-${CLANG_TOOLS} + - name: Run cmake_format + if: env.CMAKE_FORMAT == 'true' || endsWith(github.event.comment.body, 'everything') + run: | + set -ex + export PATH=/home/runner/.local/bin:$PATH + python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install -e dev/archery[lint] + archery lint --cmake-format --fix + - name: Run clang-format on cpp + if: env.CLANG_FORMAT_CPP == 'true' || endsWith(github.event.comment.body, 'everything') + run: | + . .env # To get the clang version we use + cpp/build-support/run_clang_format.py \ + --clang_format_binary=clang-format-${CLANG_TOOLS} \ + --exclude_glob=cpp/build-support/lint_exclusions.txt \ + --source_dir=cpp/src --quiet --fix + - name: Run clang-format on r + if: env.CLANG_FORMAT_R == 'true' || endsWith(github.event.comment.body, 'everything') + run: | + . .env # To get the clang version we use + cpp/build-support/run_clang_format.py \ + --clang_format_binary=clang-format-${CLANG_TOOLS} \ + --exclude_glob=cpp/build-support/lint_exclusions.txt \ + --source_dir=r/src --quiet --fix + - uses: r-lib/actions/setup-r@v1 + if: env.R_DOCS == 'true' || env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') + - name: Update R docs + if: env.R_DOCS == 'true' || endsWith(github.event.comment.body, 'everything') + shell: Rscript {0} + run: | + source("ci/etc/rprofile") + install.packages(c("remotes", "roxygen2")) + # We currently need dev roxygen2 (> 7.1.1) until they release + remotes::install_github("r-lib/roxygen2") + remotes::install_deps("r") + roxygen2::roxygenize("r") + - name: Style R code + if: env.R_CODE == 'true' || endsWith(github.event.comment.body, 'everything') + shell: Rscript {0} + run: | + changed_files <- system("git diff --name-only HEAD..upstream/master 2>&1", intern = TRUE) + # only grab the .R files under r/ + changed_files <- grep('^r/.*\\.R$', changed_files, value = TRUE) + # remove latin1 which is unstylable due to encoding and codegen.R which is unique + changed_files <- changed_files[!changed_files %in% file.path("r", source("r/.styler_excludes.R")$value)] + source("ci/etc/rprofile") + install.packages(c("remotes", "styler")) + remotes::install_deps("r") + styler::style_file(changed_files) + - name: Commit results + run: | + git config user.name "$(git log -1 --pretty=format:%an)" + git config user.email "$(git log -1 --pretty=format:%ae)" + git commit -a -m 'Autoformat/render all the things [automated commit]' || echo "No changes to commit" + - uses: r-lib/actions/pr-push@master + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + rebase: + name: "Rebase" + if: startsWith(github.event.comment.body, '@github-actions rebase') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: r-lib/actions/pr-fetch@master + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Rebase on ${{ github.repository }} master + run: | + set -ex + git config user.name "$(git log -1 --pretty=format:%an)" + git config user.email "$(git log -1 --pretty=format:%ae)" + git remote add upstream https://github.com/${{ github.repository }} + git fetch --unshallow upstream master + git rebase upstream/master + - uses: r-lib/actions/pr-push@master + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + args: "--force" diff --git a/src/arrow/.github/workflows/cpp.yml b/src/arrow/.github/workflows/cpp.yml new file mode 100644 index 000000000..0f19f7351 --- /dev/null +++ b/src/arrow/.github/workflows/cpp.yml @@ -0,0 +1,329 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: C++ + +on: + push: + paths: + - '.github/workflows/cpp.yml' + - 'ci/docker/**' + - 'ci/scripts/cpp_*' + - 'ci/scripts/msys2_*' + - 'ci/scripts/util_*' + - 'cpp/**' + - 'format/Flight.proto' + pull_request: + paths: + - '.github/workflows/cpp.yml' + - 'ci/docker/**' + - 'ci/scripts/cpp_*' + - 'ci/scripts/msys2_*' + - 'ci/scripts/util_*' + - 'cpp/**' + - 'format/Flight.proto' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + ARROW_ENABLE_TIMING_TESTS: OFF + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + docker: + name: ${{ matrix.title }} + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + image: + - conda-cpp + - ubuntu-cpp-sanitizer + include: + - image: conda-cpp + title: AMD64 Conda C++ + - image: ubuntu-cpp-sanitizer + title: AMD64 Ubuntu 20.04 C++ ASAN UBSAN + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }} + restore-keys: ${{ matrix.image }}- + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + archery docker run ${{ matrix.image }} + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push ${{ matrix.image }} + + build-example: + name: C++ Minimal Build Example + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 45 + strategy: + fail-fast: false + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Run + run: | + cd cpp/examples/minimal_build + docker-compose run --rm minimal + + macos: + name: AMD64 MacOS 10.15 C++ + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 45 + strategy: + fail-fast: false + env: + ARROW_BUILD_TESTS: ON + ARROW_DATASET: ON + ARROW_FLIGHT: ON + ARROW_GANDIVA: ON + ARROW_HDFS: ON + ARROW_HOME: /usr/local + ARROW_JEMALLOC: ON + # TODO(kszucs): link error in the tests + ARROW_ORC: OFF + ARROW_PARQUET: ON + ARROW_PLASMA: ON + ARROW_S3: ON + ARROW_WITH_BROTLI: ON + ARROW_WITH_BZ2: ON + ARROW_WITH_LZ4: ON + ARROW_WITH_SNAPPY: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_ZSTD: ON + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Install Dependencies + shell: bash + run: | + rm -f /usr/local/bin/2to3 + brew update --preinstall + brew bundle --file=cpp/Brewfile + - name: Build + shell: bash + run: ci/scripts/cpp_build.sh $(pwd) $(pwd)/build + - name: Test + shell: bash + run: | + sudo sysctl -w kern.coredump=1 + sudo sysctl -w kern.corefile=core.%N.%P + ulimit -c unlimited # must enable within the same shell + ci/scripts/cpp_test.sh $(pwd) $(pwd)/build + + windows: + name: AMD64 ${{ matrix.name }} C++ + runs-on: ${{ matrix.os }} + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + os: + - windows-latest + include: + - os: windows-latest + name: Windows 2019 + generator: Visual Studio 16 2019 + env: + ARROW_BOOST_USE_SHARED: OFF + ARROW_BUILD_BENCHMARKS: ON + ARROW_BUILD_SHARED: ON + ARROW_BUILD_STATIC: OFF + ARROW_BUILD_TESTS: ON + ARROW_DATASET: ON + ARROW_FLIGHT: OFF + ARROW_HDFS: ON + ARROW_HOME: /usr + ARROW_JEMALLOC: OFF + ARROW_MIMALLOC: ON + ARROW_PARQUET: ON + ARROW_USE_GLOG: OFF + ARROW_VERBOSE_THIRDPARTY_BUILD: OFF + ARROW_WITH_BROTLI: OFF + ARROW_WITH_BZ2: OFF + ARROW_WITH_LZ4: OFF + ARROW_WITH_SNAPPY: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_ZSTD: ON + BOOST_SOURCE: BUNDLED + CMAKE_ARGS: '-A x64 -DOPENSSL_ROOT_DIR=C:\Program Files\OpenSSL-Win64' + CMAKE_GENERATOR: ${{ matrix.generator }} + CMAKE_INSTALL_LIBDIR: bin + CMAKE_INSTALL_PREFIX: /usr + CMAKE_UNITY_BUILD: ON + NPROC: 2 + steps: + - name: Disable Crash Dialogs + run: | + reg add ` + "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` + /v DontShowUI ` + /t REG_DWORD ` + /d 1 ` + /f + - name: Installed Packages + run: choco list -l + - name: Install Dependencies + run: choco install -y --no-progress openssl + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Build + shell: bash + run: | + ci/scripts/cpp_build.sh $(pwd) $(pwd)/build + - name: Test + shell: bash + run: ci/scripts/cpp_test.sh $(pwd) $(pwd)/build + + windows-mingw: + name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} C++ + runs-on: windows-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + mingw-n-bits: + - 32 + - 64 + env: + ARROW_BUILD_SHARED: ON + ARROW_BUILD_STATIC: OFF + ARROW_BUILD_TESTS: ON + ARROW_BUILD_TYPE: release + ARROW_DATASET: ON + ARROW_FLIGHT: ON + ARROW_GANDIVA: ON + ARROW_HDFS: OFF + ARROW_HOME: /mingw${{ matrix.mingw-n-bits }} + ARROW_JEMALLOC: OFF + ARROW_PARQUET: ON + ARROW_PYTHON: ON + ARROW_S3: ON + ARROW_USE_GLOG: OFF + ARROW_VERBOSE_THIRDPARTY_BUILD: OFF + ARROW_WITH_BROTLI: ON + ARROW_WITH_BZ2: ON + ARROW_WITH_LZ4: ON + ARROW_WITH_SNAPPY: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_ZSTD: ON + # Don't use preinstalled Boost by empty BOOST_ROOT and + # -DBoost_NO_BOOST_CMAKE=ON + BOOST_ROOT: "" + CMAKE_ARGS: >- + -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} + -DBoost_NO_BOOST_CMAKE=ON + CMAKE_UNITY_BUILD: ON + steps: + - name: Disable Crash Dialogs + run: | + reg add ` + "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` + /v DontShowUI ` + /t REG_DWORD ` + /d 1 ` + /f + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - uses: msys2/setup-msys2@v2 + with: + msystem: MINGW${{ matrix.mingw-n-bits }} + update: true + - name: Setup MSYS2 + shell: msys2 {0} + run: | + ci/scripts/msys2_setup.sh cpp + - name: Cache ccache + uses: actions/cache@v2 + with: + path: ccache + key: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} + restore-keys: cpp-ccache-mingw${{ matrix.mingw-n-bits }}- + - name: Build + shell: msys2 {0} + run: | + export CMAKE_BUILD_PARALLEL_LEVEL=$NUMBER_OF_PROCESSORS + ci/scripts/cpp_build.sh "$(pwd)" "$(pwd)/build" + - name: Download MinIO + shell: msys2 {0} + run: | + mkdir -p /usr/local/bin + wget \ + --output-document /usr/local/bin/minio.exe \ + https://dl.min.io/server/minio/release/windows-amd64/minio.exe + chmod +x /usr/local/bin/minio.exe + - name: Test + shell: msys2 {0} + run: | + python_version=$(python -c "import sys; print('.'.join(map(str, sys.version_info[0:2])))") + export PYTHONHOME="$(cygpath --windows ${MINGW_PREFIX})\lib\python${python_version}" + PYTHONPATH="${PYTHONHOME}" + PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\lib-dynload" + PYTHONPATH="${PYTHONPATH};${PYTHONHOME}\site-packages" + export PYTHONPATH + ci/scripts/cpp_test.sh "$(pwd)" "$(pwd)/build" diff --git a/src/arrow/.github/workflows/cpp_cron.yml b/src/arrow/.github/workflows/cpp_cron.yml new file mode 100644 index 000000000..c031e5961 --- /dev/null +++ b/src/arrow/.github/workflows/cpp_cron.yml @@ -0,0 +1,85 @@ + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: C++ Cron + +on: + push: + paths: + - '.github/workflows/cpp_cron.yml' + pull_request: + paths: + - '.github/workflows/cpp_cron.yml' + schedule: + - cron: | + 0 */12 * * * + +env: + ARROW_ENABLE_TIMING_TESTS: OFF + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + oss-fuzz: + name: OSS-Fuzz build check + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + ubuntu: [18.04] + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + shell: bash + run: ci/scripts/util_cleanup.sh + - name: Checkout OSS-Fuzz + uses: actions/checkout@v1 + with: + path: oss-fuzz + repository: google/oss-fuzz + ref: master + - name: Install dependencies + working-directory: ../oss-fuzz + run: | + python3 -m pip install setuptools + python3 -m pip install -r infra/ci/requirements.txt + - name: Build image + shell: bash + working-directory: ../oss-fuzz + run: | + python3 infra/helper.py build_image --pull arrow + - name: Build fuzzers + shell: bash + working-directory: ../oss-fuzz + run: | + python3 infra/helper.py build_fuzzers arrow `pwd`/../arrow + - name: Check build + shell: bash + working-directory: ../oss-fuzz + run: | + python3 infra/helper.py check_build arrow diff --git a/src/arrow/.github/workflows/csharp.yml b/src/arrow/.github/workflows/csharp.yml new file mode 100644 index 000000000..b339b8f46 --- /dev/null +++ b/src/arrow/.github/workflows/csharp.yml @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: C# + +on: + push: + paths: + - '.github/workflows/csharp.yml' + - 'ci/scripts/csharp_*' + - 'csharp/**' + pull_request: + paths: + - '.github/workflows/csharp.yml' + - 'ci/scripts/csharp_*' + - 'csharp/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + + ubuntu: + name: AMD64 Ubuntu 18.04 C# ${{ matrix.dotnet }} + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + dotnet: ['3.1.x'] + steps: + - name: Install C# + uses: actions/setup-dotnet@v1 + with: + dotnet-version: ${{ matrix.dotnet }} + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Install Source Link + shell: bash + run: dotnet tool install --global sourcelink + - name: Build + shell: bash + run: ci/scripts/csharp_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/csharp_test.sh $(pwd) + + windows: + name: AMD64 Windows 2019 18.04 C# ${{ matrix.dotnet }} + runs-on: windows-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + dotnet: ['3.1.x'] + steps: + - name: Install C# + uses: actions/setup-dotnet@v1 + with: + dotnet-version: ${{ matrix.dotnet }} + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Install Source Link + run: dotnet tool install --global sourcelink + - name: Build + shell: bash + run: ci/scripts/csharp_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/csharp_test.sh $(pwd) + + macos: + name: AMD64 MacOS 10.15 C# ${{ matrix.dotnet }} + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + dotnet: ['3.1.x'] + steps: + - name: Install C# + uses: actions/setup-dotnet@v1 + with: + dotnet-version: ${{ matrix.dotnet }} + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Install Source Link + shell: bash + run: dotnet tool install --global sourcelink + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Build + shell: bash + run: ci/scripts/csharp_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/csharp_test.sh $(pwd) diff --git a/src/arrow/.github/workflows/dev.yml b/src/arrow/.github/workflows/dev.yml new file mode 100644 index 000000000..9ef46c31f --- /dev/null +++ b/src/arrow/.github/workflows/dev.yml @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Dev + +on: + # always trigger + push: + pull_request: + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + lint: + name: Lint C++, Python, R, Docker, RAT + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + archery docker run -e GITHUB_ACTIONS=true ubuntu-lint + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push ubuntu-lint + + release: + name: Source Release and Merge Script + runs-on: ubuntu-20.04 + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + env: + GIT_AUTHOR_NAME: Github Actions + GIT_AUTHOR_EMAIL: github@actions + GIT_COMMITTER_NAME: Github Actions + GIT_COMMITTER_EMAIL: github@actions + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Install Python + uses: actions/setup-python@v1 + with: + python-version: '3.6' + - name: Install Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '2.6' + - name: Install Dependencies + shell: bash + run: | + pip install cython setuptools six pytest jira + - name: Run Release Test + shell: bash + run: | + ci/scripts/release_test.sh $(pwd) + - name: Run Merge Script Test + shell: bash + run: | + pytest -v dev/test_merge_arrow_pr.py diff --git a/src/arrow/.github/workflows/dev_pr.yml b/src/arrow/.github/workflows/dev_pr.yml new file mode 100644 index 000000000..86a5c98f8 --- /dev/null +++ b/src/arrow/.github/workflows/dev_pr.yml @@ -0,0 +1,99 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Dev PR + +on: + # TODO: Enable this when eps1lon/actions-label-merge-conflict is available. + # push: + # branches: + # - master + pull_request_target: + types: + - opened + - edited + - synchronize + +# NOTE: not using the "cancel-in-progress" feature here as the group key +# does not have enough information for linking it to a particular PR + +jobs: + process: + name: Process + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Comment JIRA link + if: | + github.event_name == 'pull_request_target' && + (github.event.action == 'opened' || + github.event.action == 'edited') + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr/link.js`); + script({github, context}); + + - name: Check title + if: | + github.event_name == 'pull_request_target' && + (github.event.action == 'opened' || + github.event.action == 'edited') + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr/title_check.js`); + script({github, context}); + + - name: Check Jira Issue + if: | + github.event_name == 'pull_request_target' && + (github.event.action == 'opened' || + github.event.action == 'edited') + uses: actions/github-script@v3 + with: + debug: true + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const script = require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/dev_pr/jira_check.js`); + script({github, context}); + + - name: Assign GitHub labels + if: | + github.event_name == 'pull_request_target' && + (github.event.action == 'opened' || + github.event.action == 'synchronize') + uses: actions/labeler@2.2.0 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + configuration-path: .github/workflows/dev_pr/labeler.yml + sync-labels: true + + # TODO: Enable this when eps1lon/actions-label-merge-conflict is available. + # - name: Checks if PR needs rebase + # if: | + # github.event_name == 'push' || + # (github.event_name == 'pull_request_target' && + # (github.event.action == 'opened' || + # github.event.action == 'synchronize')) + # uses: eps1lon/actions-label-merge-conflict@releases/2.x + # with: + # dirtyLabel: "needs-rebase" + # repoToken: "${{ secrets.GITHUB_TOKEN }}" diff --git a/src/arrow/.github/workflows/dev_pr/helpers.js b/src/arrow/.github/workflows/dev_pr/helpers.js new file mode 100644 index 000000000..d5f275d27 --- /dev/null +++ b/src/arrow/.github/workflows/dev_pr/helpers.js @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const https = require('https'); + +/** + * Given the title of a PullRequest return the ID of the JIRA issue + * @param {String} title + * @returns {String} the ID of the associated JIRA issue + */ +function detectJIRAID(title) { + if (!title) { + return null; + } + const matched = /^(WIP:?\s*)?((ARROW|PARQUET)-\d+)/.exec(title); + if (!matched) { + return null; + } + return matched[2]; +} + +/** + * Given the title of a PullRequest checks if it contains a JIRA issue ID + * @param {String} title + * @returns {Boolean} true if it starts with a JIRA ID or MINOR: + */ +function haveJIRAID(title) { + if (!title) { + return false; + } + if (title.startsWith("MINOR: ")) { + return true; + } + return /^(WIP:?\s*)?(ARROW|PARQUET)-\d+/.test(title); +} + +/** + * Retrieves information about a JIRA issue. + * @param {String} jiraID + * @returns {Object} the information about a JIRA issue. + */ +async function getJiraInfo(jiraID) { + const jiraURL = `https://issues.apache.org/jira/rest/api/2/issue/${jiraID}`; + + return new Promise((resolve) => { + https.get(jiraURL, res => { + let data = ''; + + res.on('data', chunk => { data += chunk }) + + res.on('end', () => { + resolve(JSON.parse(data)); + }) + }) + }); +} + +module.exports = { + detectJIRAID, + haveJIRAID, + getJiraInfo +};
\ No newline at end of file diff --git a/src/arrow/.github/workflows/dev_pr/jira_check.js b/src/arrow/.github/workflows/dev_pr/jira_check.js new file mode 100644 index 000000000..3c294f8c7 --- /dev/null +++ b/src/arrow/.github/workflows/dev_pr/jira_check.js @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const helpers = require("./helpers.js"); + +async function verifyJIRAIssue(github, context, pullRequestNumber, jiraID) { + const ticketInfo = await helpers.getJiraInfo(jiraID); + if(!ticketInfo["fields"]["components"].length) { + await commentMissingComponents(github, context, pullRequestNumber); + } + + if(ticketInfo["fields"]["status"]["id"] == 1) { + // "status": {"name":"Open","id":"1" + // "description":"The issue is open and ready for the assignee to start work on it.", + await commentNotStartedTicket(github, context, pullRequestNumber); + } +} + +async function commentMissingComponents(github, context, pullRequestNumber) { + const {data: comments} = await github.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + per_page: 100 + }); + + var found = false; + for(var i=0; i<comments.length; i++) { + if (comments[i].body.includes("has no components in JIRA")) { + found = true; + } + } + if (!found) { + await github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body: ":warning: Ticket **has no components in JIRA**, make sure you assign one." + }); + } +} + +async function commentNotStartedTicket(github, context, pullRequestNumber) { + const {data: comments} = await github.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + per_page: 100 + }); + + var found = false; + for(var i=0; i<comments.length; i++) { + if (comments[i].body.includes("has not been started in JIRA")) { + found = true; + } + } + if (!found) { + await github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body: ":warning: Ticket **has not been started in JIRA**, please click 'Start Progress'." + }); + } +} + +module.exports = async ({github, context}) => { + const pullRequestNumber = context.payload.number; + const title = context.payload.pull_request.title; + const jiraID = helpers.detectJIRAID(title); + if (jiraID) { + await verifyJIRAIssue(github, context, pullRequestNumber, jiraID); + } +}; diff --git a/src/arrow/.github/workflows/dev_pr/labeler.yml b/src/arrow/.github/workflows/dev_pr/labeler.yml new file mode 100644 index 000000000..8860d91f8 --- /dev/null +++ b/src/arrow/.github/workflows/dev_pr/labeler.yml @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"lang-c++": + - cpp/**/* + +lang-c-glib: + - c_glib/**/* + +lang-csharp: + - csharp/**/* + +lang-go: + - go/**/* + +lang-java: + - java/**/* + +lang-js: + - js/**/* + +lang-julia: + - julia/**/* + +lang-matlab: + - matlab/**/* + +lang-python: + - python/**/* + +lang-R: + - r/**/* + +lang-ruby: + - ruby/**/* + +flight: + - cpp/src/arrow/flight/**/* + - r/R/flight.* + - python/pyarrow/*flight.* + +gandiva: + - c_glib/gandiva-glib/**/* + - cpp/src/gandiva/**/* + - ruby/red-gandiva/**/* + - python/pyarrow/gandiva.* + +parquet: + - c_glib/parquet-glib/**/* + - cpp/src/parquet/**/* + - r/R/parquet.* + - ruby/red-parquet/**/* diff --git a/src/arrow/.github/workflows/dev_pr/link.js b/src/arrow/.github/workflows/dev_pr/link.js new file mode 100644 index 000000000..404ff4643 --- /dev/null +++ b/src/arrow/.github/workflows/dev_pr/link.js @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const helpers = require("./helpers.js"); + + +async function haveComment(github, context, pullRequestNumber, body) { + const options = { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + page: 1 + }; + while (true) { + const response = await github.issues.listComments(options); + if (response.data.some(comment => comment.body === body)) { + return true; + } + if (!/;\s*rel="next"/.test(response.headers.link || "")) { + break; + } + options.page++; + } + return false; +} + +async function commentJIRAURL(github, context, pullRequestNumber, jiraID) { + const jiraURL = `https://issues.apache.org/jira/browse/${jiraID}`; + if (await haveComment(github, context, pullRequestNumber, jiraURL)) { + return; + } + await github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body: jiraURL + }); +} + +module.exports = async ({github, context}) => { + const pullRequestNumber = context.payload.number; + const title = context.payload.pull_request.title; + const jiraID = helpers.detectJIRAID(title); + if (jiraID) { + await commentJIRAURL(github, context, pullRequestNumber, jiraID); + } +}; diff --git a/src/arrow/.github/workflows/dev_pr/title_check.js b/src/arrow/.github/workflows/dev_pr/title_check.js new file mode 100644 index 000000000..392108269 --- /dev/null +++ b/src/arrow/.github/workflows/dev_pr/title_check.js @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +const fs = require("fs"); +const helpers = require("./helpers.js"); + +async function commentOpenJIRAIssue(github, context, pullRequestNumber) { + const {data: comments} = await github.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + per_page: 1 + }); + if (comments.length > 0) { + return; + } + const commentPath = ".github/workflows/dev_pr/title_check.md"; + const comment = fs.readFileSync(commentPath).toString(); + await github.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body: comment + }); +} + +module.exports = async ({github, context}) => { + const pullRequestNumber = context.payload.number; + const title = context.payload.pull_request.title; + if (!helpers.haveJIRAID(title)) { + await commentOpenJIRAIssue(github, context, pullRequestNumber); + } +}; diff --git a/src/arrow/.github/workflows/dev_pr/title_check.md b/src/arrow/.github/workflows/dev_pr/title_check.md new file mode 100644 index 000000000..1db9fcf63 --- /dev/null +++ b/src/arrow/.github/workflows/dev_pr/title_check.md @@ -0,0 +1,37 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +Thanks for opening a pull request! + +If this is not a [minor PR](https://github.com/apache/arrow/blob/master/CONTRIBUTING.md#Minor-Fixes). Could you open an issue for this pull request on JIRA? https://issues.apache.org/jira/browse/ARROW + +Opening JIRAs ahead of time contributes to the [Openness](http://theapacheway.com/open/#:~:text=Openness%20allows%20new%20users%20the,must%20happen%20in%20the%20open.) of the Apache Arrow project. + +Then could you also rename pull request title in the following format? + + ARROW-${JIRA_ID}: [${COMPONENT}] ${SUMMARY} + +or + + MINOR: [${COMPONENT}] ${SUMMARY} + +See also: + + * [Other pull requests](https://github.com/apache/arrow/pulls/) + * [Contribution Guidelines - How to contribute patches](https://arrow.apache.org/docs/developers/contributing.html#how-to-contribute-patches) diff --git a/src/arrow/.github/workflows/go.yml b/src/arrow/.github/workflows/go.yml new file mode 100644 index 000000000..5d8034ff0 --- /dev/null +++ b/src/arrow/.github/workflows/go.yml @@ -0,0 +1,287 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Go + +on: + push: + paths: + - '.github/workflows/go.yml' + - 'ci/docker/*_go.dockerfile' + - 'ci/scripts/go_*' + - 'go/**' + pull_request: + paths: + - '.github/workflows/go.yml' + - 'ci/docker/*_go.dockerfile' + - 'ci/docker/**' + - 'ci/scripts/go_*' + - 'go/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + docker: + name: AMD64 Debian 11 Go ${{ matrix.go }} + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + go: [1.15] + env: + GO: ${{ matrix.go }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run debian-go + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push debian-go + + docker_cgo: + name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + go: [1.15] + env: + GO: ${{ matrix.go }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run debian-go-cgo + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push debian-go-cgo + + + docker_cgo_python: + name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO Python + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + go: [1.15] + env: + GO: ${{ matrix.go }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run debian-go-cgo-python + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push debian-go-cgo-python + + windows: + name: AMD64 Windows 2019 Go ${{ matrix.go }} + runs-on: windows-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + go: [1.15] + steps: + - name: Install go + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go }} + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Build + shell: bash + run: ci/scripts/go_build.sh . + - name: Test + shell: bash + run: ci/scripts/go_test.sh . + + macos: + name: AMD64 MacOS 10.15 Go ${{ matrix.go }} + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + go: [1.15] + steps: + - name: Install go + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go }} + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Build + shell: bash + run: ci/scripts/go_build.sh . + - name: Test + shell: bash + run: ci/scripts/go_test.sh . + + macos-cgo: + name: AMD64 MacOS 10.15 Go ${{ matrix.go }} - CGO + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + go: [1.15] + env: + ARROW_GO_TESTCGO: "1" + steps: + - name: Install go + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go }} + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Brew Install Arrow + shell: bash + run: brew install apache-arrow + - name: Build + shell: bash + run: ci/scripts/go_build.sh . + - name: Test + shell: bash + run: ci/scripts/go_test.sh . + + windows-mingw: + name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} CGO + runs-on: windows-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + go: [1.15] + mingw-n-bits: + #- 32 runtime handling for CGO needs 64-bit currently + - 64 + env: + ARROW_GO_TESTCGO: "1" + steps: + - name: Disable Crash Dialogs + run: | + reg add ` + "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` + /v DontShowUI ` + /t REG_DWORD ` + /d 1 ` + /f + - name: Install go + uses: actions/setup-go@v1 + with: + go-version: ${{ matrix.go }} + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - uses: msys2/setup-msys2@v2 + with: + msystem: MINGW${{ matrix.mingw-n-bits }} + update: true + - name: Setup MSYS2 + shell: msys2 {0} + run: | + ci/scripts/msys2_setup.sh cgo + - name: Update CGO Env vars + shell: msys2 {0} + run: | + echo "CGO_CPPFLAGS=-I$(cygpath --windows ${MINGW_PREFIX}/include)" >> $GITHUB_ENV + echo "CGO_LDFLAGS=-g -O2 -L$(cygpath --windows ${MINGW_PREFIX}/lib) -L$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_ENV + echo "$(cygpath --windows ${MINGW_PREFIX}/bin)" >> $GITHUB_PATH + - name: Build + shell: bash + run: ci/scripts/go_build.sh . + - name: Test + shell: bash + run: ci/scripts/go_test.sh . diff --git a/src/arrow/.github/workflows/integration.yml b/src/arrow/.github/workflows/integration.yml new file mode 100644 index 000000000..7a4deb8e3 --- /dev/null +++ b/src/arrow/.github/workflows/integration.yml @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Integration + +on: + push: + paths: + - '.github/workflows/integration.yml' + - 'ci/**' + - 'dev/archery/**' + - 'go/**' + - 'integration/**' + - 'js/**' + - 'cpp/**' + - 'java/**' + - 'format/**' + pull_request: + paths: + - '.github/workflows/integration.yml' + - 'ci/**' + - 'dev/archery/**' + - 'go/**' + - 'integration/**' + - 'js/**' + - 'cpp/**' + - 'java/**' + - 'format/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + docker: + name: AMD64 Conda Integration Test + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Checkout Arrow Rust + uses: actions/checkout@v2 + with: + repository: apache/arrow-rs + path: rust + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: conda-${{ hashFiles('cpp/**') }} + restore-keys: conda- + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run -e ARCHERY_INTEGRATION_WITH_RUST=1 conda-integration + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push conda-integration diff --git a/src/arrow/.github/workflows/java.yml b/src/arrow/.github/workflows/java.yml new file mode 100644 index 000000000..72f4df7e3 --- /dev/null +++ b/src/arrow/.github/workflows/java.yml @@ -0,0 +1,118 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Java + +on: + push: + paths: + - '.github/workflows/java.yml' + - 'ci/docker/*java*' + - 'ci/scripts/java*.sh' + - 'ci/scripts/util_*.sh' + - 'format/Flight.proto' + - 'java/**' + pull_request: + paths: + - '.github/workflows/java.yml' + - 'ci/docker/*java*' + - 'ci/scripts/java*.sh' + - 'ci/scripts/util_*.sh' + - 'format/Flight.proto' + - 'java/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + debian: + name: AMD64 Debian 9 Java JDK ${{ matrix.jdk }} Maven ${{ matrix.maven }} + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + jdk: [11] + maven: [3.6.2] + env: + JDK: ${{ matrix.jdk }} + MAVEN: ${{ matrix.maven }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + shell: bash + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: maven-${{ hashFiles('java/**') }} + restore-keys: maven- + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run debian-java + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push debian-java + + macos: + name: AMD64 MacOS 10.15 Java JDK ${{ matrix.jdk }} + runs-on: macos-latest + if: github.event_name == 'push' + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + jdk: [11] + steps: + - name: Set up Java + uses: actions/setup-java@v1 + with: + java-version: ${{ matrix.jdk }} + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Build + shell: bash + run: ci/scripts/java_build.sh $(pwd) $(pwd)/build + - name: Test + shell: bash + run: ci/scripts/java_test.sh $(pwd) $(pwd)/build diff --git a/src/arrow/.github/workflows/java_jni.yml b/src/arrow/.github/workflows/java_jni.yml new file mode 100644 index 000000000..48351f3c2 --- /dev/null +++ b/src/arrow/.github/workflows/java_jni.yml @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Java JNI + +on: + push: + paths: + - '.github/workflows/java_jni.yml' + - 'ci/docker/**' + - 'ci/scripts/cpp_build.sh' + - 'ci/scripts/java_*' + - 'cpp/**' + - 'java/**' + pull_request: + paths: + - '.github/workflows/java_jni.yml' + - 'ci/docker/**' + - 'ci/scripts/cpp_build.sh' + - 'ci/scripts/java_*' + - 'cpp/**' + - 'java/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + docker: + name: AMD64 Debian 9 Java JNI (Gandiva, Plasma, ORC, Dataset) + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 90 + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: maven-${{ hashFiles('java/**') }} + restore-keys: maven- + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: archery docker run debian-java-jni + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push debian-java-jni diff --git a/src/arrow/.github/workflows/js.yml b/src/arrow/.github/workflows/js.yml new file mode 100644 index 000000000..cbd6ce0ac --- /dev/null +++ b/src/arrow/.github/workflows/js.yml @@ -0,0 +1,124 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: NodeJS + +on: + push: + paths: + - '.github/workflows/js.yml' + - 'ci/docker/*js.dockerfile' + - 'ci/scripts/js_*' + - 'js/**' + pull_request: + paths: + - '.github/workflows/js.yml' + - 'ci/docker/*js.dockerfile' + - 'ci/scripts/js_*' + - 'js/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + docker: + name: AMD64 Debian 11 NodeJS 14 + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + archery docker run debian-js + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push debian-js + + macos: + name: AMD64 MacOS 10.15 NodeJS ${{ matrix.node }} + runs-on: macos-latest + if: github.event_name == 'push' + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + node: [14] + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Install NodeJS + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node }} + - name: Build + shell: bash + run: ci/scripts/js_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/js_test.sh $(pwd) + + windows: + name: AMD64 Windows 2019 NodeJS ${{ matrix.node }} + runs-on: windows-latest + if: github.event_name == 'push' + strategy: + fail-fast: false + matrix: + node: [14] + steps: + - name: Checkout Arrow + uses: actions/checkout@v1 + with: + submodules: true + - name: Install NodeJS + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node }} + - name: Build + shell: bash + run: ci/scripts/js_build.sh $(pwd) + - name: Test + shell: bash + run: ci/scripts/js_test.sh $(pwd) diff --git a/src/arrow/.github/workflows/julia.yml b/src/arrow/.github/workflows/julia.yml new file mode 100644 index 000000000..226ec3e6a --- /dev/null +++ b/src/arrow/.github/workflows/julia.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Julia +on: + push: + paths: + - '.github/workflows/julia.yml' + - 'julia/**' + pull_request: + paths: + - '.github/workflows/julia.yml' + - 'julia/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + test: + name: AMD64 ${{ matrix.os }} Julia ${{ matrix.version }} + env: + JULIA_NUM_THREADS: 2 + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + version: + - '1.3' + - '1' # automatically expands to the latest stable 1.x release of Julia + - 'nightly' + os: + - ubuntu-latest + - windows-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: x64 + - uses: julia-actions/julia-runtest@v1 + with: + project: julia/Arrow diff --git a/src/arrow/.github/workflows/matlab.yml b/src/arrow/.github/workflows/matlab.yml new file mode 100644 index 000000000..1ac3df602 --- /dev/null +++ b/src/arrow/.github/workflows/matlab.yml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: MATLAB + +on: + push: + paths: + - '.github/workflows/matlab.yml' + - 'ci/scripts/matlab*.sh' + - 'matlab/**' + - 'cpp/src/arrow/**' + pull_request: + paths: + - '.github/workflows/matlab.yml' + - 'ci/scripts/matlab*.sh' + - 'matlab/**' + - 'cpp/src/arrow/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + + matlab: + name: AMD64 Ubuntu 20.04 MATLAB + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Install ninja-build + run: sudo apt-get install ninja-build + - name: Install MATLAB + uses: matlab-actions/setup-matlab@v0 + - name: Build MATLAB Interface + run: ci/scripts/matlab_build.sh $(pwd) + - name: Run MATLAB Tests + env: + # libarrow.so requires a more recent version of libstdc++.so + # than is bundled with MATLAB under <matlabroot>/sys/os/glnxa64. + # Therefore, if a MEX function that depends on libarrow.so + # is executed within the MATLAB address space, runtime linking + # errors will occur. To work around this issue, we can explicitly + # force MATLAB to use the system libstdc++.so via LD_PRELOAD. + LD_PRELOAD: /usr/lib/x86_64-linux-gnu/libstdc++.so.6 + uses: matlab-actions/run-tests@v1 + with: + select-by-folder: matlab/test diff --git a/src/arrow/.github/workflows/python.yml b/src/arrow/.github/workflows/python.yml new file mode 100644 index 000000000..cce66fe71 --- /dev/null +++ b/src/arrow/.github/workflows/python.yml @@ -0,0 +1,161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Python + +on: + push: + paths: + - '.github/workflows/python.yml' + - 'ci/**' + - 'cpp/**' + - 'python/**' + pull_request: + paths: + - '.github/workflows/python.yml' + - 'ci/**' + - 'cpp/**' + - 'python/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + docker: + name: ${{ matrix.title }} + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + name: + - conda-python-3.8-nopandas + - conda-python-3.6-pandas-0.23 + - conda-python-3.7-pandas-latest + include: + - name: conda-python-3.8-nopandas + cache: conda-python-3.8 + image: conda-python + title: AMD64 Conda Python 3.8 Without Pandas + python: 3.8 + - name: conda-python-3.6-pandas-0.23 + cache: conda-python-3.6 + image: conda-python-pandas + title: AMD64 Conda Python 3.6 Pandas 0.23 + python: 3.6 + pandas: 0.23 + numpy: 1.16 + - name: conda-python-3.7-pandas-latest + cache: conda-python-3.7 + image: conda-python-pandas + title: AMD64 Conda Python 3.7 Pandas latest + python: 3.7 + pandas: latest + env: + PYTHON: ${{ matrix.python || 3.7 }} + UBUNTU: ${{ matrix.ubuntu || 18.04 }} + PANDAS: ${{ matrix.pandas || 'latest' }} + NUMPY: ${{ matrix.numpy || 'latest' }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }} + restore-keys: ${{ matrix.cache }}- + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + archery docker run ${{ matrix.image }} + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push ${{ matrix.image }} + + macos: + name: AMD64 MacOS 10.15 Python 3 + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + env: + ARROW_HOME: /usr/local + ARROW_DATASET: ON + ARROW_GANDIVA: ON + ARROW_JEMALLOC: ON + ARROW_ORC: ON + ARROW_PARQUET: ON + ARROW_PLASMA: ON + ARROW_PYTHON: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_LZ4: ON + ARROW_WITH_BZ2: ON + ARROW_WITH_ZSTD: ON + ARROW_WITH_SNAPPY: ON + ARROW_WITH_BROTLI: ON + ARROW_BUILD_TESTS: OFF + CMAKE_ARGS: "-DPython3_EXECUTABLE=/usr/local/bin/python3" + PYARROW_TEST_LARGE_MEMORY: ON + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Install Dependencies + shell: bash + run: | + rm -f /usr/local/bin/2to3 + brew update --preinstall + brew bundle --file=cpp/Brewfile + brew install coreutils + python3 -mpip install \ + -r python/requirements-build.txt \ + -r python/requirements-test.txt + - name: Build + shell: bash + run: | + export PYTHON=python3 + ci/scripts/cpp_build.sh $(pwd) $(pwd)/build + ci/scripts/python_build.sh $(pwd) $(pwd)/build + - name: Test + shell: bash + run: ci/scripts/python_test.sh $(pwd) $(pwd)/build diff --git a/src/arrow/.github/workflows/r-without-arrow.yml b/src/arrow/.github/workflows/r-without-arrow.yml new file mode 100644 index 000000000..a517cfcd1 --- /dev/null +++ b/src/arrow/.github/workflows/r-without-arrow.yml @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: R without Arrow + +on: + push: + paths: + - ".github/workflows/r-without-arrow.yml" + - "r/src/**" + pull_request: + paths: + - ".github/workflows/r-without-arrow.yml" + - "r/src/**" + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + bundled: + name: "R package without arrow" + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + env: + R_ORG: rhub + R_IMAGE: ubuntu-gcc-release + R_TAG: latest + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: ubuntu-gcc-release-r-${{ hashFiles('cpp/**') }} + restore-keys: ubuntu-gcc-release-r- + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + archery docker run -e LIBARROW_DOWNLOAD=FALSE -e LIBARROW_BUILD=FALSE -e TEST_R_WITHOUT_LIBARROW=TRUE -e NOT_CRAN=FALSE r + - name: Dump install logs + run: cat r/check/arrow.Rcheck/00install.out + if: always() + - name: Dump test logs + run: cat r/check/arrow.Rcheck/tests/testthat.Rout* + if: always() + - name: Save the test output + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-output + path: r/check/arrow.Rcheck/tests/testthat.Rout* + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push r diff --git a/src/arrow/.github/workflows/r.yml b/src/arrow/.github/workflows/r.yml new file mode 100644 index 000000000..900acdac7 --- /dev/null +++ b/src/arrow/.github/workflows/r.yml @@ -0,0 +1,371 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: R + +on: + push: + paths: + - ".github/workflows/r.yml" + - "ci/scripts/r_*.sh" + - "ci/scripts/cpp_*.sh" + - "ci/scripts/PKGBUILD" + - "ci/etc/rprofile" + - "ci/docker/**" + - "cpp/**" + - "r/**" + pull_request: + paths: + - ".github/workflows/r.yml" + - "ci/scripts/r_*.sh" + - "ci/scripts/cpp_*.sh" + - "ci/scripts/PKGBUILD" + - "ci/etc/rprofile" + - "ci/docker/**" + - "cpp/**" + - "r/**" + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + ubuntu: + name: AMD64 Ubuntu ${{ matrix.ubuntu }} R ${{ matrix.r }} + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + r: ["4.1"] + ubuntu: [20.04] + env: + R: ${{ matrix.r }} + UBUNTU: ${{ matrix.ubuntu }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/**') }} + restore-keys: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}- + - name: Check pkgdown reference sections + run: ci/scripts/r_pkgdown_check.sh + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + # Setting a non-default and non-probable Marquesas French Polynesia time + # it has both with a .45 offset and very very few people who live there. + archery docker run -e TZ=MART ubuntu-r + - name: Dump install logs + run: cat r/check/arrow.Rcheck/00install.out + if: always() + - name: Dump test logs + run: cat r/check/arrow.Rcheck/tests/testthat.Rout* + if: always() + - name: Save the test output + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-output + path: r/check/arrow.Rcheck/tests/testthat.Rout* + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push ubuntu-r + + bundled: + name: "${{ matrix.config.org }}/${{ matrix.config.image }}:${{ matrix.config.tag }}" + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + config: + - { org: "rstudio", image: "r-base", tag: "4.0-centos7" } + - { org: "rhub", image: "debian-gcc-devel", tag: "latest" } + env: + R_ORG: ${{ matrix.config.org }} + R_IMAGE: ${{ matrix.config.image }} + R_TAG: ${{ matrix.config.tag }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: ${{ matrix.config.image }}-r-${{ hashFiles('cpp/**') }} + restore-keys: ${{ matrix.config.image }}-r- + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + # Don't set a TZ here to test that case. These builds will have the following warning in them: + # System has not been booted with systemd as init system (PID 1). Can't operate. + # Failed to connect to bus: Host is down + archery docker run -e TZ="" r + - name: Dump install logs + run: cat r/check/arrow.Rcheck/00install.out + if: always() + - name: Dump test logs + run: cat r/check/arrow.Rcheck/tests/testthat.Rout* + if: always() + - name: Save the test output + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-output + path: r/check/arrow.Rcheck/tests/testthat.Rout* + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + run: archery docker push r + + windows-cpp: + name: AMD64 Windows C++ RTools ${{ matrix.config.rtools }} ${{ matrix.config.arch }} + runs-on: windows-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + config: + - { rtools: 35, arch: 'mingw32' } + - { rtools: 35, arch: 'mingw64' } + - { rtools: 40, arch: 'mingw32' } + - { rtools: 40, arch: 'mingw64' } + - { rtools: 40, arch: 'ucrt64' } + steps: + - run: git config --global core.autocrlf false + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Setup ccache + shell: bash + run: | + ci/scripts/ccache_setup.sh + echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV + # We must enable actions/cache before r-lib/actions/setup-r to ensure + # using system tar instead of tar provided by Rtools. + # We can use tar provided by Rtools when we drop support for Rtools 3.5. + # Because Rtools 4.0 or later has zstd. actions/cache requires zstd + # when tar is GNU tar. + - name: Cache ccache + uses: actions/cache@v2 + with: + path: ccache + key: r-${{ matrix.config.rtools }}-ccache-mingw-${{ hashFiles('cpp/**') }} + restore-keys: r-${{ matrix.config.rtools }}-ccache-mingw- + # We use the makepkg-mingw setup that is included in rtools40 even when + # we use the rtools35 compilers, so we always install R 4.0/Rtools40 + - uses: r-lib/actions/setup-r@master + with: + r-version: "4.1" + rtools-version: 40 + Ncpus: 2 + - uses: r-lib/actions/setup-r@master + if: ${{ matrix.config.rtools == 35 }} + with: + rtools-version: 35 + r-version: "3.6" + Ncpus: 2 + - name: Build Arrow C++ + shell: bash + env: + RTOOLS_VERSION: ${{ matrix.config.rtools }} + MINGW_ARCH: ${{ matrix.config.arch }} + run: ci/scripts/r_windows_build.sh + - name: Rename libarrow.zip + # So that they're unique when multiple are downloaded in the next step + shell: bash + run: mv libarrow.zip libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip + - uses: actions/upload-artifact@v1 + with: + name: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip + path: libarrow-rtools${{ matrix.config.rtools }}-${{ matrix.config.arch }}.zip + # We can remove this when we drop support for Rtools 3.5. + - name: Ensure using system tar in actions/cache + run: | + Write-Output "${Env:windir}\System32" | ` + Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + windows-r: + needs: [windows-cpp] + name: AMD64 Windows R RTools ${{ matrix.rtools }} + runs-on: windows-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + rtools: [35, 40] + env: + ARROW_R_CXXFLAGS: "-Werror" + _R_CHECK_TESTS_NLINES_: 0 + steps: + - run: git config --global core.autocrlf false + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - run: mkdir r/windows + - name: Download artifacts + if: ${{ matrix.rtools == 35 }} + uses: actions/download-artifact@v2 + with: + name: libarrow-rtools35-mingw32.zip + path: r/windows + - name: Download artifacts + if: ${{ matrix.rtools == 35 }} + uses: actions/download-artifact@v2 + with: + name: libarrow-rtools35-mingw64.zip + path: r/windows + - name: Download artifacts + if: ${{ matrix.rtools == 40 }} + uses: actions/download-artifact@v2 + with: + name: libarrow-rtools40-mingw32.zip + path: r/windows + - name: Download artifacts + if: ${{ matrix.rtools == 40 }} + uses: actions/download-artifact@v2 + with: + name: libarrow-rtools40-mingw64.zip + path: r/windows + - name: Download artifacts + if: ${{ matrix.rtools == 40 }} + uses: actions/download-artifact@v2 + with: + name: libarrow-rtools40-ucrt64.zip + path: r/windows + - name: Unzip and rezip libarrows + shell: bash + run: | + cd r/windows + ls *.zip | xargs -n 1 unzip -uo + rm -rf *.zip + - name: Setup ccache + shell: bash + run: | + ci/scripts/ccache_setup.sh + echo "CCACHE_DIR=$(cygpath --absolute --windows ccache)" >> $GITHUB_ENV + # We must enable actions/cache before r-lib/actions/setup-r to ensure + # using system tar instead of tar provided by Rtools. + # We can use tar provided by Rtools when we drop support for Rtools 3.5. + # Because Rtools 4.0 or later has zstd. actions/cache requires zstd + # when tar is GNU tar. + - name: Cache ccache + uses: actions/cache@v2 + with: + path: ccache + key: r-${{ matrix.rtools }}-ccache-mingw-${{ hashFiles('cpp/**') }} + restore-keys: r-${{ matrix.rtools }}-ccache-mingw- + - uses: r-lib/actions/setup-r@master + if: ${{ matrix.rtools == 40 }} + with: + r-version: "4.1" + rtools-version: 40 + Ncpus: 2 + - uses: r-lib/actions/setup-r@master + if: ${{ matrix.rtools == 35 }} + with: + rtools-version: 35 + r-version: "3.6" + Ncpus: 2 + - name: Make R tests verbose + # If you get a segfault/mysterious test Execution halted, + # make this `true` to see where it dies. + if: false + shell: cmd + run: | + cd r/tests + sed -i.bak -E -e 's/"arrow"/"arrow", reporter = "location"/' testthat.R + rm -f testthat.R.bak + - name: Install R package dependencies + shell: Rscript {0} + run: | + options(pkgType="win.binary") + install.packages(c("remotes", "rcmdcheck")) + remotes::install_deps("r", dependencies = TRUE) + - name: Check + shell: Rscript {0} + run: | + # Because we do R CMD build and r/windows is in .Rbuildignore, + # assemble the libarrow.zip file and pass it as an env var + setwd("r/windows") + zip("libarrow.zip", ".") + setwd("..") + + Sys.setenv( + RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "r", "windows", "libarrow.zip"), + MAKEFLAGS = paste0("-j", parallel::detectCores()), + ARROW_R_DEV = TRUE, + "_R_CHECK_FORCE_SUGGESTS_" = FALSE + ) + rcmdcheck::rcmdcheck(".", + build_args = '--no-build-vignettes', + args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'), + error_on = 'warning', + check_dir = 'check', + timeout = 3600 + ) + - name: Dump install logs + shell: cmd + run: cat r/check/arrow.Rcheck/00install.out + if: always() + # We can remove this when we drop support for Rtools 3.5. + - name: Ensure using system tar in actions/cache + run: | + Write-Output "${Env:windir}\System32" | ` + Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append diff --git a/src/arrow/.github/workflows/ruby.yml b/src/arrow/.github/workflows/ruby.yml new file mode 100644 index 000000000..067b40aef --- /dev/null +++ b/src/arrow/.github/workflows/ruby.yml @@ -0,0 +1,299 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: C GLib & Ruby + +on: + push: + paths: + - '.github/workflows/ruby.yml' + - 'ci/docker/**' + - 'ci/scripts/c_glib_*' + - 'ci/scripts/cpp_*' + - 'ci/scripts/msys2_*' + - 'ci/scripts/ruby_*' + - 'ci/scripts/util_*' + - 'c_glib/**' + - 'cpp/**' + - 'ruby/**' + pull_request: + paths: + - '.github/workflows/ruby.yml' + - 'ci/docker/**' + - 'ci/scripts/c_glib_*' + - 'ci/scripts/cpp_*' + - 'ci/scripts/msys2_*' + - 'ci/scripts/ruby_*' + - 'ci/scripts/util_*' + - 'c_glib/**' + - 'cpp/**' + - 'ruby/**' + +concurrency: + group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +env: + DOCKER_VOLUME_PREFIX: ".docker/" + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + +jobs: + + ubuntu: + name: AMD64 Ubuntu ${{ matrix.ubuntu }} GLib & Ruby + runs-on: ubuntu-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + ubuntu: + - 20.04 + env: + UBUNTU: ${{ matrix.ubuntu }} + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Free Up Disk Space + shell: bash + run: ci/scripts/util_cleanup.sh + - name: Cache Docker Volumes + uses: actions/cache@v2 + with: + path: .docker + key: ubuntu-${{ matrix.ubuntu }}-ruby-${{ hashFiles('cpp/**') }} + restore-keys: ubuntu-${{ matrix.ubuntu }}-ruby- + - name: Setup Python + uses: actions/setup-python@v1 + with: + python-version: 3.8 + - name: Setup Archery + run: pip install -e dev/archery[docker] + - name: Execute Docker Build + run: | + sudo sysctl -w kernel.core_pattern="core.%e.%p" + ulimit -c unlimited + archery docker run \ + -e ARROW_FLIGHT=ON \ + -e Protobuf_SOURCE=BUNDLED \ + -e gRPC_SOURCE=BUNDLED \ + ubuntu-ruby + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'apache/arrow' + continue-on-error: true + shell: bash + run: archery docker push ubuntu-ruby + + macos: + name: AMD64 MacOS 10.15 GLib & Ruby + runs-on: macos-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 40 + strategy: + fail-fast: false + env: + ARROW_BUILD_TESTS: OFF + ARROW_FLIGHT: ON + ARROW_GANDIVA: ON + ARROW_GLIB_DEVELOPMENT_MODE: true + ARROW_GLIB_GTK_DOC: true + ARROW_HOME: /usr/local + ARROW_JEMALLOC: OFF + ARROW_ORC: OFF + ARROW_PARQUET: ON + ARROW_WITH_BROTLI: ON + ARROW_WITH_LZ4: ON + ARROW_WITH_SNAPPY: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_ZSTD: ON + XML_CATALOG_FILES: /usr/local/etc/xml/catalog + steps: + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Install Homebrew Dependencies + shell: bash + run: | + rm -f /usr/local/bin/2to3 + brew update --preinstall + brew bundle --file=cpp/Brewfile + brew bundle --file=c_glib/Brewfile + - name: Install Ruby Dependencies + run: | + export MAKEFLAGS="-j$(sysctl -n hw.ncpu)" + bundle install --gemfile c_glib/Gemfile + bundle install --gemfile ruby/Gemfile + for ruby_package_gemfile in ruby/*/Gemfile; do \ + bundle install --gemfile ${ruby_package_gemfile} + done + - name: Setup ccache + run: | + ci/scripts/ccache_setup.sh + - name: ccache info + id: ccache-info + run: | + echo "::set-output name=cache-dir::$(ccache --get-config cache_dir)" + - name: Cache ccache + uses: actions/cache@v2 + with: + path: ${{ steps.ccache-info.outputs.cache-dir }} + key: ruby-ccache-macos-${{ hashFiles('cpp/**') }} + restore-keys: ruby-ccache-macos- + - name: Build C++ + run: | + ci/scripts/cpp_build.sh $(pwd) $(pwd)/build + - name: Build GLib + run: | + ci/scripts/c_glib_build.sh $(pwd) $(pwd)/build + - name: Test GLib + shell: bash + run: ci/scripts/c_glib_test.sh $(pwd) $(pwd)/build + - name: Test Ruby + shell: bash + run: ci/scripts/ruby_test.sh $(pwd) $(pwd)/build + + windows: + name: AMD64 Windows MinGW ${{ matrix.mingw-n-bits }} GLib & Ruby + runs-on: windows-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + mingw-n-bits: + - 64 + ruby-version: + - "3.0" + env: + ARROW_BUILD_SHARED: ON + ARROW_BUILD_STATIC: OFF + ARROW_BUILD_TESTS: OFF + ARROW_BUILD_TYPE: release + ARROW_FLIGHT: ON + ARROW_GANDIVA: ON + ARROW_HDFS: OFF + ARROW_HOME: /mingw${{ matrix.mingw-n-bits }} + ARROW_JEMALLOC: OFF + ARROW_PARQUET: ON + ARROW_PYTHON: OFF + ARROW_S3: ON + ARROW_USE_GLOG: OFF + ARROW_WITH_BROTLI: ON + ARROW_WITH_BZ2: ON + ARROW_WITH_LZ4: ON + ARROW_WITH_SNAPPY: ON + ARROW_WITH_ZLIB: ON + ARROW_WITH_ZSTD: ON + # Don't use preinstalled Boost by empty BOOST_ROOT and + # -DBoost_NO_BOOST_CMAKE=ON + BOOST_ROOT: "" + CMAKE_ARGS: >- + -DARROW_PACKAGE_PREFIX=/mingw${{ matrix.mingw-n-bits }} + -DBoost_NO_BOOST_CMAKE=ON + CMAKE_UNITY_BUILD: ON + steps: + - name: Disable Crash Dialogs + run: | + reg add ` + "HKCU\SOFTWARE\Microsoft\Windows\Windows Error Reporting" ` + /v DontShowUI ` + /t REG_DWORD ` + /d 1 ` + /f + - name: Checkout Arrow + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Fetch Submodules and Tags + shell: bash + run: ci/scripts/util_checkout.sh + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + - name: Upgrade MSYS2 + run: | + ridk exec bash ci\scripts\msys2_system_upgrade.sh + taskkill /F /FI "MODULES eq msys-2.0.dll" + - name: Clean MSYS2 + run: | + ridk exec bash ci\scripts\msys2_system_clean.sh + - name: Setup MSYS2 + run: | + ridk exec bash ci\scripts\msys2_setup.sh ruby + - name: Cache ccache + uses: actions/cache@v2 + with: + path: ccache + key: ruby-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }} + restore-keys: ruby-ccache-mingw${{ matrix.mingw-n-bits }}- + - name: Build C++ + run: | + $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS + $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" + $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" + $ErrorActionPreference = "Continue" + ridk exec bash ci\scripts\cpp_build.sh "${source_dir}" "${build_dir}" + - name: Build GLib + run: | + $Env:CMAKE_BUILD_PARALLEL_LEVEL = $Env:NUMBER_OF_PROCESSORS + $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" + $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" + $ErrorActionPreference = "Continue" + ridk exec bash ci\scripts\c_glib_build.sh "${source_dir}" "${build_dir}" + - name: RubyGems info + id: rubygems-info + run: | + Write-Output "::set-output name=gem-dir::$(ridk exec gem env gemdir)" + - name: Cache RubyGems + uses: actions/cache@v2 + with: + path: ${{ steps.rubygems-info.outputs.gem-dir }} + key: ruby-rubygems-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('**/Gemfile', 'ruby/*/*.gemspec') }} + restore-keys: ruby-rubygems-mingw${{ matrix.mingw-n-bits }}- + - name: Install test dependencies + run: | + bundle install --gemfile c_glib\Gemfile + bundle install --gemfile ruby\Gemfile + Get-ChildItem ruby\*\Gemfile | ` + ForEach-Object {bundle install --gemfile $_} + - name: Test GLib + run: | + $source_dir = "$(ridk exec cygpath --unix "$(Get-Location)")" + $build_dir = "$(ridk exec cygpath --unix "$(Get-Location)\build")" + $ErrorActionPreference = "Continue" + ridk exec bash ci\scripts\c_glib_test.sh "${source_dir}" "${build_dir}" + - name: Test Ruby + run: | + $Env:PKG_CONFIG_PATH = ` + "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/pkgconfig")" + $Env:GI_TYPELIB_PATH = ` + "$(ridk exec cygpath --absolute --windows "${Env:ARROW_HOME}/lib/girepository-1.0")" + $Env:RUBYOPTS = "-rdevkit" + $Env:MAKE = "ridk exec make" + $ErrorActionPreference = "Continue" + rake -f ruby\Rakefile |