diff options
Diffstat (limited to 'src/arrow/dev/tasks')
207 files changed, 12511 insertions, 0 deletions
diff --git a/src/arrow/dev/tasks/README.md b/src/arrow/dev/tasks/README.md new file mode 100644 index 000000000..1af9739db --- /dev/null +++ b/src/arrow/dev/tasks/README.md @@ -0,0 +1,19 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +See the usage guide under the [documentation page](../../docs/source/developers/crossbow.rst) diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml new file mode 100644 index 000000000..dfc87c80b --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.6.____cpython.yaml @@ -0,0 +1,70 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-cuda:10.2 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.6.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml new file mode 100644 index 000000000..3416b952c --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.7.____cpython.yaml @@ -0,0 +1,70 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-cuda:10.2 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.7.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml new file mode 100644 index 000000000..f819ba722 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.17python3.8.____cpython.yaml @@ -0,0 +1,70 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-cuda:10.2 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.8.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml new file mode 100644 index 000000000..3e2e0ef51 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_version10.2numpy1.19python3.9.____cpython.yaml @@ -0,0 +1,70 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- '10.2' +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-cuda:10.2 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.19' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml new file mode 100644 index 000000000..3aba0f129 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml @@ -0,0 +1,70 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.6.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml new file mode 100644 index 000000000..ff26bc521 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml @@ -0,0 +1,70 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.7.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml new file mode 100644 index 000000000..5703aba68 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml @@ -0,0 +1,70 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.8.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml new file mode 100644 index 000000000..8ff58d717 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml @@ -0,0 +1,70 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.19' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cuda_compiler_version + - cdt_name + - docker_image +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.17python3.6.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.17python3.6.____cpython.yaml new file mode 100644 index 000000000..5bb4381fe --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.17python3.6.____cpython.yaml @@ -0,0 +1,69 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.6.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-aarch64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.17python3.7.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.17python3.7.____cpython.yaml new file mode 100644 index 000000000..2b1715d58 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.17python3.7.____cpython.yaml @@ -0,0 +1,69 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.7.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-aarch64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.17python3.8.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.17python3.8.____cpython.yaml new file mode 100644 index 000000000..5a0e7313e --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.17python3.8.____cpython.yaml @@ -0,0 +1,69 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.8.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-aarch64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.19python3.9.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.19python3.9.____cpython.yaml new file mode 100644 index 000000000..16ace00bd --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/linux_aarch64_numpy1.19python3.9.____cpython.yaml @@ -0,0 +1,69 @@ +BUILD: +- aarch64-conda_cos7-linux-gnu +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_arch: +- aarch64 +cdt_name: +- cos7 +channel_sources: +- conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.19' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- linux-aarch64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml new file mode 100644 index 000000000..0be59fe1a --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.6.____cpython.yaml @@ -0,0 +1,65 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +macos_machine: +- x86_64-apple-darwin13.4.0 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.6.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- osx-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml new file mode 100644 index 000000000..d2c046ab2 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.7.____cpython.yaml @@ -0,0 +1,65 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +macos_machine: +- x86_64-apple-darwin13.4.0 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.7.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- osx-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml new file mode 100644 index 000000000..43f634454 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.17python3.8.____cpython.yaml @@ -0,0 +1,65 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +macos_machine: +- x86_64-apple-darwin13.4.0 +numpy: +- '1.17' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.8.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- osx-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml new file mode 100644 index 000000000..7cc730f9b --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_64_numpy1.19python3.9.____cpython.yaml @@ -0,0 +1,65 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +macos_machine: +- x86_64-apple-darwin13.4.0 +numpy: +- '1.19' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- osx-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml new file mode 100644 index 000000000..e5f8e2ba2 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.8.____cpython.yaml @@ -0,0 +1,65 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge/label/rust_dev,conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +macos_machine: +- arm64-apple-darwin20.0.0 +numpy: +- '1.19' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.8.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- osx-arm64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml new file mode 100644 index 000000000..cd3eca6d2 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/osx_arm64_python3.9.____cpython.yaml @@ -0,0 +1,65 @@ +MACOSX_DEPLOYMENT_TARGET: +- '11.0' +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge/label/rust_dev,conda-forge +channel_targets: +- conda-forge main +cuda_compiler_version: +- None +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +macos_machine: +- arm64-apple-darwin20.0.0 +numpy: +- '1.19' +orc: +- 1.6.8 +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- osx-arm64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - python + - numpy +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml new file mode 100644 index 000000000..dfdfae966 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.0.yaml @@ -0,0 +1,29 @@ +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.0' +target_platform: +- linux-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cdt_name + - docker_image diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml new file mode 100644 index 000000000..c5f455c19 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/linux_64_r_base4.1.yaml @@ -0,0 +1,29 @@ +c_compiler: +- gcc +c_compiler_version: +- '9' +cdt_name: +- cos6 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- gxx +cxx_compiler_version: +- '9' +docker_image: +- quay.io/condaforge/linux-anvil-comp7 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- linux-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version +- - cdt_name + - docker_image diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml new file mode 100644 index 000000000..08bb81d08 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.0.yaml @@ -0,0 +1,27 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +macos_machine: +- x86_64-apple-darwin13.4.0 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.0' +target_platform: +- osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml new file mode 100644 index 000000000..9974c6638 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/osx_64_r_base4.1.yaml @@ -0,0 +1,27 @@ +MACOSX_DEPLOYMENT_TARGET: +- '10.9' +c_compiler: +- clang +c_compiler_version: +- '11' +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cxx_compiler: +- clangxx +cxx_compiler_version: +- '11' +macos_machine: +- x86_64-apple-darwin13.4.0 +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- osx-64 +zip_keys: +- - c_compiler_version + - cxx_compiler_version diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml new file mode 100644 index 000000000..02c2a7075 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.0.yaml @@ -0,0 +1,12 @@ +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.0' +target_platform: +- win-64 diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.1.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.1.yaml new file mode 100644 index 000000000..2fe9ad314 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/r/win_64_r_base4.1.yaml @@ -0,0 +1,12 @@ +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +pin_run_as_build: + r-base: + min_pin: x.x + max_pin: x.x +r_base: +- '4.1' +target_platform: +- win-64 diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml new file mode 100644 index 000000000..8d4e25167 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython.yaml @@ -0,0 +1,55 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- vs2017 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- vs2017 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.6.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- win-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml new file mode 100644 index 000000000..8da4a8380 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython.yaml @@ -0,0 +1,55 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- vs2017 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- vs2017 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.7.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- win-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml new file mode 100644 index 000000000..1980e1be3 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython.yaml @@ -0,0 +1,55 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- vs2017 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- vs2017 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.17' +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.8.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- win-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml b/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml new file mode 100644 index 000000000..1106037d3 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.ci_support/win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython.yaml @@ -0,0 +1,55 @@ +aws_sdk_cpp: +- 1.8.186 +bzip2: +- '1' +c_compiler: +- vs2017 +channel_sources: +- conda-forge,defaults +channel_targets: +- conda-forge main +cuda_compiler: +- nvcc +cuda_compiler_version: +- None +cxx_compiler: +- vs2017 +gflags: +- '2.2' +glog: +- '0.5' +grpc_cpp: +- '1.38' +libprotobuf: +- '3.16' +lz4_c: +- 1.9.3 +numpy: +- '1.19' +pin_run_as_build: + bzip2: + max_pin: x + lz4-c: + max_pin: x.x.x + python: + min_pin: x.x + max_pin: x.x + zlib: + max_pin: x.x +python: +- 3.9.* *_cpython +re2: +- 2021.06.01 +snappy: +- '1' +target_platform: +- win-64 +thrift_cpp: +- 0.14.2 +zip_keys: +- - numpy + - python +zlib: +- '1.2' +zstd: +- '1.5' diff --git a/src/arrow/dev/tasks/conda-recipes/.scripts/logging_utils.sh b/src/arrow/dev/tasks/conda-recipes/.scripts/logging_utils.sh new file mode 100644 index 000000000..a53ef3f2c --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/.scripts/logging_utils.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Provide a unified interface for the different logging +# utilities CI providers offer. If unavailable, provide +# a compatible fallback (e.g. bare `echo xxxxxx`). + +function startgroup { + # Start a foldable group of log lines + # Pass a single argument, quoted + case ${CI:-} in + azure ) + echo "##[group]$1";; + travis ) + echo "$1" + echo -en 'travis_fold:start:'"${1// /}"'\\r';; + * ) + echo "$1";; + esac +} + +function endgroup { + # End a foldable group of log lines + # Pass a single argument, quoted + case ${CI:-} in + azure ) + echo "##[endgroup]";; + travis ) + echo -en 'travis_fold:end:'"${1// /}"'\\r';; + esac +} diff --git a/src/arrow/dev/tasks/conda-recipes/README.md b/src/arrow/dev/tasks/conda-recipes/README.md new file mode 100644 index 000000000..39f82f1b0 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/README.md @@ -0,0 +1,67 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# Conda Forge recipes + +This directory must be migrated periodically with the upstrem updates of +[arrow-cpp-feedstock][arrow-cpp-feedstock], +[parquet-cpp-feedstock][parquet-cpp-feedstock]. +conda-forge repositories because of multiple vendored files. + +## Keeping the recipes synchronized + +The recipes here are tested on nightly basis, so they follow the development +versions of arrow instead of the upstream recipes, which are suitable for the +latest releases. + +### Backporting from the upstream feedstocks + +In most of the cases these recipes are more accurate, then the upstream +feedstocks. Although the upstream feedstocks regularly receive automatic updates +by the conda-forge team so we need to backport those changes to the crossbow +recipes. Most of these updates are touching the version pinning files +(under `.ci_support`) and other CI related configuration files. + +Because all three recipes must be built in the same continuous integration +job prefer porting from the [arrpw-cpp feedstock][arrow-cpp-feedstock]. + +#### Updating the variants: + +Copy the configuration files from `arrow-cpp-feedstock/.ci_support` to the +`.ci_support` folder. + +#### Updating the CI configurations: + +The `.azure-pipelines/azure-pipelines-[linux|osx|win].yml` should be ported +to the local counterparts under `.azure-pipelines` with keeping the crossbow +related parts (the cloning of arrow and the jinja templated variables) and +moving the matrix definitions like [this][matrix-definition] to the crossbow +[tasks.yml][../tasks.yml] config file. + + +### Porting recipes from crossbow to the upstream feedstocks + +Theoretically these recipes should be up to date with the actual version of +Arrow, so during the release procedure the content of these recipes should be +copied to the upstream feedstocks. + + +[arrow-cpp-feedstock]: https://github.com/conda-forge/arrow-cpp-feedstock +[parquet-cpp-feedstock]: https://github.com/conda-forge/parquet-cpp-feedstock +[matrix-definition]: https://github.com/conda-forge/arrow-cpp-feedstock/blob/master/.azure-pipelines/azure-pipelines-linux.yml#L12 diff --git a/src/arrow/dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt new file mode 100644 index 000000000..461398bab --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/LLVM_LICENSE.txt @@ -0,0 +1,68 @@ +============================================================================== +LLVM Release License +============================================================================== +University of Illinois/NCSA +Open Source License + +Copyright (c) 2003-2018 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== +Copyrights and Licenses for Third Party Software Distributed with LLVM: +============================================================================== +The LLVM software contains code written by third parties. Such software will +have its own individual LICENSE.TXT file in the directory in which it appears. +This file will describe the copyrights, license, and restrictions which apply +to that code. + +The disclaimer of warranty in the University of Illinois Open Source License +applies to all code in the LLVM Distribution, and nothing in any of the +other licenses gives permission to use the names of the LLVM Team or the +University of Illinois to endorse or promote products derived from this +Software. + +The following pieces of software have additional or alternate copyrights, +licenses, and/or restrictions: + +Program Directory +------- --------- +Google Test llvm/utils/unittest/googletest +OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} +pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} +ARM contributions llvm/lib/Target/ARM/LICENSE.TXT +md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h diff --git a/src/arrow/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat new file mode 100644 index 000000000..0527356f7 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/bld-arrow.bat @@ -0,0 +1,55 @@ +@echo on + +mkdir "%SRC_DIR%"\cpp\build +pushd "%SRC_DIR%"\cpp\build + +:: Enable CUDA support +if "%cuda_compiler_version%"=="None" ( + set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=OFF" +) else ( + REM this should move to nvcc-feedstock + set "CUDA_PATH=%CUDA_PATH:\=/%" + set "CUDA_HOME=%CUDA_HOME:\=/%" + + set "EXTRA_CMAKE_ARGS=-DARROW_CUDA=ON" +) + +cmake -G "Ninja" ^ + -DBUILD_SHARED_LIBS=ON ^ + -DCMAKE_INSTALL_PREFIX="%LIBRARY_PREFIX%" ^ + -DARROW_DEPENDENCY_SOURCE=SYSTEM ^ + -DARROW_PACKAGE_PREFIX="%LIBRARY_PREFIX%" ^ + -DLLVM_TOOLS_BINARY_DIR="%LIBRARY_BIN%" ^ + -DPython3_EXECUTABLE="%PYTHON%" ^ + -DARROW_WITH_BZ2:BOOL=ON ^ + -DARROW_WITH_ZLIB:BOOL=ON ^ + -DARROW_WITH_ZSTD:BOOL=ON ^ + -DARROW_WITH_LZ4:BOOL=ON ^ + -DARROW_WITH_SNAPPY:BOOL=ON ^ + -DARROW_WITH_BROTLI:BOOL=ON ^ + -DARROW_BOOST_USE_SHARED:BOOL=ON ^ + -DARROW_BUILD_TESTS:BOOL=OFF ^ + -DARROW_BUILD_UTILITIES:BOOL=OFF ^ + -DARROW_BUILD_STATIC:BOOL=OFF ^ + -DCMAKE_BUILD_TYPE=release ^ + -DARROW_SSE42:BOOL=OFF ^ + -DARROW_PYTHON:BOOL=ON ^ + -DARROW_MIMALLOC:BOOL=ON ^ + -DARROW_DATASET:BOOL=ON ^ + -DARROW_FLIGHT:BOOL=ON ^ + -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS:BOOL=ON ^ + -DARROW_HDFS:BOOL=ON ^ + -DARROW_PARQUET:BOOL=ON ^ + -DARROW_GANDIVA:BOOL=ON ^ + -DARROW_ORC:BOOL=ON ^ + -DARROW_S3:BOOL=ON ^ + -DBoost_NO_BOOST_CMAKE=ON ^ + -DCMAKE_UNITY_BUILD=ON ^ + %EXTRA_CMAKE_ARGS% ^ + .. +if errorlevel 1 exit 1 + +cmake --build . --target install --config Release +if errorlevel 1 exit 1 + +popd diff --git a/src/arrow/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat new file mode 100644 index 000000000..89cec3710 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/bld-pyarrow.bat @@ -0,0 +1,44 @@ +@echo on +pushd "%SRC_DIR%"\python + +@rem the symlinks for cmake modules don't work here +@rem NOTE: In contrast to conda-forge, they work here as we clone from git. +@rem del cmake_modules\BuildUtils.cmake +@rem del cmake_modules\SetupCxxFlags.cmake +@rem del cmake_modules\CompilerInfo.cmake +@rem del cmake_modules\FindNumPy.cmake +@rem del cmake_modules\FindPythonLibsNew.cmake +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\BuildUtils.cmake" cmake_modules\ +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\SetupCxxFlags.cmake" cmake_modules\ +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\CompilerInfo.cmake" cmake_modules\ +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindNumPy.cmake" cmake_modules\ +@rem copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\ + +SET ARROW_HOME=%LIBRARY_PREFIX% +SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION% +SET PYARROW_BUILD_TYPE=release +SET PYARROW_WITH_S3=1 +SET PYARROW_WITH_HDFS=1 +SET PYARROW_WITH_DATASET=1 +SET PYARROW_WITH_FLIGHT=1 +SET PYARROW_WITH_GANDIVA=1 +SET PYARROW_WITH_PARQUET=1 +SET PYARROW_CMAKE_GENERATOR=Ninja + +:: Enable CUDA support +if "%cuda_compiler_version%"=="None" ( + set "PYARROW_WITH_CUDA=0" +) else ( + set "PYARROW_WITH_CUDA=1" +) + +%PYTHON% setup.py ^ + build_ext ^ + install --single-version-externally-managed ^ + --record=record.txt +if errorlevel 1 exit 1 +popd + +if [%PKG_NAME%] == [pyarrow] ( + rd /s /q %SP_DIR%\pyarrow\tests +) diff --git a/src/arrow/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh new file mode 100644 index 000000000..9e4c02c5c --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/build-arrow.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +set -e +set -x + +mkdir cpp/build +pushd cpp/build + +EXTRA_CMAKE_ARGS="" + +# Include g++'s system headers +if [ "$(uname)" == "Linux" ]; then + SYSTEM_INCLUDES=$(echo | ${CXX} -E -Wp,-v -xc++ - 2>&1 | grep '^ ' | awk '{print "-isystem;" substr($1, 1)}' | tr '\n' ';') + EXTRA_CMAKE_ARGS=" -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INCLUDES}" +fi + +# Enable CUDA support +if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]] +then + if [[ -z "${CUDA_HOME+x}" ]] + then + echo "cuda_compiler_version=${cuda_compiler_version} CUDA_HOME=$CUDA_HOME" + CUDA_GDB_EXECUTABLE=$(which cuda-gdb || exit 0) + if [[ -n "$CUDA_GDB_EXECUTABLE" ]] + then + CUDA_HOME=$(dirname $(dirname $CUDA_GDB_EXECUTABLE)) + else + echo "Cannot determine CUDA_HOME: cuda-gdb not in PATH" + return 1 + fi + fi + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=${CUDA_HOME} -DCMAKE_LIBRARY_PATH=${CUDA_HOME}/lib64/stubs" +else + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" +fi + +if [[ "${target_platform}" == "osx-arm64" ]]; then + # We need llvm 11+ support in Arrow for this + # Tell jemalloc to support 16K page size on apple arm64 silicon + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=OFF -DARROW_JEMALLOC_LG_PAGE=14" + sed -ie "s;protoc-gen-grpc.*$;protoc-gen-grpc=${BUILD_PREFIX}/bin/grpc_cpp_plugin\";g" ../src/arrow/flight/CMakeLists.txt +elif [[ "${target_platform}" == "linux-aarch64" ]]; then + # Tell jemalloc to support both 4k and 64k page arm64 systems + # See https://github.com/apache/arrow/pull/10940 + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=ON -DARROW_JEMALLOC_LG_PAGE=16" +else + EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=ON" +fi + +cmake \ + -DARROW_BOOST_USE_SHARED=ON \ + -DARROW_BUILD_BENCHMARKS=OFF \ + -DARROW_BUILD_STATIC=OFF \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_BUILD_UTILITIES=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DARROW_DATASET=ON \ + -DARROW_DEPENDENCY_SOURCE=SYSTEM \ + -DARROW_FLIGHT=ON \ + -DARROW_FLIGHT_REQUIRE_TLSCREDENTIALSOPTIONS=ON \ + -DARROW_HDFS=ON \ + -DARROW_JEMALLOC=ON \ + -DARROW_MIMALLOC=ON \ + -DARROW_ORC=ON \ + -DARROW_PACKAGE_PREFIX=$PREFIX \ + -DARROW_PARQUET=ON \ + -DARROW_PLASMA=ON \ + -DARROW_PYTHON=ON \ + -DARROW_S3=ON \ + -DARROW_SIMD_LEVEL=NONE \ + -DARROW_USE_LD_GOLD=ON \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DCMAKE_BUILD_TYPE=release \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \ + -DPython3_EXECUTABLE=${PYTHON} \ + -DProtobuf_PROTOC_EXECUTABLE=$BUILD_PREFIX/bin/protoc \ + -GNinja \ + ${EXTRA_CMAKE_ARGS} \ + .. + +# Commented out until jemalloc and mimalloc are fixed upstream +if [[ "${target_platform}" == "osx-arm64" ]]; then + ninja jemalloc_ep-prefix/src/jemalloc_ep-stamp/jemalloc_ep-patch mimalloc_ep-prefix/src/mimalloc_ep-stamp/mimalloc_ep-patch + cp $BUILD_PREFIX/share/gnuconfig/config.* jemalloc_ep-prefix/src/jemalloc_ep/build-aux/ + sed -ie 's/list(APPEND mi_cflags -march=native)//g' mimalloc_ep-prefix/src/mimalloc_ep/CMakeLists.txt + # Use the correct register for thread-local storage + sed -ie 's/tpidr_el0/tpidrro_el0/g' mimalloc_ep-prefix/src/mimalloc_ep/include/mimalloc-internal.h +fi + +ninja install + +popd diff --git a/src/arrow/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh new file mode 100644 index 000000000..f0cf9ceb4 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/build-pyarrow.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env sh + +set -e +set -x + +# Build dependencies +export ARROW_HOME=$PREFIX +export PARQUET_HOME=$PREFIX +export SETUPTOOLS_SCM_PRETEND_VERSION=$PKG_VERSION +export PYARROW_BUILD_TYPE=release +export PYARROW_BUNDLE_ARROW_CPP_HEADERS=0 +export PYARROW_WITH_DATASET=1 +export PYARROW_WITH_FLIGHT=1 +if [[ "${target_platform}" == "osx-arm64" ]]; then + # We need llvm 11+ support in Arrow for this + export PYARROW_WITH_GANDIVA=0 +else + export PYARROW_WITH_GANDIVA=1 +fi +export PYARROW_WITH_HDFS=1 +export PYARROW_WITH_ORC=1 +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_PLASMA=1 +export PYARROW_WITH_S3=1 +export PYARROW_CMAKE_GENERATOR=Ninja +BUILD_EXT_FLAGS="" + +# Enable CUDA support +if [[ ! -z "${cuda_compiler_version+x}" && "${cuda_compiler_version}" != "None" ]]; then + export PYARROW_WITH_CUDA=1 +else + export PYARROW_WITH_CUDA=0 +fi + +# Resolve: Make Error at cmake_modules/SetupCxxFlags.cmake:338 (message): Unsupported arch flag: -march=. +if [[ "${target_platform}" == "linux-aarch64" ]]; then + export PYARROW_CMAKE_OPTIONS="-DARROW_ARMV8_ARCH=armv8-a" +fi + +cd python + +$PYTHON setup.py \ + build_ext \ + install --single-version-externally-managed \ + --record=record.txt + +if [[ "$PKG_NAME" == "pyarrow" ]]; then + rm -r ${SP_DIR}/pyarrow/tests +fi diff --git a/src/arrow/dev/tasks/conda-recipes/arrow-cpp/meta.yaml b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/meta.yaml new file mode 100644 index 000000000..48a862986 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/arrow-cpp/meta.yaml @@ -0,0 +1,302 @@ +# NOTE: In constrast to the conda-forge recipe, ARROW_VERSION is a templated variable here. +{% set version = ARROW_VERSION %} +{% set cuda_enabled = cuda_compiler_version != "None" %} +{% set build_ext_version = ARROW_VERSION %} +{% set build_ext = "cuda" if cuda_enabled else "cpu" %} +{% set proc_build_number = "0" %} + +package: + name: arrow-cpp-ext + version: {{ version }} + +source: + path: ../../../../ + +build: + number: 0 + # for cuda on win/linux, building with 9.2 is enough to be compatible with all later versions, + # since arrow is only using libcuda, and not libcudart. + skip: true # [(win or linux) and cuda_compiler_version not in ("None", "10.2")] + skip: true # [osx and cuda_compiler_version != "None"] + run_exports: + - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }} + +outputs: + - name: arrow-cpp-proc + version: {{ build_ext_version }} + build: + number: {{ proc_build_number }} + string: "{{ build_ext }}" + test: + commands: + - exit 0 + about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: + - LICENSE.txt + summary: 'A meta-package to select Arrow build variant' + + - name: arrow-cpp + script: build-arrow.sh # [not win] + script: bld-arrow.bat # [win] + version: {{ version }} + build: + string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} + run_exports: + - {{ pin_subpackage("arrow-cpp", max_pin="x.x.x") }} + ignore_run_exports: + - cudatoolkit + track_features: + {{ "- arrow-cuda" if cuda_enabled else "" }} + requirements: + build: + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - cython # [build_platform != target_platform] + - numpy # [build_platform != target_platform] + - gnuconfig # [osx and arm64] + - libprotobuf + - grpc-cpp + - cmake + - autoconf # [unix] + - ninja + - make # [unix] + - {{ compiler('c') }} + - {{ compiler('cxx') }} + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] + host: + - aws-sdk-cpp + - boost-cpp >=1.70 + - brotli + - bzip2 + - c-ares + - gflags + - glog + - grpc-cpp + - libprotobuf + - clangdev 10 # [not (osx and arm64)] + - llvmdev 10 # [not (osx and arm64)] + - libutf8proc + - lz4-c + - numpy + - orc # [unix] + - python + - rapidjson + - re2 + - snappy + - thrift-cpp + - zlib + - zstd + run: + - {{ pin_compatible('numpy', lower_bound='1.16') }} + - python + run_constrained: + - arrow-cpp-proc * {{ build_ext }} + - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] + + about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: + - LICENSE.txt + summary: C++ libraries for Apache Arrow + + test: + commands: + # headers + - test -f $PREFIX/include/arrow/api.h # [unix] + - test -f $PREFIX/include/arrow/flight/types.h # [unix] + - test -f $PREFIX/include/plasma/client.h # [unix] + - test -f $PREFIX/include/gandiva/engine.h # [unix and not (osx and arm64)] + - test -f $PREFIX/include/parquet/api/reader.h # [unix] + - if not exist %LIBRARY_INC%\\arrow\\api.h exit 1 # [win] + - if not exist %LIBRARY_INC%\\gandiva\\engine.h exit 1 # [win] + - if not exist %LIBRARY_INC%\\parquet\\api\\reader.h exit 1 # [win] + + # shared + - test -f $PREFIX/lib/libarrow.so # [linux] + - test -f $PREFIX/lib/libarrow_dataset.so # [linux] + - test -f $PREFIX/lib/libarrow_flight.so # [linux] + - test -f $PREFIX/lib/libarrow_python.so # [linux] + - test -f $PREFIX/lib/libparquet.so # [linux] + - test -f $PREFIX/lib/libgandiva.so # [linux] + - test -f $PREFIX/lib/libplasma.so # [linux] + - test -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version != "None") and unix] + - test ! -f $PREFIX/lib/libarrow_cuda${SHLIB_EXT} # [(cuda_compiler_version == "None") and unix] + - if not exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version != "None") and win] + - if exist %PREFIX%\\Library\\bin\\arrow_cuda.dll exit 1 # [(cuda_compiler_version == "None") and win] + - test -f $PREFIX/lib/libarrow.dylib # [osx] + - test -f $PREFIX/lib/libarrow_dataset.dylib # [osx] + - test -f $PREFIX/lib/libarrow_python.dylib # [osx] + - test -f $PREFIX/lib/libgandiva.dylib # [osx and not arm64] + - test -f $PREFIX/lib/libparquet.dylib # [osx] + - test -f $PREFIX/lib/libplasma.dylib # [osx] + - if not exist %PREFIX%\\Library\\bin\\arrow.dll exit 1 # [win] + - if not exist %PREFIX%\\Library\\bin\\arrow_dataset.dll exit 1 # [win] + - if not exist %PREFIX%\\Library\\bin\\arrow_flight.dll exit 1 # [win] + - if not exist %PREFIX%\\Library\\bin\\arrow_python.dll exit 1 # [win] + - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win] + - if not exist %PREFIX%\\Library\\bin\\gandiva.dll exit 1 # [win] + + # absence of static libraries + - test ! -f $PREFIX/lib/libarrow.a # [unix] + - test ! -f $PREFIX/lib/libarrow_dataset.a # [unix] + - test ! -f $PREFIX/lib/libarrow_flight.a # [unix] + - test ! -f $PREFIX/lib/libarrow_python.a # [unix] + - test ! -f $PREFIX/lib/libplasma.a # [unix] + - test ! -f $PREFIX/lib/libparquet.a # [unix] + - test ! -f $PREFIX/lib/libgandiva.a # [unix] + - if exist %PREFIX%\\Library\\lib\\arrow_static.lib exit 1 # [win] + - if exist %PREFIX%\\Library\\lib\\arrow_dataset_static.lib exit 1 # [win] + - if exist %PREFIX%\\Library\\lib\\arrow_flight_static.lib exit 1 # [win] + - if exist %PREFIX%\\Library\\lib\\arrow_python_static.lib exit 1 # [win] + - if exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win] + - if exist %PREFIX%\\Library\\lib\\gandiva_static.lib exit 1 # [win] + + - name: pyarrow + script: build-pyarrow.sh # [not win] + script: bld-pyarrow.bat # [win] + version: {{ version }} + build: + string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} + ignore_run_exports: + - cudatoolkit + track_features: + {{ "- arrow-cuda" if cuda_enabled else "" }} + requirements: + build: + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - cython # [build_platform != target_platform] + - numpy # [build_platform != target_platform] + - cmake + - ninja + - make # [unix] + - {{ compiler('c') }} + - {{ compiler('cxx') }} + # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] + host: + - {{ pin_subpackage('arrow-cpp', exact=True) }} + - cython + - numpy + - python + - setuptools + - setuptools_scm + - six + run: + - {{ pin_subpackage('arrow-cpp', exact=True) }} + - {{ pin_compatible('numpy', lower_bound='1.16') }} + # empty parquet-cpp metapackage, force old versions to be uninstalled + - parquet-cpp 1.5.1.* + - python + run_constrained: + - arrow-cpp-proc * {{ build_ext }} + - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] + + about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: + - LICENSE.txt + summary: Python libraries for Apache Arrow + + test: + imports: + - pyarrow + - pyarrow.dataset + - pyarrow.flight + - pyarrow.gandiva # [not (osx and arm64)] + - pyarrow.orc # [unix] + - pyarrow.parquet + - pyarrow.plasma # [unix] + - pyarrow.fs + - pyarrow._s3fs + - pyarrow._hdfs + # We can only test importing cuda package but cannot run when a + # CUDA device is not available, for instance, when building from CI. + # On Windows, we cannot even do that due to `nvcuda.dll` not being found, see + # https://conda-forge.org/docs/maintainer/knowledge_base.html#nvcuda-dll-cannot-be-found-on-windows + # However, we check below for (at least) the presence of a correctly-compiled module + - pyarrow.cuda # [cuda_compiler_version != "None" and not win] + commands: + - test ! -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix] + - if exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win] + # Need to remove dot from PY_VER; %MYVAR:x=y% replaces "x" in %MYVAR% with "y" + - if not exist %SP_DIR%/pyarrow/_cuda.cp%PY_VER:.=%-win_amd64.pyd exit 1 # [win and cuda_compiler_version != "None"] + + - name: pyarrow-tests + script: build-pyarrow.sh # [not win] + script: bld-pyarrow.bat # [win] + version: {{ version }} + build: + string: py{{ CONDA_PY }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }}_{{ build_ext }} + ignore_run_exports: + - cudatoolkit + track_features: + {{ "- arrow-cuda" if cuda_enabled else "" }} + requirements: + build: + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - cython # [build_platform != target_platform] + - numpy # [build_platform != target_platform] + - cmake + - ninja + - make # [unix] + - {{ compiler('c') }} + - {{ compiler('cxx') }} + # pyarrow does not require nvcc but it needs to link against libraries in arrow-cpp=*=*cuda + - {{ compiler("cuda") }} # [cuda_compiler_version != "None"] + host: + - {{ pin_subpackage('arrow-cpp', exact=True) }} + - {{ pin_subpackage('pyarrow', exact=True) }} + - cython + - numpy + - python + - setuptools + - setuptools_scm + - six + run: + - {{ pin_subpackage('pyarrow', exact=True) }} + - python + run_constrained: + - arrow-cpp-proc * {{ build_ext }} + - cudatoolkit >=9.2 # [cuda_compiler_version != "None"] + + about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: + - LICENSE.txt + summary: Python test files for Apache Arrow + + test: + commands: + - test -f ${SP_DIR}/pyarrow/tests/test_array.py # [unix] + - if not exist %SP_DIR%/pyarrow/tests/test_array.py exit 1 # [win] + +about: + home: http://github.com/apache/arrow + license: Apache-2.0 + license_file: + - LICENSE.txt + summary: C++ and Python libraries for Apache Arrow + +extra: + recipe-maintainers: + - wesm + - xhochy + - leifwalsh + - jreback + - cpcloud + - pcmoritz + - robertnishihara + - siddharthteotia + - kou + - kszucs + - pitrou + - pearu + - nealrichardson + - jakirkham diff --git a/src/arrow/dev/tasks/conda-recipes/azure.clean.yml b/src/arrow/dev/tasks/conda-recipes/azure.clean.yml new file mode 100644 index 000000000..84f167812 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/azure.clean.yml @@ -0,0 +1,28 @@ +jobs: +- job: linux + pool: + vmImage: ubuntu-latest + timeoutInMinutes: 360 + + steps: + - script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + displayName: Clone arrow + + - script: | + conda install -y -c conda-forge pandas anaconda-client packaging + displayName: Install requirements + + - script: | + {% if arrow.branch == 'master' %} + mkdir -p $HOME/.continuum/anaconda-client/tokens/ + echo $(CROSSBOW_ANACONDA_TOKEN) > $HOME/.continuum/anaconda-client/tokens/https%3A%2F%2Fapi.anaconda.org.token + {% endif %} + eval "$(conda shell.bash hook)" + conda activate base + python3 arrow/dev/tasks/conda-recipes/clean.py {% if arrow.branch == 'master' %}FORCE{% endif %} + displayName: Delete outdated packages + diff --git a/src/arrow/dev/tasks/conda-recipes/azure.linux.yml b/src/arrow/dev/tasks/conda-recipes/azure.linux.yml new file mode 100755 index 000000000..c05d284d2 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/azure.linux.yml @@ -0,0 +1,38 @@ +{% import 'macros.jinja' as macros with context %} + +jobs: +- job: linux + pool: + vmImage: ubuntu-latest + timeoutInMinutes: 360 + + variables: + CONFIG: {{ config }} + R_CONFIG: {{ r_config|default("") }} + ARROW_VERSION: {{ arrow.no_rc_version }} + UPLOAD_PACKAGES: False + + steps: + # configure qemu binfmt-misc running. This allows us to run docker containers + # embedded qemu-static + - script: | + docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes + ls /proc/sys/fs/binfmt_misc/ + displayName: Configure binfmt_misc + condition: not(startsWith(variables['CONFIG'], 'linux_64')) + + {{ macros.azure_checkout_arrow() }} + + - task: CondaEnvironment@1 + inputs: + packageSpecs: 'anaconda-client shyaml' + installOptions: '-c conda-forge' + updateConda: false + + - script: | + mkdir build_artifacts + CI=azure arrow/dev/tasks/conda-recipes/run_docker_build.sh $(pwd)/build_artifacts + displayName: Run docker build + + {{ macros.azure_upload_releases("build_artifacts/*/*.tar.bz2") }} + {{ macros.azure_upload_anaconda("build_artifacts/*/*.tar.bz2") }} diff --git a/src/arrow/dev/tasks/conda-recipes/azure.osx.yml b/src/arrow/dev/tasks/conda-recipes/azure.osx.yml new file mode 100755 index 000000000..99bb76ba5 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/azure.osx.yml @@ -0,0 +1,83 @@ +{% import 'macros.jinja' as macros with context %} + +jobs: +- job: osx + pool: + vmImage: macOS-10.14 + timeoutInMinutes: 360 + variables: + CONFIG: {{ config }} + R_CONFIG: {{ r_config|default("") }} + ARROW_VERSION: {{ arrow.no_rc_version }} + UPLOAD_PACKAGES: False + steps: + - bash: | + echo "##vso[task.prependpath]$CONDA/bin" + sudo chown -R $USER $CONDA + displayName: Add conda to PATH + + - script: | + source activate base + conda config --set channel_priority strict + conda install -n base -c conda-forge --quiet --yes conda-forge-ci-setup=3 conda-build + displayName: 'Add conda-forge-ci-setup=3' + + - script: | + echo "Removing homebrew from Azure to avoid conflicts." + /usr/bin/sudo mangle_homebrew + /usr/bin/sudo -k + displayName: Mangle homebrew + + {{ macros.azure_checkout_arrow() }} + + - script: | + source activate base + echo "Configuring conda." + + setup_conda_rc ./ ./ ./.ci_support/${CONFIG}.yaml + export CI=azure + source run_conda_forge_build_setup + conda update --yes --quiet --override-channels -c conda-forge --all + displayName: Configure conda and conda-build + workingDirectory: arrow/dev/tasks/conda-recipes + env: + OSX_FORCE_SDK_DOWNLOAD: "1" + + - script: | + source activate base + mangle_compiler ./ ./ ./.ci_support/${CONFIG}.yaml + workingDirectory: arrow/dev/tasks/conda-recipes + displayName: Mangle compiler + + - script: | + source activate base + make_build_number ./ ./ ./.ci_support/${CONFIG}.yaml + workingDirectory: arrow/dev/tasks/conda-recipes + displayName: Generate build number clobber file + + - script: | + source activate base + set +x + if [[ "${CONFIG}" == osx_arm* ]]; then + EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test" + fi + conda build arrow-cpp \ + -m ./.ci_support/${CONFIG}.yaml \ + --clobber-file ./.ci_support/clobber_${CONFIG}.yaml \ + ${EXTRA_CB_OPTIONS:-} \ + --output-folder ./build_artifacts + + if [ ! -z "${R_CONFIG}" ]; then + conda build r-arrow \ + -m ./.ci_support/r/${R_CONFIG}.yaml \ + --output-folder ./build_artifacts + fi + workingDirectory: arrow/dev/tasks/conda-recipes + displayName: Build recipes + + - script: | + sudo mv /usr/local/conda_mangled/* /usr/local/ + displayName: Unmangle homebrew + + {{ macros.azure_upload_releases("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }} + {{ macros.azure_upload_anaconda("arrow/dev/tasks/conda-recipes/build_artifacts/osx-*/*.tar.bz2") }} diff --git a/src/arrow/dev/tasks/conda-recipes/azure.win.yml b/src/arrow/dev/tasks/conda-recipes/azure.win.yml new file mode 100755 index 000000000..422e2f0e9 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/azure.win.yml @@ -0,0 +1,77 @@ +{% import 'macros.jinja' as macros with context %} + +jobs: +- job: win + pool: + vmImage: vs2017-win2016 + timeoutInMinutes: 360 + variables: + CONFIG: {{ config }} + R_CONFIG: {{ r_config|default("") }} + ARROW_VERSION: {{ arrow.no_rc_version }} + CONDA_BLD_PATH: D:\\bld\\ + UPLOAD_PACKAGES: False + + steps: + - script: | + choco install vcpython27 -fdv -y --debug + condition: contains(variables['CONFIG'], 'vs2008') + displayName: Install vcpython27.msi (if needed) + + - powershell: | + Set-PSDebug -Trace 1 + $batchcontent = @" + ECHO ON + SET vcpython=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0 + DIR "%vcpython%" + CALL "%vcpython%\vcvarsall.bat" %* + "@ + $batchDir = "C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC" + $batchPath = "$batchDir" + "\vcvarsall.bat" + New-Item -Path $batchPath -ItemType "file" -Force + Set-Content -Value $batchcontent -Path $batchPath + Get-ChildItem -Path $batchDir + Get-ChildItem -Path ($batchDir + '\..') + condition: contains(variables['CONFIG'], 'vs2008') + displayName: Patch vs2008 (if needed) + + - task: CondaEnvironment@1 + inputs: + packageSpecs: 'python=3.6 conda-build conda conda-forge::conda-forge-ci-setup=3 pip' # Optional + installOptions: "-c conda-forge" + updateConda: true + displayName: Install conda-build and activate environment + - script: set PYTHONUNBUFFERED=1 + + {{ macros.azure_checkout_arrow()|indent(2) }} + + # Configure the VM + - script: setup_conda_rc .\ .\ .\.ci_support\%CONFIG%.yaml + workingDirectory: arrow\dev\tasks\conda-recipes + + # Configure the VM. + - script: | + set "CI=azure" + call activate base + run_conda_forge_build_setup + displayName: conda-forge build setup + workingDirectory: arrow\dev\tasks\conda-recipes + + - script: | + conda.exe build arrow-cpp parquet-cpp -m .ci_support\%CONFIG%.yaml + displayName: Build recipe + workingDirectory: arrow\dev\tasks\conda-recipes + env: + PYTHONUNBUFFERED: 1 + condition: not(contains(variables['CONFIG'], 'vs2008')) + + - script: | + conda.exe build r-arrow -m .ci_support\r\%R_CONFIG%.yaml + displayName: Build recipe + workingDirectory: arrow\dev\tasks\conda-recipes + env: + PYTHONUNBUFFERED: 1 + condition: contains(variables['R_CONFIG'], 'win') + + {{ macros.azure_upload_releases("D:\\bld\\win-64\\*.tar.bz2")|indent(2) }} + {{ macros.azure_upload_anaconda("D:\\bld\\win-64\\*.tar.bz2")|indent(2) }} diff --git a/src/arrow/dev/tasks/conda-recipes/azure.yml b/src/arrow/dev/tasks/conda-recipes/azure.yml new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/azure.yml diff --git a/src/arrow/dev/tasks/conda-recipes/build_steps.sh b/src/arrow/dev/tasks/conda-recipes/build_steps.sh new file mode 100755 index 000000000..25864c08a --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/build_steps.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# NOTE: This script has been slightly adopted to suite the Apache Arrow / crossbow CI +# setup. The next time this is updated to the current version on conda-forge, +# you will also make this additions afterwards. + +# PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here +# will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent +# changes to this script, consider a proposal to conda-smithy so that other feedstocks can also +# benefit from the improvement. + +set -xeuo pipefail + +output_dir=${1} + +export PYTHONUNBUFFERED=1 +export FEEDSTOCK_ROOT="${FEEDSTOCK_ROOT:-/home/conda/feedstock_root}" +export CI_SUPPORT="${FEEDSTOCK_ROOT}/.ci_support" +export CONFIG_FILE="${CI_SUPPORT}/${CONFIG}.yaml" + +cat >~/.condarc <<CONDARC + +conda-build: + root-dir: ${output_dir} + +CONDARC + +conda install --yes --quiet conda-forge-ci-setup=3 conda-build pip -c conda-forge + +# set up the condarc +setup_conda_rc "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" + +source run_conda_forge_build_setup + +# make the build number clobber +make_build_number "${FEEDSTOCK_ROOT}" "${FEEDSTOCK_ROOT}" "${CONFIG_FILE}" + +export CONDA_BLD_PATH="${output_dir}" + +conda build \ + "${FEEDSTOCK_ROOT}/arrow-cpp" \ + "${FEEDSTOCK_ROOT}/parquet-cpp" \ + -m "${CI_SUPPORT}/${CONFIG}.yaml" \ + --clobber-file "${CI_SUPPORT}/clobber_${CONFIG}.yaml" \ + --output-folder "${output_dir}" + +if [ ! -z "${R_CONFIG:-}" ]; then + conda build \ + "${FEEDSTOCK_ROOT}/r-arrow" \ + -m "${CI_SUPPORT}/r/${R_CONFIG}.yaml" \ + --output-folder "${output_dir}" +fi + + +touch "${output_dir}/conda-forge-build-done-${CONFIG}" diff --git a/src/arrow/dev/tasks/conda-recipes/clean.py b/src/arrow/dev/tasks/conda-recipes/clean.py new file mode 100644 index 000000000..bd31c875d --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/clean.py @@ -0,0 +1,80 @@ +from subprocess import check_output, check_call +from typing import List + +import json +import os +import pandas as pd +import sys + +from packaging.version import Version + + +VERSIONS_TO_KEEP = 5 +PACKAGES = [ + "arrow-cpp", + "arrow-cpp-proc", + "parquet-cpp", + "pyarrow", + "pyarrow-tests", + "r-arrow", +] +PLATFORMS = [ + "linux-64", + "linux-aarch64", + "osx-64", + "win-64", +] +EXCLUDED_PATTERNS = [ + ["r-arrow", "linux-aarch64"], +] + + +def packages_to_delete(package_name: str, platform: str) -> List[str]: + env = os.environ.copy() + env["CONDA_SUBDIR"] = platform + pkgs_json = check_output( + [ + "conda", + "search", + "--json", + "-c", + "arrow-nightlies", + "--override-channels", + package_name, + ], + env=env, + ) + pkgs = pd.DataFrame(json.loads(pkgs_json)[package_name]) + pkgs["version"] = pkgs["version"].map(Version) + pkgs["py_version"] = pkgs["build"].str.slice(0, 4) + + to_delete = [] + + for (subdir, python), group in pkgs.groupby(["subdir", "py_version"]): + group = group.sort_values(by="version", ascending=False) + + if len(group) > VERSIONS_TO_KEEP: + del_candidates = group[VERSIONS_TO_KEEP:] + to_delete += ( + f"arrow-nightlies/{package_name}/" + + del_candidates["version"].astype(str) + + del_candidates["url"].str.replace( + "https://conda.anaconda.org/arrow-nightlies", "" + ) + ).to_list() + + return to_delete + + +if __name__ == "__main__": + to_delete = [] + for package in PACKAGES: + for platform in PLATFORMS: + if [package, platform] in EXCLUDED_PATTERNS: + continue + to_delete += packages_to_delete(package, platform) + + for name in to_delete: + print(f"Deleting {name} …") + if "FORCE" in sys.argv: + check_call(["anaconda", "remove", "-f", name]) diff --git a/src/arrow/dev/tasks/conda-recipes/conda-forge.yml b/src/arrow/dev/tasks/conda-recipes/conda-forge.yml new file mode 100644 index 000000000..4c07b5dd3 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/conda-forge.yml @@ -0,0 +1 @@ +channel_priority: strict diff --git a/src/arrow/dev/tasks/conda-recipes/parquet-cpp/meta.yaml b/src/arrow/dev/tasks/conda-recipes/parquet-cpp/meta.yaml new file mode 100644 index 000000000..5de06c32b --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/parquet-cpp/meta.yaml @@ -0,0 +1,51 @@ +# ARROW-3229: this is a meta-package to prevent conflicts in the future + +{% set parquet_version = "1.5.1" %} + +package: + name: parquet-cpp + version: {{ parquet_version }} + +build: + number: 0 + skip: true # [win32] + skip: true # [win and py<35] + +requirements: + host: + # NOTE: in the upstream feedstock use >= instead of = + - arrow-cpp ={{ ARROW_VERSION }} + run: + - arrow-cpp ={{ ARROW_VERSION }} + +test: + commands: + # headers + - test -f $PREFIX/include/parquet/api/reader.h # [unix] + - if not exist %LIBRARY_INC%\\parquet\\api\\reader.h exit 1 # [win] + + # shared + - test -f $PREFIX/lib/libparquet.so # [linux] + - test -f $PREFIX/lib/libparquet.dylib # [osx] + - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1 # [win] + + # absence of static libraries + - test ! -f $PREFIX/lib/libparquet.a # [unix] + - if exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1 # [win] + +about: + home: http://github.com/apache/arrow + license: Apache 2.0 + summary: 'C++ libraries for the Apache Parquet file format' + +extra: + recipe-maintainers: + - wesm + - xhochy + - leifwalsh + - jreback + - cpcloud + - siddharthteotia + - kou + - kszucs + - pitrou diff --git a/src/arrow/dev/tasks/conda-recipes/r-arrow/bld.bat b/src/arrow/dev/tasks/conda-recipes/r-arrow/bld.bat new file mode 100644 index 000000000..a193ddc0a --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/r-arrow/bld.bat @@ -0,0 +1,9 @@ +bash %RECIPE_DIR%/build_win.sh
+IF %ERRORLEVEL% NEQ 0 exit 1
+cp %RECIPE_DIR%/configure.win r
+IF %ERRORLEVEL% NEQ 0 exit 1
+cp %RECIPE_DIR%/install.libs.R r/src
+IF %ERRORLEVEL% NEQ 0 exit 1
+set "MAKEFLAGS=-j%CPU_COUNT%"
+"%R%" CMD INSTALL --build r
+IF %ERRORLEVEL% NEQ 0 exit 1
diff --git a/src/arrow/dev/tasks/conda-recipes/r-arrow/build.sh b/src/arrow/dev/tasks/conda-recipes/r-arrow/build.sh new file mode 100644 index 000000000..e868189a2 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/r-arrow/build.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +export DISABLE_AUTOBREW=1 +$R CMD INSTALL --build r/. diff --git a/src/arrow/dev/tasks/conda-recipes/r-arrow/build_win.sh b/src/arrow/dev/tasks/conda-recipes/r-arrow/build_win.sh new file mode 100755 index 000000000..22c07d6e0 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/r-arrow/build_win.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -exuo pipefail + + +# Rename arrow.dll to lib_arrow.dll to avoid conflicts with the arrow-cpp arrow.dll +sed -i -e 's/void R_init_arrow/__declspec(dllexport) void R_init_lib_arrow/g' r/src/arrowExports.cpp +sed -i -e 's/useDynLib(arrow/useDynLib(lib_arrow/g' r/NAMESPACE diff --git a/src/arrow/dev/tasks/conda-recipes/r-arrow/configure.win b/src/arrow/dev/tasks/conda-recipes/r-arrow/configure.win new file mode 100755 index 000000000..0b11d1335 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/r-arrow/configure.win @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# Remove the -I../inst/include/ when unvendoring cpp11 in ARROW-13610 +echo "PKG_CPPFLAGS=-DNDEBUG -I\"${LIBRARY_PREFIX}/include\" -I\"${PREFIX}/include\" -DARROW_R_WITH_ARROW -DARROW_R_WITH_PARQUET -DARROW_R_WITH_DATASET -DARROW_R_WITH_S3 -DARROW_R_WITH_JSON -I../inst/include/" > src/Makevars.win +echo "PKG_CXXFLAGS=\$(CXX_VISIBILITY)" >> src/Makevars.win +echo 'CXX_STD=CXX11' >> src/Makevars.win +echo "PKG_LIBS=-L\"${LIBRARY_PREFIX}/lib\" -larrow_dataset -lparquet -larrow" >> src/Makevars.win diff --git a/src/arrow/dev/tasks/conda-recipes/r-arrow/install.libs.R b/src/arrow/dev/tasks/conda-recipes/r-arrow/install.libs.R new file mode 100644 index 000000000..005bbe16b --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/r-arrow/install.libs.R @@ -0,0 +1,5 @@ +src_dir <- file.path(R_PACKAGE_SOURCE, "src", fsep = "/") +dest_dir <- file.path(R_PACKAGE_DIR, paste0("libs", R_ARCH), fsep="/") + +dir.create(file.path(R_PACKAGE_DIR, paste0("libs", R_ARCH), fsep="/"), recursive = TRUE, showWarnings = FALSE) +file.copy(file.path(src_dir, "arrow.dll", fsep = "/"), file.path(dest_dir, "lib_arrow.dll", fsep = "/")) diff --git a/src/arrow/dev/tasks/conda-recipes/r-arrow/meta.yaml b/src/arrow/dev/tasks/conda-recipes/r-arrow/meta.yaml new file mode 100644 index 000000000..5f0643bef --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/r-arrow/meta.yaml @@ -0,0 +1,66 @@ +{% set version = ARROW_VERSION %} +{% set posix = 'm2-' if win else '' %} +{% set native = 'm2w64-' if win else '' %} + +package: + name: r-arrow + version: {{ version|replace("-", "_") }} + +source: + path: ../../../../ + +build: + merge_build_host: true # [win] + number: 0 + rpaths: + - lib/R/lib/ + - lib/ + +requirements: + build: + - {{ compiler('c') }} # [not win] + - {{ compiler('cxx') }} # [not win] + - {{ compiler('r_clang') }} # [win] + - pkg-config # [not win] + - {{ posix }}make + - {{ posix }}sed # [win] + - {{ posix }}coreutils # [win] + - {{ posix }}filesystem # [win] + - {{ posix }}zip # [win] + host: + # Needs to be here, otherwise merge_build_host runs into issues + - pkg-config # [win] + - r-base + - arrow-cpp {{ version }} + - r-cpp11 + - r-r6 + - r-assertthat + - r-bit64 + - r-purrr + - r-rlang + - r-tidyselect + run: + - r-base + - r-r6 + - r-assertthat + - r-bit64 + - r-purrr + - r-rlang + - r-tidyselect + +test: + commands: + - $R -e "library('arrow')" # [not win] + - "\"%R%\" -e \"library('arrow'); data(mtcars); write_parquet(mtcars, 'test.parquet')\"" # [win] + +about: + home: https://github.com/apache/arrow + license: Apache-2.0 + license_file: LICENSE.txt + summary: R Integration to 'Apache' 'Arrow'. + license_family: APACHE + +extra: + recipe-maintainers: + - conda-forge/r + - conda-forge/arrow-cpp diff --git a/src/arrow/dev/tasks/conda-recipes/run_docker_build.sh b/src/arrow/dev/tasks/conda-recipes/run_docker_build.sh new file mode 100755 index 000000000..7645c43e2 --- /dev/null +++ b/src/arrow/dev/tasks/conda-recipes/run_docker_build.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# NOTE: This script has been slightly adopted to suite the Apache Arrow / crossbow CI +# setup. The next time this is updated to the current version on conda-forge, +# you will also make this additions afterwards. + +# PLEASE NOTE: This script has been automatically generated by conda-smithy. Any changes here +# will be lost next time ``conda smithy rerender`` is run. If you would like to make permanent +# changes to this script, consider a proposal to conda-smithy so that other feedstocks can also +# benefit from the improvement. + +set -xeo pipefail + +build_dir=${1} + +THISDIR="$( cd "$( dirname "$0" )" >/dev/null && pwd )" +ARROW_ROOT=$(cd "$THISDIR/../../.."; pwd;) +FEEDSTOCK_ROOT=$THISDIR + +docker info + +# In order for the conda-build process in the container to write to the mounted +# volumes, we need to run with the same id as the host machine, which is +# normally the owner of the mounted volumes, or at least has write permission +export HOST_USER_ID=$(id -u) +# Check if docker-machine is being used (normally on OSX) and get the uid from +# the VM +if hash docker-machine 2> /dev/null && docker-machine active > /dev/null; then + export HOST_USER_ID=$(docker-machine ssh $(docker-machine active) id -u) +fi + +if [ -z "$CONFIG" ]; then + set +x + FILES=`ls .ci_support/linux_*` + CONFIGS="" + for file in $FILES; do + CONFIGS="${CONFIGS}'${file:12:-5}' or "; + done + echo "Need to set CONFIG env variable. Value can be one of ${CONFIGS:0:-4}" + exit 1 +fi + +if [ -z "${DOCKER_IMAGE}" ]; then + SHYAML_INSTALLED="$(shyaml -h || echo NO)" + if [ "${SHYAML_INSTALLED}" == "NO" ]; then + echo "WARNING: DOCKER_IMAGE variable not set and shyaml not installed. Falling back to condaforge/linux-anvil-comp7" + DOCKER_IMAGE="condaforge/linux-anvil-comp7" + else + DOCKER_IMAGE="$(cat "${FEEDSTOCK_ROOT}/.ci_support/${CONFIG}.yaml" | shyaml get-value docker_image.0 condaforge/linux-anvil-comp7 )" + fi +fi + +mkdir -p "${build_dir}" +DONE_CANARY="${build_dir}/conda-forge-build-done-${CONFIG}" +rm -f "$DONE_CANARY" + +if [ -z "${CI}" ]; then + DOCKER_RUN_ARGS="-it " +fi + +export UPLOAD_PACKAGES="${UPLOAD_PACKAGES:-True}" +docker run ${DOCKER_RUN_ARGS} \ + --shm-size=2G \ + -v "${ARROW_ROOT}":/arrow:rw,z \ + -v "${build_dir}":/build:rw \ + -e FEEDSTOCK_ROOT="/arrow/dev/tasks/conda-recipes" \ + -e CONFIG \ + -e R_CONFIG \ + -e HOST_USER_ID \ + -e UPLOAD_PACKAGES \ + -e ARROW_VERSION \ + -e CI \ + $DOCKER_IMAGE \ + bash /arrow/dev/tasks/conda-recipes/build_steps.sh /build + +# verify that the end of the script was reached +test -f "$DONE_CANARY" diff --git a/src/arrow/dev/tasks/cpp-examples/github.linux.yml b/src/arrow/dev/tasks/cpp-examples/github.linux.yml new file mode 100644 index 000000000..717d3c443 --- /dev/null +++ b/src/arrow/dev/tasks/cpp-examples/github.linux.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + test: + name: C++ Example + runs-on: ubuntu-latest + steps: + - name: Checkout Arrow + shell: bash + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + - name: Run + shell: bash + run: | + cd arrow/cpp/examples/{{ type }} + docker-compose run --rm {{ run }} diff --git a/src/arrow/dev/tasks/docker-tests/azure.linux.yml b/src/arrow/dev/tasks/docker-tests/azure.linux.yml new file mode 100644 index 000000000..b8f1151f7 --- /dev/null +++ b/src/arrow/dev/tasks/docker-tests/azure.linux.yml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +jobs: +- job: linux + pool: + vmImage: ubuntu-latest + timeoutInMinutes: 360 + {% if env is defined %} + variables: + {% for key, value in env.items() %} + {{ key }}: {{ value }} + {% endfor %} + {% endif %} + + steps: + - task: DockerInstaller@0 + displayName: Docker Installer + inputs: + dockerVersion: 17.09.0-ce + releaseType: stable + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + + - script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + displayName: Clone arrow + + - script: pip install -e arrow/dev/archery[docker] + displayName: Setup Archery + + - script: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ run }} + displayName: Execute Docker Build diff --git a/src/arrow/dev/tasks/docker-tests/circle.linux.yml b/src/arrow/dev/tasks/docker-tests/circle.linux.yml new file mode 100644 index 000000000..3ddb93dc9 --- /dev/null +++ b/src/arrow/dev/tasks/docker-tests/circle.linux.yml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: 2 +jobs: + build: + machine: + image: ubuntu-1604:202004-01 + {%- if env is defined %} + environment: + {%- for key, value in env.items() %} + {{ key }}: {{ value }} + {%- endfor %} + {%- endif %} + steps: + - run: | + docker -v + docker-compose -v + - run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - run: + name: Execute Docker Build + command: | + pyenv versions + pyenv global 3.6.10 + pip install -e arrow/dev/archery[docker] + archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ run }} + no_output_timeout: "1h" + +workflows: + version: 2 + build: + jobs: + - build diff --git a/src/arrow/dev/tasks/docker-tests/github.linux.yml b/src/arrow/dev/tasks/docker-tests/github.linux.yml new file mode 100644 index 000000000..1faf482cb --- /dev/null +++ b/src/arrow/dev/tasks/docker-tests/github.linux.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + test: + name: Docker Test + runs-on: ubuntu-latest + {% if env is defined %} + env: + {% for key, value in env.items() %} + {{ key }}: "{{ value }}" + {% endfor %} + {% endif %} + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + + - name: Execute Docker Build + shell: bash + run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ flags|default("") }} {{ image }} {{ command|default("") }} + + {% if '-r' in image %} + - name: Dump R install logs + run: cat arrow/r/check/arrow.Rcheck/00install.out + continue-on-error: true + if: always() + {% endif %} + + {% if arrow.branch == 'master' %} + {{ macros.github_login_dockerhub()|indent }} + - name: Push Docker Image + shell: bash + run: archery docker push {{ image }} + {% endif %} diff --git a/src/arrow/dev/tasks/homebrew-formulae/apache-arrow.rb b/src/arrow/dev/tasks/homebrew-formulae/apache-arrow.rb new file mode 100644 index 000000000..5f39666bd --- /dev/null +++ b/src/arrow/dev/tasks/homebrew-formulae/apache-arrow.rb @@ -0,0 +1,69 @@ +class ApacheArrow < Formula + desc "Columnar in-memory analytics layer designed to accelerate big data" + homepage "https://arrow.apache.org/" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-6.0.1/apache-arrow-6.0.1.tar.gz" + sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" + license "Apache-2.0" + head "https://github.com/apache/arrow.git" + + depends_on "boost" => :build + depends_on "cmake" => :build + depends_on "llvm" => :build + depends_on "brotli" + depends_on "glog" + depends_on "grpc" + depends_on "lz4" + depends_on "numpy" + depends_on "openssl@1.1" + depends_on "protobuf" + depends_on "python@3.9" + depends_on "rapidjson" + depends_on "snappy" + depends_on "thrift" + depends_on "zstd" + + def install + ENV.cxx11 + # link against system libc++ instead of llvm provided libc++ + ENV.remove "HOMEBREW_LIBRARY_PATHS", Formula["llvm"].opt_lib + args = %W[ + -DARROW_FLIGHT=ON + -DARROW_GANDIVA=ON + -DARROW_JEMALLOC=ON + -DARROW_MIMALLOC=ON + -DARROW_ORC=ON + -DARROW_PARQUET=ON + -DARROW_PLASMA=ON + -DARROW_PROTOBUF_USE_SHARED=ON + -DARROW_PYTHON=ON + -DARROW_WITH_BZ2=ON + -DARROW_WITH_ZLIB=ON + -DARROW_WITH_ZSTD=ON + -DARROW_WITH_LZ4=ON + -DARROW_WITH_SNAPPY=ON + -DARROW_WITH_BROTLI=ON + -DARROW_INSTALL_NAME_RPATH=OFF + -DPython3_EXECUTABLE=#{Formula["python@3.9"].bin/"python3"} + ] + # Re-enable -DARROW_S3=ON and add back aws-sdk-cpp to depends_on in ARROW-6437 + + mkdir "build" + cd "build" do + system "cmake", "../cpp", *std_cmake_args, *args + system "make" + system "make", "install" + end + end + + test do + (testpath/"test.cpp").write <<~EOS + #include "arrow/api.h" + int main(void) { + arrow::int64(); + return 0; + } + EOS + system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", "-larrow", "-o", "test" + system "./test" + end +end diff --git a/src/arrow/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb b/src/arrow/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb new file mode 100644 index 000000000..2a77b01fc --- /dev/null +++ b/src/arrow/dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# https://github.com/autobrew/homebrew-core/blob/master/Formula/apache-arrow.rb +class ApacheArrow < Formula + desc "Columnar in-memory analytics layer designed to accelerate big data" + homepage "https://arrow.apache.org/" + url "https://www.apache.org/dyn/closer.lua?path=arrow/arrow-6.0.1/apache-arrow-6.0.1.tar.gz" + sha256 "9948ddb6d4798b51552d0dca3252dd6e3a7d0f9702714fc6f5a1b59397ce1d28" + head "https://github.com/apache/arrow.git" + + bottle do + cellar :any + sha256 "a55211ba6f464681b7ca1b48defdad9cfbe1cf6fad8ff9ec875dc5a3c8f3c5ed" => :el_capitan_or_later + root_url "https://autobrew.github.io/bottles" + end + + # NOTE: if you add something here, be sure to add to PKG_LIBS in r/tools/autobrew + depends_on "boost" => :build + depends_on "cmake" => :build + depends_on "aws-sdk-cpp" + depends_on "lz4" + depends_on "snappy" + depends_on "thrift" + depends_on "zstd" + + def install + ENV.cxx11 + args = %W[ + -DARROW_BUILD_SHARED=OFF + -DARROW_BUILD_UTILITIES=ON + -DARROW_COMPUTE=ON + -DARROW_CSV=ON + -DARROW_DATASET=ON + -DARROW_FILESYSTEM=ON + -DARROW_HDFS=OFF + -DARROW_JEMALLOC=ON + -DARROW_JSON=ON + -DARROW_MIMALLOC=ON + -DARROW_PARQUET=ON + -DARROW_PYTHON=OFF + -DARROW_S3=ON + -DARROW_USE_GLOG=OFF + -DARROW_VERBOSE_THIRDPARTY_BUILD=ON + -DARROW_WITH_LZ4=ON + -DARROW_WITH_SNAPPY=ON + -DARROW_WITH_ZLIB=ON + -DARROW_WITH_ZSTD=ON + -DCMAKE_UNITY_BUILD=OFF + -DPARQUET_BUILD_EXECUTABLES=ON + -DLZ4_HOME=#{Formula["lz4"].prefix} + -DTHRIFT_HOME=#{Formula["thrift"].prefix} + ] + + mkdir "build" + cd "build" do + system "cmake", "../cpp", *std_cmake_args, *args + system "make" + system "make", "install" + end + end + + test do + (testpath/"test.cpp").write <<~EOS + #include "arrow/api.h" + int main(void) { + arrow::int64(); + return 0; + } + EOS + system ENV.cxx, "test.cpp", "-std=c++11", "-I#{include}", "-L#{lib}", "-larrow", "-o", "test" + system "./test" + end +end diff --git a/src/arrow/dev/tasks/homebrew-formulae/github.macos.yml b/src/arrow/dev/tasks/homebrew-formulae/github.macos.yml new file mode 100644 index 000000000..232cc38a9 --- /dev/null +++ b/src/arrow/dev/tasks/homebrew-formulae/github.macos.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + autobrew: + name: "Autobrew" + runs-on: macOS-latest + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Configure homebrew formula for testing + env: + ARROW_FORMULA: ./arrow/dev/tasks/homebrew-formulae/{{ formula }} + run: | + # Pin the current commit in the formula to test so that we're not always pulling from master + sed -i.bak -E -e 's@https://github.com/apache/arrow.git"$@{{ arrow.remote }}.git", revision: "{{ arrow.head }}"@' $ARROW_FORMULA && rm -f $ARROW_FORMULA.bak + # Sometimes crossbow gives a remote URL with .git and sometimes not. Make sure there's only one + sed -i.bak -E -e 's@.git.git@.git@' $ARROW_FORMULA && rm -f $ARROW_FORMULA.bak + brew update + brew --version + brew unlink python@2 || true + brew config + brew doctor || true + cp $ARROW_FORMULA $(brew --repository homebrew/core)/Formula/apache-arrow.rb + - name: Test formula + run: | + brew install -v --HEAD apache-arrow + brew test apache-arrow + brew audit --strict apache-arrow diff --git a/src/arrow/dev/tasks/java-jars/README.md b/src/arrow/dev/tasks/java-jars/README.md new file mode 100644 index 000000000..1d61662d4 --- /dev/null +++ b/src/arrow/dev/tasks/java-jars/README.md @@ -0,0 +1,29 @@ +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +# Jars. + +This directory is responsible to generate the jar files for the Arrow components that depend on C++ shared libraries to execute. + +The Arrow C++ libraries are compiled both on MacOS and Linux distributions, with their dependencies linked statically, and they are added +in the jars at the end, so the file can be used on both systems. + +## Linux Docker Image +To compile the C++ libraries in Linux, a docker image is used. +It is created used the **ci/docker/java-bundled-jars.dockerfile** file. +If it is necessary to add any new dependency, you need to change that file.
\ No newline at end of file diff --git a/src/arrow/dev/tasks/java-jars/github.yml b/src/arrow/dev/tasks/java-jars/github.yml new file mode 100644 index 000000000..81d31dd4c --- /dev/null +++ b/src/arrow/dev/tasks/java-jars/github.yml @@ -0,0 +1,115 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + + build-cpp-ubuntu: + name: Build C++ Libs Ubuntu + runs-on: ubuntu-18.04 + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + - name: Build C++ Libs + run: archery docker run java-jni-manylinux-2014 + - name: Compress into single artifact + run: tar -cvzf arrow-shared-libs-linux.tar.gz arrow/java-dist/ + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: ubuntu-shared-lib + path: arrow-shared-libs-linux.tar.gz + {% if arrow.branch == 'master' %} + {{ macros.github_login_dockerhub()|indent }} + - name: Push Docker Image + shell: bash + run: archery docker push java-jni-manylinux-2014 + {% endif %} + + build-cpp-macos: + name: Build C++ Libs MacOS + runs-on: macos-latest + env: + MACOSX_DEPLOYMENT_TARGET: "10.11" + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + - name: Install Dependencies + run: brew bundle --file=arrow/cpp/Brewfile + - name: Build C Data Interface lib + run: | + set -e + arrow/ci/scripts/java_cdata_build.sh \ + $GITHUB_WORKSPACE/arrow \ + $GITHUB_WORKSPACE/arrow/java-native-build \ + $GITHUB_WORKSPACE/arrow/java-dist + - name: Build C++ Libs + run: | + set -e + arrow/ci/scripts/java_jni_macos_build.sh \ + $GITHUB_WORKSPACE/arrow \ + $GITHUB_WORKSPACE/arrow/cpp-build \ + $GITHUB_WORKSPACE/arrow/java-dist + - name: Compress into single artifact + run: tar -cvzf arrow-shared-libs-macos.tar.gz arrow/java-dist/ + - name: Upload Artifacts + uses: actions/upload-artifact@v2 + with: + name: macos-shared-lib + path: arrow-shared-libs-macos.tar.gz + + package-jars: + name: Build Jar Files + runs-on: macos-latest + needs: [build-cpp-macos, build-cpp-ubuntu] + steps: + {{ macros.github_checkout_arrow()|indent }} + - name: Download Linux C++ Libraries + uses: actions/download-artifact@v2 + with: + name: ubuntu-shared-lib + - name: Download MacOS C++ Library + uses: actions/download-artifact@v2 + with: + name: macos-shared-lib + - name: Descompress artifacts + run: | + tar -xvzf arrow-shared-libs-macos.tar.gz + tar -xvzf arrow-shared-libs-linux.tar.gz + - name: Test that Shared Libraries Exist + run: | + test -f arrow/java-dist/libarrow_cdata_jni.dylib + test -f arrow/java-dist/libarrow_dataset_jni.dylib + test -f arrow/java-dist/libgandiva_jni.dylib + test -f arrow/java-dist/libarrow_orc_jni.dylib + test -f arrow/java-dist/libarrow_cdata_jni.so + test -f arrow/java-dist/libarrow_dataset_jni.so + test -f arrow/java-dist/libarrow_orc_jni.so + test -f arrow/java-dist/libgandiva_jni.so + - name: Build Bundled Jar + run: | + set -e + pushd arrow/java + mvn versions:set -DnewVersion={{ arrow.no_rc_version }} + popd + arrow/ci/scripts/java_full_build.sh \ + $GITHUB_WORKSPACE/arrow \ + $GITHUB_WORKSPACE/arrow/java-dist + {{ macros.github_upload_releases(["arrow/java-dist/*.jar", "arrow/java-dist/*.pom"])|indent }} diff --git a/src/arrow/dev/tasks/linux-packages/.gitignore b/src/arrow/dev/tasks/linux-packages/.gitignore new file mode 100644 index 000000000..0e49a90c1 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/.gitignore @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +/*/*.tar.gz +/*/apt/repositories/ +/*/apt/tmp/ +/*/apt/build.sh +/*/apt/env.sh +/*/yum/repositories/ +/*/yum/tmp/ +/*/yum/build.sh +/*/yum/env.sh +/apt/repositories/ +/yum/repositories/ diff --git a/src/arrow/dev/tasks/linux-packages/README.md b/src/arrow/dev/tasks/linux-packages/README.md new file mode 100644 index 000000000..cafcc04ed --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/README.md @@ -0,0 +1,40 @@ +<!--- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +--> + +# Linux packages for Apache Arrow C++ and GLib + +## Requirements + + * Ruby + * Docker + * Tools to build tar.gz for Apache Arrow C++ and GLib + +## How to build .deb packages + +```console +% rake version:update +% rake apt +``` + +## How to build .rpm packages + +```console +% rake version:update +% rake yum +``` diff --git a/src/arrow/dev/tasks/linux-packages/Rakefile b/src/arrow/dev/tasks/linux-packages/Rakefile new file mode 100644 index 000000000..58cd981fe --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/Rakefile @@ -0,0 +1,249 @@ +# -*- ruby -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "English" + +require_relative "../../release/binary-task" +require_relative "helper" + +packages = [ + "apache-arrow", + "apache-arrow-apt-source", + "apache-arrow-release", +] + + +namespace :apt do + desc "Build deb packages" + task :build do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "apt:build") + end + end + end +end + +namespace :yum do + desc "Build RPM packages" + task :build do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "yum:build") + end + end + end +end + +namespace :version do + desc "Update versions" + task :update do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "version:update") + end + end + end +end + +namespace :docker do + desc "Pull built images" + task :pull do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "docker:pull") + end + end + end + + desc "Push built images" + task :push do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "docker:push") + end + end + end +end + + +class LocalBinaryTask < BinaryTask + include Helper::ApacheArrow + + def initialize(packages) + @packages = packages + super() + end + + def define + define_apt_test_task + define_yum_test_task + end + + private + def latest_commit_time(git_directory) + cd(git_directory) do + return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc + end + end + + def version + @version ||= detect_version(detect_release_time) + end + + def resolve_docker_image(target) + image = "" + target = target.gsub(/\Aamazon-linux/, "amazonlinux") + case target + when /-(?:arm64|aarch64)\z/ + target = $PREMATCH + image << "arm64v8/" + end + image << target.gsub(/-/, ":") + end + + def verify(target) + verify_command_line = [ + "docker", + "run", + "--rm", + "--log-driver", "none", + "--volume", "#{File.expand_path(arrow_source_dir)}:/arrow:delegated", + ] + if $stdin.tty? + verify_command_line << "--interactive" + verify_command_line << "--tty" + else + verify_command_line.concat(["--attach", "STDOUT"]) + verify_command_line.concat(["--attach", "STDERR"]) + end + verify_command_line << resolve_docker_image(target) + case target + when /\Adebian-/, /\Aubuntu-/ + verify_command_line << "/arrow/dev/release/verify-apt.sh" + else + verify_command_line << "/arrow/dev/release/verify-yum.sh" + end + verify_command_line << version + verify_command_line << "local" + sh(*verify_command_line) + end + + def apt_test_targets + targets = (ENV["APT_TARGETS"] || "").split(",") + targets = apt_test_targets_default if targets.empty? + targets + end + + def apt_test_targets_default + # Disable arm64 targets by default for now + # because they require some setups on host. + [ + "debian-buster", + # "debian-buster-arm64", + "debian-bullseye", + # "debian-bullseye-arm64", + "ubuntu-xenial", + # "ubuntu-xenial-arm64", + "ubuntu-bionic", + # "ubuntu-bionic-arm64", + "ubuntu-focal", + # "ubuntu-focal-arm64", + "ubuntu-hirsute", + # "ubuntu-hirsute-arm64", + "ubuntu-impish", + # "ubuntu-impish-arm64", + ] + end + + def define_apt_test_task + namespace :apt do + desc "Test deb packages" + task :test do + repositories_dir = "apt/repositories" + rm_rf(repositories_dir) + @packages.each do |package| + package_repositories = "#{package}/apt/repositories" + next unless File.exist?(package_repositories) + sh("rsync", "-a", "#{package_repositories}/", repositories_dir) + end + Dir.glob("#{repositories_dir}/ubuntu/pool/*") do |code_name_dir| + universe_dir = "#{code_name_dir}/universe" + next unless File.exist?(universe_dir) + mv(universe_dir, "#{code_name_dir}/main") + end + base_dir = "nonexistent" + merged_dir = "apt/merged" + apt_update(base_dir, repositories_dir, merged_dir) + Dir.glob("#{merged_dir}/*/dists/*") do |dists_code_name_dir| + prefix = dists_code_name_dir.split("/")[-3..-1].join("/") + mv(Dir.glob("#{dists_code_name_dir}/*Release*"), + "#{repositories_dir}/#{prefix}") + end + apt_test_targets.each do |target| + verify(target) + end + end + end + end + + def yum_test_targets + targets = (ENV["YUM_TARGETS"] || "").split(",") + targets = yum_test_targets_default if targets.empty? + targets + end + + def yum_test_targets_default + # Disable aarch64 targets by default for now + # because they require some setups on host. + [ + "almalinux-8", + # "almalinux-8-aarch64", + "amazon-linux-2", + # "amazon-linux-2-aarch64", + "centos-7", + "centos-8", + # "centos-8-aarch64", + ] + end + + def define_yum_test_task + namespace :yum do + desc "Test RPM packages" + task :test do + repositories_dir = "yum/repositories" + rm_rf(repositories_dir) + @packages.each do |package| + package_repositories = "#{package}/yum/repositories" + next unless File.exist?(package_repositories) + sh("rsync", "-a", "#{package_repositories}/", repositories_dir) + end + rpm_sign(repositories_dir) + base_dir = "nonexistent" + yum_update(base_dir, repositories_dir) + yum_test_targets.each do |target| + verify(target) + end + end + end + end +end + +local_binary_task = LocalBinaryTask.new(packages) +local_binary_task.define diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/Rakefile b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/Rakefile new file mode 100644 index 000000000..210fa951e --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/Rakefile @@ -0,0 +1,64 @@ +# -*- ruby -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative "../helper" +require_relative "../package-task" + +class ApacheArrowAptSourcePackageTask < PackageTask + include Helper::ApacheArrow + + def initialize + release_time = detect_release_time + super("apache-arrow-apt-source", + detect_version(release_time), + release_time, + :rc_build_type => :release) + end + + private + def define_archive_task + file @archive_name do + rm_rf(@archive_base_name) + mkdir(@archive_base_name) + download("https://downloads.apache.org/arrow/KEYS", + "#{@archive_base_name}/KEYS") + sh("tar", "czf", @archive_name, @archive_base_name) + rm_rf(@archive_base_name) + end + + if deb_archive_name != @archive_name + file deb_archive_name => @archive_name do + if @archive_base_name == deb_archive_base_name + cp(@archive_name, deb_archive_name) + else + sh("tar", "xf", @archive_name) + mv(@archive_base_name, deb_archive_base_name) + sh("tar", "czf", deb_archive_name, deb_archive_base_name) + end + end + end + end + + def enable_yum? + false + end +end + +task = ApacheArrowAptSourcePackageTask.new +task.define diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bookworm/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bookworm/Dockerfile new file mode 100644 index 000000000..f9541efde --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bookworm/Dockerfile @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM debian:bookworm + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + gnupg \ + lsb-release && \ + apt clean diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bullseye/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bullseye/Dockerfile new file mode 100644 index 000000000..b0842a0c0 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-bullseye/Dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM debian:bullseye + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + gnupg \ + lsb-release && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-buster/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-buster/Dockerfile new file mode 100644 index 000000000..0d37f5dee --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/debian-buster/Dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM debian:buster + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + gnupg \ + lsb-release && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-bionic/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-bionic/Dockerfile new file mode 100644 index 000000000..53e11fb7e --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-bionic/Dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:bionic + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + gnupg \ + lsb-release && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-focal/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-focal/Dockerfile new file mode 100644 index 000000000..dc902d14d --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-focal/Dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:focal + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + gnupg \ + lsb-release && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-hirsute/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-hirsute/Dockerfile new file mode 100644 index 000000000..8b6fd7f0e --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-hirsute/Dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:hirsute + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + gnupg \ + lsb-release && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-impish/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-impish/Dockerfile new file mode 100644 index 000000000..640cd715d --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/apt/ubuntu-impish/Dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu:impish + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + debhelper \ + devscripts \ + fakeroot \ + gnupg \ + lsb-release && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install new file mode 100644 index 000000000..7bcb2ecc9 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/apache-arrow-apt-source.install @@ -0,0 +1,2 @@ +etc/apt/sources.list.d/* +usr/share/keyrings/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog new file mode 100644 index 000000000..27e14c802 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/changelog @@ -0,0 +1,11 @@ +apache-arrow-apt-source (6.0.1-1) unstable; urgency=low + + * New upstream release. + + -- Sutou Kouhei <kou@clear-code.com> Wed, 10 Nov 2021 02:04:45 -0000 + +apache-arrow-apt-source (6.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Krisztián Szűcs <szucs.krisztian@gmail.com> Thu, 21 Oct 2021 19:01:54 -0000 diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat new file mode 100644 index 000000000..ec635144f --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/compat @@ -0,0 +1 @@ +9 diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/control b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/control new file mode 100644 index 000000000..f54d52f98 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/control @@ -0,0 +1,23 @@ +Source: apache-arrow-apt-source +Section: misc +Priority: important +Maintainer: Apache Arrow Developers <dev@arrow.apache.org> +Build-Depends: + debhelper (>= 9), + gnupg, + lsb-release +Standards-Version: 3.9.7 +Homepage: https://arrow.apache.org/ + +Package: apache-arrow-apt-source +Section: misc +Architecture: all +Replaces: apache-arrow-archive-keyring +Breaks: apache-arrow-archive-keyring +Depends: + ${misc:Depends}, + apt-transport-https, + gnupg +Description: GnuPG archive key of the Apache Arrow archive + The Apache Arrow project digitally signs its Release files. This + package contains the archive key used for that. diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/copyright b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/copyright new file mode 100644 index 000000000..274d64ca0 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/copyright @@ -0,0 +1,26 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: Apache Arrow +Upstream-Contact: Apache Arrow Developers <dev@arrow.apache.org> + +Files: * +Copyright: 2016 The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the full text of the Apache Software License version 2 can + be found in the file `/usr/share/common-licenses/Apache-2.0'. diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules new file mode 100755 index 000000000..bf7a85c8c --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/rules @@ -0,0 +1,37 @@ +#!/usr/bin/make -f +# -*- makefile-gmake -*- +# +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 +# This has to be exported to make some magic below work. +export DH_OPTIONS + +%: + dh $@ + +override_dh_auto_build: + gpg \ + --no-default-keyring \ + --keyring ./apache-arrow-apt-source.gpg \ + --import KEYS + + ( \ + distribution=$$(lsb_release --id --short | tr 'A-Z' 'a-z'); \ + code_name=$$(lsb_release --codename --short); \ + echo "Types: deb deb-src"; \ + echo "URIs: https://apache.jfrog.io/artifactory/arrow/$${distribution}/"; \ + echo "Suites: $${code_name}"; \ + echo "Components: main"; \ + echo "Signed-By: /usr/share/keyrings/apache-arrow-apt-source.gpg"; \ + ) > apache-arrow.sources + +override_dh_install: + install -d debian/tmp/usr/share/keyrings/ + install -m 0644 apache-arrow-apt-source.gpg \ + debian/tmp/usr/share/keyrings/ + + install -d debian/tmp/etc/apt/sources.list.d/ + install -m 0644 apache-arrow.sources \ + debian/tmp/etc/apt/sources.list.d/ + + dh_install diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format new file mode 100644 index 000000000..163aaf8d8 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-apt-source/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-release/Rakefile b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/Rakefile new file mode 100644 index 000000000..4a341c6f1 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/Rakefile @@ -0,0 +1,66 @@ +# -*- ruby -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative "../helper" +require_relative "../package-task" + +class ApacheArrowReleasePackageTask < PackageTask + include Helper::ApacheArrow + + def initialize + release_time = detect_release_time + super("apache-arrow-release", + detect_version(release_time), + release_time, + :rc_build_type => :release) + end + + private + def repo_path + "#{yum_dir}/Apache-Arrow.repo" + end + + def define_archive_task + file @archive_name => [repo_path] do + rm_rf(@archive_base_name) + mkdir(@archive_base_name) + download("https://downloads.apache.org/arrow/KEYS", + "#{@archive_base_name}/KEYS") + cp(repo_path, @archive_base_name) + sh("tar", "czf", @archive_name, @archive_base_name) + rm_rf(@archive_base_name) + end + + if rpm_archive_name != @archive_name + file rpm_archive_name => @archive_name do + sh("tar", "xf", @archive_name) + rpm_archive_base_name = File.basename(rpm_archive_name, ".tar.gz") + mv(@archive_base_name, rpm_archive_base_name) + sh("tar", "czf", rpm_archive_name, rpm_archive_base_name) + end + end + end + + def enable_apt? + false + end +end + +task = ApacheArrowReleasePackageTask.new +task.define diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo new file mode 100644 index 000000000..0d1666520 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/Apache-Arrow.repo @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[apache-arrow-almalinux] +name=Apache Arrow for AlmaLinux $releasever - $basearch +baseurl=https://apache.jfrog.io/artifactory/arrow/almalinux/$releasever/$basearch/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow + +[apache-arrow-amazon-linux] +name=Apache Arrow for Amazon Linux $releasever - $basearch +baseurl=https://apache.jfrog.io/artifactory/arrow/amazon-linux/$releasever/$basearch/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow + +[apache-arrow-centos] +name=Apache Arrow for CentOS $releasever - $basearch +baseurl=https://apache.jfrog.io/artifactory/arrow/centos/$releasever/$basearch/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow + +[apache-arrow-rhel] +name=Apache Arrow for RHEL $releasever - $basearch +baseurl=https://apache.jfrog.io/artifactory/arrow/centos/$releasever/$basearch/ +gpgcheck=1 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/almalinux-8/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/almalinux-8/Dockerfile new file mode 100644 index 000000000..5be1edc3d --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/almalinux-8/Dockerfile @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM almalinux:8 + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + dnf install --enablerepo=powertools -y ${quiet} \ + rpmdevtools && \ + dnf clean ${quiet} all diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/amazon-linux-2/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/amazon-linux-2/Dockerfile new file mode 100644 index 000000000..800df6c68 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/amazon-linux-2/Dockerfile @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM amazonlinux:2 + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + yum install -y ${quiet} \ + rpmdevtools && \ + yum clean ${quiet} all diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in new file mode 100644 index 000000000..594cda373 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/apache-arrow-release.spec.in @@ -0,0 +1,113 @@ +# -*- sh-shell: rpm -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +%define use_dnf (%{rhel} >= 8) +%if %{use_dnf} +%define yum_repository_enable() (dnf config-manager --set-enabled %1) +%define yum_repository_disable() (dnf config-manager --set-disabled %1) +%else +%define yum_repository_enable() (yum-config-manager --enable %1) +%define yum_repository_disable() (yum-config-manager --disable %1) +%endif + +Name: @PACKAGE@ +Version: @VERSION@ +Release: @RELEASE@%{?dist} +Summary: Apache Arrow release files + +License: Apache-2.0 +URL: https://arrow.apache.org/ +Source0: @PACKAGE@-%{version}.tar.gz + +BuildArch: noarch + +Requires: epel-release +%if %{use_dnf} +Requires: dnf-command(config-manager) +%else +Requires: yum-utils +%endif + +%description +Apache Arrow release files. + +%prep +%setup -q + +%build +distribution=$(. /etc/os-release && echo "${ID}") +if [ "${distribution}" = "rhel" ]; then + # We use distribution version explicitly for RHEL because we can't + # use symbolic link on Artifactory. CentOS uses 7 and 8 but RHEL uses + # 7Server and 8Server for $releasever. If we can use symbolic link + # on Artifactory we can use $releasever directly. + distribution_version=$(. /etc/os-release && echo "${VERSION_ID}") + sed -i'' -e "s/\\\$releasever/${distribution_version}/g" Apache-Arrow.repo +fi + +%install +rm -rf $RPM_BUILD_ROOT + +%{__install} -Dp -m0644 KEYS \ + $RPM_BUILD_ROOT%{_sysconfdir}/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow + +%{__install} -d $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d/ +%{__install} -Dp -m0644 Apache-Arrow.repo \ + $RPM_BUILD_ROOT%{_sysconfdir}/yum.repos.d/Apache-Arrow.repo + +%files +%defattr(-, root, root, 0755) +%doc +%dir %{_sysconfdir}/yum.repos.d/ +%dir %{_sysconfdir}/pki/rpm-gpg/ +%{_sysconfdir}/pki/rpm-gpg/RPM-GPG-KEY-Apache-Arrow +%config(noreplace) %{_sysconfdir}/yum.repos.d/Apache-Arrow.repo + +%post +if grep -q 'Amazon Linux release 2' /etc/system-release 2>/dev/null; then + %{yum_repository_enable apache-arrow-amazon-linux} +elif grep -q 'Red Hat Enterprise Linux' /etc/system-release 2>/dev/null; then + %{yum_repository_enable apache-arrow-rhel} +elif grep -q 'AlmaLinux' /etc/system-release 2>/dev/null; then + %{yum_repository_enable apache-arrow-almalinux} +else + %{yum_repository_enable apache-arrow-centos} +fi + +%changelog +* Wed Nov 10 2021 Sutou Kouhei <kou@clear-code.com> - 6.0.1-1 +- New upstream release. + +* Thu Oct 21 2021 Krisztián Szűcs <szucs.krisztian@gmail.com> - 6.0.0-1 +- New upstream release. + +* Mon Jan 18 2021 Krisztián Szűcs <szucs.krisztian@gmail.com> - 3.0.0-1 +- New upstream release. + +* Mon Oct 12 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 2.0.0-1 +- New upstream release. + +* Mon Jul 20 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 1.0.0-1 +- New upstream release. + +* Thu Apr 16 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 0.17.0-1 +- New upstream release. + +* Thu Jan 30 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 0.16.0-1 +- New upstream release. diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/centos-7/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/centos-7/Dockerfile new file mode 100644 index 000000000..236b0e297 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/centos-7/Dockerfile @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM centos:7 + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + yum install -y ${quiet} \ + rpmdevtools && \ + yum clean ${quiet} all diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/centos-8/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/centos-8/Dockerfile new file mode 100644 index 000000000..e36850630 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow-release/yum/centos-8/Dockerfile @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM centos:8 + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + dnf install --enablerepo=powertools -y ${quiet} \ + rpmdevtools && \ + dnf clean ${quiet} all diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/Rakefile b/src/arrow/dev/tasks/linux-packages/apache-arrow/Rakefile new file mode 100644 index 000000000..9dfae955e --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/Rakefile @@ -0,0 +1,162 @@ +# -*- ruby -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative "../helper" +require_relative "../package-task" + +class ApacheArrowPackageTask < PackageTask + include Helper::ApacheArrow + + def initialize + release_time = detect_release_time + super("apache-arrow", + detect_version(release_time), + release_time, + :rc_build_type => :release) + @rpm_package = "arrow" + end + + private + def define_archive_task + file @archive_name do + case @version + when /\A\d+\.\d+\.\d+-rc\d+\z/ + download_rc_archive + when /\A\d+\.\d+\.\d+\z/ + download_released_archive + else + build_archive + end + end + + if deb_archive_name != @archive_name + file deb_archive_name => @archive_name do + cp(@archive_name, deb_archive_name) + end + end + + if rpm_archive_name != @archive_name + file rpm_archive_name => @archive_name do + cp(@archive_name, rpm_archive_name) + end + end + end + + def download_rc_archive + base_url = "https://dist.apache.org/repos/dist/dev/arrow" + archive_name_no_rc = @archive_name.gsub(/-rc\d+(\.tar\.gz)\z/, "\\1") + url = "#{base_url}/#{@package}-#{@version}/#{archive_name_no_rc}" + download(url, @archive_name) + end + + def download_released_archive + mirror_base_url = "https://www.apache.org/dyn/closer.lua/arrow" + mirror_list_url = "#{mirror_base_url}/arrow-#{@version}/#{@archive_name}" + open(mirror_list_url) do |response| + if /href="(.+?\/#{Regexp.escape(@archive_name)})"/ =~ response.read + download($1, ".") + end + end + end + + def build_archive + cd(arrow_source_dir) do + sh("git", "archive", "HEAD", + "--prefix", "#{@archive_base_name}/", + "--output", @full_archive_name) + end + end + + def apt_arm64_cuda_available_target?(target) + false + end + + def apt_prepare_debian_control_cuda_architecture(control, target) + if apt_arm64_cuda_available_target?(target) + cuda_architecture = "any" + else + cuda_architecture = "i386 amd64" + end + control.gsub(/@CUDA_ARCHITECTURE@/, cuda_architecture) + end + + def apt_prepare_debian_control_c_ares(control, target) + case target + when /\Aubuntu-bionic/ + use_system_c_ares = "#" + else + use_system_c_ares = "" + end + control.gsub(/@USE_SYSTEM_C_ARES@/, use_system_c_ares) + end + + def apt_prepare_debian_control_grpc(control, target) + case target + when /\Adebian-buster/, /\Aubuntu-(?:bionic|focal)/ + use_system_grpc = "#" + else + use_system_grpc = "" + end + control.gsub(/@USE_SYSTEM_GRPC@/, use_system_grpc) + end + + def apt_prepare_debian_control_thrift(control, target) + case target + when /\Aubuntu-bionic/ + use_system_thrift = "#" + else + use_system_thrift = "" + end + control.gsub(/@USE_SYSTEM_THRIFT@/, use_system_thrift) + end + + def apt_prepare_debian_control_utf8proc(control, target) + case target + when /\Aubuntu-bionic/ + use_system_utf8proc = "#" + else + use_system_utf8proc = "" + end + control.gsub(/@USE_SYSTEM_UTF8PROC@/, use_system_utf8proc) + end + + def apt_prepare_debian_control_zstd(control, target) + case target + when /\Adebian-buster/, /\Aubuntu-bionic/ + use_system_zstd = "#" + else + use_system_zstd = "" + end + control.gsub(/@USE_SYSTEM_ZSTD@/, use_system_zstd) + end + + def apt_prepare_debian_control(control_in, target) + control = control_in.dup + control = apt_prepare_debian_control_cuda_architecture(control, target) + control = apt_prepare_debian_control_c_ares(control, target) + control = apt_prepare_debian_control_grpc(control, target) + control = apt_prepare_debian_control_thrift(control, target) + control = apt_prepare_debian_control_utf8proc(control, target) + control = apt_prepare_debian_control_zstd(control, target) + control + end +end + +task = ApacheArrowPackageTask.new +task.define diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm-arm64/from b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm-arm64/from new file mode 100644 index 000000000..022eaf290 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm-arm64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/debian:bookworm diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile new file mode 100644 index 000000000..e88ed53ff --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bookworm/Dockerfile @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=debian:bookworm +FROM ${FROM} + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +RUN sed -i'' -e 's/main$/main contrib non-free/g' /etc/apt/sources.list + +ARG DEBUG +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + ccache \ + clang \ + cmake \ + debhelper \ + devscripts \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgmock-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + libgtest-dev \ + liblz4-dev \ + libprotoc-dev \ + libprotobuf-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + llvm-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + protobuf-compiler-grpc \ + python3-dev \ + python3-numpy \ + python3-pip \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + if apt list | grep '^nvidia-cuda-toolkit/'; then \ + apt install -y -V ${quiet} nvidia-cuda-toolkit; \ + fi && \ + pip3 install --upgrade meson && \ + ln -s /usr/local/bin/meson /usr/bin/ && \ + apt clean diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye-arm64/from b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye-arm64/from new file mode 100644 index 000000000..34187b2af --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye-arm64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/debian:bullseye diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile new file mode 100644 index 000000000..2a5b18f5b --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-bullseye/Dockerfile @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=debian:bullseye +FROM ${FROM} + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +RUN sed -i'' -e 's/main$/main contrib non-free/g' /etc/apt/sources.list + +ARG DEBUG +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + ccache \ + clang \ + cmake \ + debhelper \ + devscripts \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgmock-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + libgtest-dev \ + liblz4-dev \ + libprotoc-dev \ + libprotobuf-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + llvm-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + protobuf-compiler-grpc \ + python3-dev \ + python3-numpy \ + python3-pip \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + if apt list | grep '^nvidia-cuda-toolkit/'; then \ + apt install -y -V ${quiet} nvidia-cuda-toolkit; \ + fi && \ + pip3 install --upgrade meson && \ + ln -s /usr/local/bin/meson /usr/bin/ && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-buster-arm64/from b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-buster-arm64/from new file mode 100644 index 000000000..8da222b86 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-buster-arm64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/debian:buster diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile new file mode 100644 index 000000000..11a33a130 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/debian-buster/Dockerfile @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=debian:buster +FROM ${FROM} + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +RUN sed -i'' -e 's/main$/main contrib non-free/g' /etc/apt/sources.list + +RUN \ + echo "deb http://deb.debian.org/debian buster-backports main" > \ + /etc/apt/sources.list.d/backports.list + +ARG DEBUG +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + ccache \ + cmake \ + debhelper \ + devscripts \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgmock-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + liblz4-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + python3-dev \ + python3-numpy \ + python3-pip \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + apt install -y -V -t buster-backports ${quiet} \ + clang-11 \ + llvm-11-dev && \ + if apt list | grep '^nvidia-cuda-toolkit/'; then \ + apt install -y -V ${quiet} nvidia-cuda-toolkit; \ + fi && \ + pip3 install --upgrade meson && \ + ln -s /usr/local/bin/meson /usr/bin/ && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic-arm64/from b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic-arm64/from new file mode 100644 index 000000000..c3ba00cf0 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic-arm64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/ubuntu:bionic diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile new file mode 100644 index 000000000..ac0c6a58d --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-bionic/Dockerfile @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=ubuntu:bionic +FROM ${FROM} + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + ccache \ + clang-10 \ + cmake \ + devscripts \ + fakeroot \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libcurl4-openssl-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgoogle-glog-dev \ + libgtest-dev \ + liblz4-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + llvm-10-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + python3-dev \ + python3-numpy \ + python3-pip \ + python3-setuptools \ + python3-wheel \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + (echo "includedir=/usr/include" && \ + echo "libdir=/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)" && \ + echo "" && \ + echo "Name: re2" && \ + echo "Description: RE2 is a fast, safe, thread-friendly regular expression engine." && \ + echo "Version: 0.0.0" && \ + echo "Cflags: -std=c++11 -pthread -I\${includedir}" && \ + echo "Libs: -pthread -L\${libdir} -lre2") | \ + tee "/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/pkgconfig/re2.pc" && \ + if apt list | grep '^nvidia-cuda-toolkit/'; then \ + apt install -y -V ${quiet} nvidia-cuda-toolkit; \ + fi && \ + apt install -y -V -t bionic-backports ${quiet} \ + debhelper && \ + pip3 install --upgrade meson && \ + ln -s /usr/local/bin/meson /usr/bin/ && \ + apt clean && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal-arm64/from b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal-arm64/from new file mode 100644 index 000000000..52ab48b66 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal-arm64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/ubuntu:focal diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile new file mode 100644 index 000000000..112cc1846 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-focal/Dockerfile @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=ubuntu:focal +FROM ${FROM} + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + ccache \ + clang \ + cmake \ + debhelper \ + devscripts \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgmock-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + liblz4-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + llvm-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + python3-dev \ + python3-numpy \ + python3-pip \ + python3-setuptools \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + if apt list | grep '^nvidia-cuda-toolkit/'; then \ + apt install -y -V ${quiet} nvidia-cuda-toolkit; \ + fi && \ + apt clean && \ + python3 -m pip install --no-use-pep517 meson && \ + ln -s /usr/local/bin/meson /usr/bin/ && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-hirsute-arm64/from b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-hirsute-arm64/from new file mode 100644 index 000000000..f19ea9022 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-hirsute-arm64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/ubuntu:hirsute diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-hirsute/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-hirsute/Dockerfile new file mode 100644 index 000000000..7e26d3eb2 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-hirsute/Dockerfile @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=ubuntu:hirsute +FROM ${FROM} + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + ccache \ + clang \ + cmake \ + debhelper \ + devscripts \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgmock-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + libgtest-dev \ + liblz4-dev \ + libprotoc-dev \ + libprotobuf-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + llvm-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + protobuf-compiler-grpc \ + python3-dev \ + python3-numpy \ + python3-pip \ + python3-setuptools \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + if apt list | grep -q '^libcuda1'; then \ + apt install -y -V ${quiet} nvidia-cuda-toolkit; \ + else \ + :; \ + fi && \ + apt clean && \ + python3 -m pip install --no-use-pep517 meson && \ + ln -s /usr/local/bin/meson /usr/bin/ && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish-arm64/from b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish-arm64/from new file mode 100644 index 000000000..0c8a1c7a1 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish-arm64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/ubuntu:impish diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish/Dockerfile new file mode 100644 index 000000000..2e94dd9b6 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/apt/ubuntu-impish/Dockerfile @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=ubuntu:impish +FROM ${FROM} + +RUN \ + echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN \ + echo 'APT::Install-Recommends "false";' > \ + /etc/apt/apt.conf.d/disable-install-recommends + +ARG DEBUG +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "-qq") && \ + apt update ${quiet} && \ + apt install -y -V ${quiet} \ + build-essential \ + ccache \ + clang \ + cmake \ + debhelper \ + devscripts \ + git \ + gtk-doc-tools \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libgmock-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + libgtest-dev \ + liblz4-dev \ + libprotoc-dev \ + libprotobuf-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + llvm-dev \ + lsb-release \ + ninja-build \ + pkg-config \ + protobuf-compiler-grpc \ + python3-dev \ + python3-numpy \ + python3-pip \ + python3-setuptools \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + if apt list | grep -q '^libcuda1'; then \ + apt install -y -V ${quiet} nvidia-cuda-toolkit; \ + else \ + :; \ + fi && \ + apt clean && \ + python3 -m pip install --no-use-pep517 meson && \ + ln -s /usr/local/bin/meson /usr/bin/ && \ + rm -rf /var/lib/apt/lists/* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/changelog b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/changelog new file mode 100644 index 000000000..65948629e --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/changelog @@ -0,0 +1,123 @@ +apache-arrow (6.0.1-1) unstable; urgency=low + + * New upstream release. + + -- Sutou Kouhei <kou@clear-code.com> Wed, 10 Nov 2021 02:04:45 -0000 + +apache-arrow (6.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Krisztián Szűcs <szucs.krisztian@gmail.com> Thu, 21 Oct 2021 19:01:54 -0000 + +apache-arrow (3.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Krisztián Szűcs <szucs.krisztian@gmail.com> Mon, 18 Jan 2021 21:33:18 -0000 + +apache-arrow (2.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Krisztián Szűcs <szucs.krisztian@gmail.com> Mon, 12 Oct 2020 23:38:01 -0000 + +apache-arrow (1.0.0-1) unstable; urgency=low + + * New upstream release. + + -- Krisztián Szűcs <szucs.krisztian@gmail.com> Mon, 20 Jul 2020 20:41:07 -0000 + +apache-arrow (0.17.0-1) unstable; urgency=low + + * New upstream release. + + -- Krisztián Szűcs <szucs.krisztian@gmail.com> Thu, 16 Apr 2020 12:05:43 -0000 + +apache-arrow (0.16.0-1) unstable; urgency=low + + * New upstream release. + + -- Krisztián Szűcs <szucs.krisztian@gmail.com> Thu, 30 Jan 2020 20:21:44 -0000 + +apache-arrow (0.15.0-1) unstable; urgency=low + + * New upstream release. + + -- Krisztián Szűcs <szucs.krisztian@gmail.com> Mon, 30 Sep 2019 17:19:02 -0000 + +apache-arrow (0.14.0-1) unstable; urgency=low + + * New upstream release. + + -- Sutou Kouhei <kou@clear-code.com> Fri, 28 Jun 2019 22:22:35 -0000 + +apache-arrow (0.13.0-1) unstable; urgency=low + + * New upstream release. + + -- Kouhei Sutou <kou@clear-code.com> Thu, 28 Mar 2019 02:24:58 -0000 + +apache-arrow (0.12.0-1) unstable; urgency=low + + * New upstream release. + + -- Krisztián Szűcs <szucs.krisztian@gmail.com> Wed, 16 Jan 2019 03:29:25 -0000 + +apache-arrow (0.11.0-1) unstable; urgency=low + + * New upstream release. + + -- Kouhei Sutou <kou@clear-code.com> Thu, 04 Oct 2018 00:33:42 -0000 + +apache-arrow (0.10.0-1) unstable; urgency=low + + * New upstream release. + + -- Phillip Cloud <cpcloud@gmail.com> Thu, 02 Aug 2018 23:58:23 -0000 + +apache-arrow (0.9.0-1) unstable; urgency=low + + * New upstream release. + + -- Kouhei Sutou <kou@clear-code.com> Fri, 16 Mar 2018 16:56:31 -0000 + +apache-arrow (0.8.0-1) unstable; urgency=low + + * New upstream release. + + * Add libarrow-gpu-glib0, libarrow-gpu-glib-dev and gir1.2-arrow-gpu-1.0. + + -- Uwe L. Korn <uwelk@xhochy.com> Sun, 17 Dec 2017 20:24:44 -0000 + +apache-arrow (0.7.1-2) unstable; urgency=low + + * Add libarrow-gpu0 and libarrow-gpu-dev. + + * Add libarrow-python-dev. + + -- Kouhei Sutou <kou@clear-code.com> Sun, 29 Oct 2017 21:59:13 +0900 + +apache-arrow (0.7.1-1) unstable; urgency=low + + * New upstream release. + + -- Kouhei Sutou <kou@clear-code.com> Wed, 27 Sep 2017 13:19:05 -0000 + +apache-arrow (0.7.0-1) unstable; urgency=low + + * New upstream release. + + -- Wes McKinney <wes.mckinney@twosigma.com> Tue, 12 Sep 2017 22:01:14 -0000 + +apache-arrow (0.6.0-1) unstable; urgency=low + + * New upstream release. + + -- Kouhei Sutou <kou@clear-code.com> Fri, 11 Aug 2017 21:27:51 -0000 + +apache-arrow (0.6.0.20170802-1) unstable; urgency=low + + * New upstream release. + + -- Kouhei Sutou <kou@clear-code.com> Wed, 02 Aug 2017 22:28:18 -0000 diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/compat b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/compat new file mode 100644 index 000000000..48082f72f --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/compat @@ -0,0 +1 @@ +12 diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/control.in b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/control.in new file mode 100644 index 000000000..6b0332ed2 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/control.in @@ -0,0 +1,640 @@ +Source: apache-arrow +Section: devel +Priority: optional +Maintainer: Apache Arrow Developers <dev@arrow.apache.org> +Build-Depends: + cmake, + debhelper (>= 12), + git, + gobject-introspection, + gtk-doc-tools, + libboost-filesystem-dev, + libboost-system-dev, + libbrotli-dev, + libbz2-dev, +@USE_SYSTEM_C_ARES@ libc-ares-dev, + libcurl4-openssl-dev, + libgirepository1.0-dev, + libgoogle-glog-dev, +@USE_SYSTEM_GRPC@ libgrpc++-dev, + libgtest-dev, + liblz4-dev, +@USE_SYSTEM_GRPC@ libprotoc-dev, +@USE_SYSTEM_GRPC@ libprotobuf-dev, + libre2-dev, + libsnappy-dev, + libssl-dev, +@USE_SYSTEM_THRIFT@ libthrift-dev, +@USE_SYSTEM_UTF8PROC@ libutf8proc-dev, +@USE_SYSTEM_ZSTD@ libzstd-dev, + ninja-build, + nvidia-cuda-toolkit [!arm64], + pkg-config, +@USE_SYSTEM_GRPC@ protobuf-compiler-grpc, + python3-dev, + python3-numpy, + tzdata, + zlib1g-dev +Build-Depends-Indep: libglib2.0-doc +Standards-Version: 3.9.8 +Homepage: https://arrow.apache.org/ + +Package: libarrow600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends} +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files. + +Package: libarrow-cuda600 +Section: libs +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files for CUDA support. + +Package: libarrow-dataset600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow600 (= ${binary:Version}), + libparquet600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files for Dataset module. + +Package: libarrow-flight600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files for Flight RPC system. + +Package: libarrow-python600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow600 (= ${binary:Version}), + python3, + python3-numpy +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files for Python support. + +Package: libarrow-python-flight600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow-flight600 (= ${binary:Version}), + libarrow-python600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ library files for Flight and Python support. + +Package: libarrow-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow600 (= ${binary:Version}), + libbrotli-dev, + libbz2-dev, +@USE_SYSTEM_C_ARES@ libc-ares-dev, +@USE_SYSTEM_GRPC@ libgrpc++-dev, + liblz4-dev, + libre2-dev, + libsnappy-dev, + libssl-dev, +@USE_SYSTEM_THRIFT@ libthrift-dev, +@USE_SYSTEM_UTF8PROC@ libutf8proc-dev, +@USE_SYSTEM_ZSTD@ libzstd-dev, +@USE_SYSTEM_GRPC@ protobuf-compiler-grpc, + zlib1g-dev +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ header files. + +Package: libarrow-cuda-dev +Section: libdevel +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-dev (= ${binary:Version}), + libarrow-cuda600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ header files for CUDA support. + +Package: libarrow-dataset-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-dev (= ${binary:Version}), + libarrow-dataset600 (= ${binary:Version}), + libparquet-dev (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ header files for dataset module. + +Package: libarrow-flight-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-dev (= ${binary:Version}), + libarrow-flight600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ header files for Flight RPC system. + +Package: libarrow-python-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-dev (= ${binary:Version}), + libarrow-python600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ header files for Python support. + +Package: libarrow-python-flight-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-flight-dev (= ${binary:Version}), + libarrow-python-dev (= ${binary:Version}), + libarrow-python-flight600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides C++ header files for Flight and Python support. + +Package: libgandiva600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow600 (= ${binary:Version}) +Description: Gandiva is a toolset for compiling and evaluating expressions + on Arrow Data. + . + This package provides C++ library files. + +Package: libgandiva-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-dev (= ${binary:Version}), + libgandiva600 (= ${binary:Version}) +Description: Gandiva is a toolset for compiling and evaluating expressions + on Arrow Data. + . + This package provides C++ header files. + +Package: libplasma600 +Section: libs +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow-cuda600 (= ${binary:Version}) +Description: Plasma is an in-memory object store and cache for big data. + . + This package provides C++ library files to connect plasma-store-server. + +Package: plasma-store-server +Section: utils +Architecture: @CUDA_ARCHITECTURE@ +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libplasma600 (= ${binary:Version}) +Description: Plasma is an in-memory object store and cache for big data. + . + This package provides plasma-store-server. + +Package: libplasma-dev +Section: libdevel +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-cuda-dev (= ${binary:Version}), + libplasma600 (= ${binary:Version}) +Description: Plasma is an in-memory object store and cache for big data. + . + This package provides C++ header files. + +Package: libparquet600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends} +Description: Apache Parquet is a columnar storage format + . + This package provides C++ library files to process Apache Parquet format. + +Package: libparquet-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-dev (= ${binary:Version}), + libparquet600 (= ${binary:Version}) +Description: Apache Parquet is a columnar storage format + . + This package provides C++ header files. + +Package: libarrow-glib600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides GLib based library files. + +Package: gir1.2-arrow-1.0 +Section: introspection +Architecture: any +Multi-Arch: same +Depends: + ${gir:Depends}, + ${misc:Depends} +Description: Apache Arrow is a data processing library for analysis + . + This package provides GObject Introspection typelib files. + +Package: libarrow-glib-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libglib2.0-dev, + libarrow-dev (= ${binary:Version}), + libarrow-glib600 (= ${binary:Version}), + gir1.2-arrow-1.0 (= ${binary:Version}) +Suggests: libarrow-glib-doc +Description: Apache Arrow is a data processing library for analysis + . + This package provides GLib based header files. + +Package: libarrow-glib-doc +Section: doc +Architecture: all +Multi-Arch: foreign +Depends: + ${misc:Depends} +Recommends: libglib2.0-doc +Description: Apache Arrow is a data processing library for analysis + . + This package provides documentations. + +Package: libarrow-cuda-glib600 +Section: libs +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow-glib600 (= ${binary:Version}), + libarrow-cuda600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides GLib based library files for CUDA support. + +Package: gir1.2-arrow-cuda-1.0 +Section: introspection +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Depends: + ${gir:Depends}, + ${misc:Depends} +Description: Apache Arrow is a data processing library for analysis + . + This package provides GObject Introspection typelib files for CUDA support. + +Package: libarrow-cuda-glib-dev +Section: libdevel +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-cuda-dev (= ${binary:Version}), + libarrow-glib-dev (= ${binary:Version}), + libarrow-cuda-glib600 (= ${binary:Version}), + gir1.2-arrow-cuda-1.0 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides GLib based header files for CUDA support. + +Package: libarrow-dataset-glib600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow-glib600 (= ${binary:Version}), + libarrow-dataset600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides GLib based library files for dataset module. + +Package: gir1.2-arrow-dataset-1.0 +Section: introspection +Architecture: any +Multi-Arch: same +Depends: + ${gir:Depends}, + ${misc:Depends} +Description: Apache Arrow is a data processing library for analysis + . + This package provides GObject Introspection typelib files for dataset module. + +Package: libarrow-dataset-glib-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-dataset-dev (= ${binary:Version}), + libarrow-glib-dev (= ${binary:Version}), + libarrow-dataset-glib600 (= ${binary:Version}), + gir1.2-arrow-dataset-1.0 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides GLib based header files for dataset module. + +Package: libarrow-dataset-glib-doc +Section: doc +Architecture: any +Multi-Arch: foreign +Depends: + ${misc:Depends} +Recommends: libarrow-glib-doc +Description: Apache Arrow is a data processing library for analysis + . + This package provides documentations for dataset module. + +Package: libarrow-flight-glib600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow-glib600 (= ${binary:Version}), + libarrow-flight600 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides GLib based library files for Apache Arrow Flight. + +Package: gir1.2-arrow-flight-1.0 +Section: introspection +Architecture: any +Multi-Arch: same +Depends: + ${gir:Depends}, + ${misc:Depends} +Description: Apache Arrow is a data processing library for analysis + . + This package provides GObject Introspection typelib files for Apache Arrow + Flight. + +Package: libarrow-flight-glib-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-flight-dev (= ${binary:Version}), + libarrow-glib-dev (= ${binary:Version}), + libarrow-flight-glib600 (= ${binary:Version}), + gir1.2-arrow-flight-1.0 (= ${binary:Version}) +Description: Apache Arrow is a data processing library for analysis + . + This package provides GLib based header files for Apache Arrow Flight. + +Package: libarrow-flight-glib-doc +Section: doc +Architecture: any +Multi-Arch: foreign +Depends: + ${misc:Depends} +Recommends: libarrow-glib-doc +Description: Apache Arrow is a data processing library for analysis + . + This package provides documentations for Apache Arrow Flight. + +Package: libgandiva-glib600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow-glib600 (= ${binary:Version}), + libgandiva600 (= ${binary:Version}) +Description: Gandiva is a toolset for compiling and evaluating expressions + on Arrow Data. + . + This package provides GLib based library files. + +Package: gir1.2-gandiva-1.0 +Section: introspection +Architecture: any +Multi-Arch: same +Depends: + ${gir:Depends}, + ${misc:Depends} +Description: Gandiva is a toolset for compiling and evaluating expressions + on Arrow Data. + . + This package provides GObject Introspection typelib files. + +Package: libgandiva-glib-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libgandiva-dev (= ${binary:Version}), + libarrow-glib-dev (= ${binary:Version}), + libgandiva-glib600 (= ${binary:Version}), + gir1.2-gandiva-1.0 (= ${binary:Version}) +Description: Gandiva is a toolset for compiling and evaluating expressions + on Arrow Data. + . + This package provides GLib based header files. + +Package: libgandiva-glib-doc +Section: doc +Architecture: any +Multi-Arch: foreign +Depends: + ${misc:Depends} +Recommends: libglib2.0-doc +Description: Gandiva is a toolset for compiling and evaluating expressions + on Arrow Data. + . + This package provides documentations. + +Package: libplasma-glib600 +Section: libs +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow-cuda-glib600 (= ${binary:Version}), + libplasma600 (= ${binary:Version}) +Description: Plasma is an in-memory object store and cache for big data. + . + This package provides GLib based library files to connect plasma-store-server. + +Package: gir1.2-plasma-1.0 +Section: introspection +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Depends: + ${gir:Depends}, + ${misc:Depends} +Description: Plasma is an in-memory object store and cache for big data. + . + This package provides GObject Introspection typelib files. + +Package: libplasma-glib-dev +Section: libdevel +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: same +Depends: + ${misc:Depends}, + libplasma-dev (= ${binary:Version}), + libarrow-cuda-glib-dev (= ${binary:Version}), + libplasma-glib600 (= ${binary:Version}), + gir1.2-plasma-1.0 (= ${binary:Version}) +Description: Plasma is an in-memory object store and cache for big data. + . + This package provides GLib based header files. + +Package: libplasma-glib-doc +Section: doc +Architecture: @CUDA_ARCHITECTURE@ +Multi-Arch: foreign +Depends: + ${misc:Depends} +Recommends: libglib2.0-doc +Description: Plasma is an in-memory object store and cache for big data. + . + This package provides documentations. + +Package: libparquet-glib600 +Section: libs +Architecture: any +Multi-Arch: same +Pre-Depends: ${misc:Pre-Depends} +Depends: + ${misc:Depends}, + ${shlibs:Depends}, + libarrow-glib600 (= ${binary:Version}), + libparquet600 (= ${binary:Version}) +Description: Apache Parquet is a columnar storage format + . + This package provides GLib based library files. + +Package: gir1.2-parquet-1.0 +Section: introspection +Architecture: any +Multi-Arch: same +Depends: + ${gir:Depends}, + ${misc:Depends} +Description: Apache Parquet is a columnar storage format + . + This package provides GObject Introspection typelib files. + +Package: libparquet-glib-dev +Section: libdevel +Architecture: any +Multi-Arch: same +Depends: + ${misc:Depends}, + libarrow-glib-dev (= ${binary:Version}), + libparquet-dev (= ${binary:Version}), + libparquet-glib600 (= ${binary:Version}), + gir1.2-parquet-1.0 (= ${binary:Version}) +Suggests: libparquet-glib-doc +Description: Apache Parquet is a columnar storage format + . + This package provides GLib based header files. + +Package: libparquet-glib-doc +Section: doc +Architecture: all +Multi-Arch: foreign +Depends: + ${misc:Depends} +Recommends: libglib2.0-doc +Description: Apache Parquet is a columnar storage format + . + This package provides documentations. diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/copyright b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/copyright new file mode 100644 index 000000000..9db0ea76f --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/copyright @@ -0,0 +1,193 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: Apache Arrow +Upstream-Contact: <dev@arrow.apache.org> +Source: https://dist.apache.org/repos/dist/release/arrow/ + +Files: * +Copyright: 2016 The Apache Software Foundation +License: Apache-2.0 + +Files: TODO for "This product includes software from the SFrame project" +Copyright: 2015 Dato, Inc. + 2009 Carnegie Mellon University. +License: BSD-3-clause + +Files: TODO for "This product includes software from the Numpy project" +Copyright: 1995, 1996, 1997 Jim Hugunin, hugunin@mit.edu + 2005 Travis E. Oliphant oliphant@ee.byu.edu Brigham Young University +License: BSD-3-clause + +Files: TODO for "This product includes software from the Feather project" +Copyright: TODO +License: Apache-2.0 + +Files: TODO for "This product includes software from the DyND project" +Copyright: TODO +License: BSD-2-clause + +Files: TODO for "This product includes software from the LLVM project" +Copyright: 2003-2007 University of Illinois at Urbana-Champaign. +License: U-OF-I-BSD-LIKE + +Files: TODO for "This product includes software from the google-lint project" +Copyright: 2009 Google Inc. All rights reserved. +License: BSD-3-clause + +Files: TODO for "This product includes software from the mman-win32 project" +Copyright: 2010 kutuzov.viktor.84 +License: MIT + +Files: TODO for "This product includes software from the LevelDB project" +Copyright: 2011 The LevelDB Authors. All rights reserved. +License: BSD-3-clause + +Files: TODO for "This product includes software from the CMake project" +Copyright: 2001-2009 Kitware, Inc. + 2012-2014 Continuum Analytics, Inc. +License: BSD-3-clause + +Files: TODO for "This product includes software from https://github.com/matthew-brett/multibuild" +Copyright: 2013-2016, Matt Terry and Matthew Brett; all rights reserved. +License: BSD-2-clause + +Files: TODO for "This product includes software from the Ibis project" +Copyright: 2015 Cloudera, Inc. +License: Apache-2.0 + +Files: TODO for "This product includes code from Apache Kudu" +Copyright: 2016 The Apache Software Foundation +License: Apache-2.0 + +License: Apache-2.0 + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the full text of the Apache Software License version 2 can + be found in the file `/usr/share/common-licenses/Apache-2.0'. + +License: BSD-3-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + . + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + . + THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + SUCH DAMAGE. + +License: BSD-2-clause + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + . + 1) Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + . + 2) Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + . + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY + WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + +License: U-OF-I-BSD-LIKE + ============================================================================== + LLVM Release License + ============================================================================== + University of Illinois/NCSA + Open Source License + . + Copyright (c) 2003-2013 University of Illinois at Urbana-Champaign. + All rights reserved. + . + Developed by: + . + LLVM Team + . + University of Illinois at Urbana-Champaign + . + http://llvm.org + . + Permission is hereby granted, free of charge, to any person obtaining a copy of + this software and associated documentation files (the "Software"), to deal with + the Software without restriction, including without limitation the rights to + use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + of the Software, and to permit persons to whom the Software is furnished to do + so, subject to the following conditions: + . + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + . + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + . + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE + SOFTWARE. + +License: MIT + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + . + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + . + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install new file mode 100644 index 000000000..e0197fcd3 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-1.0.install @@ -0,0 +1 @@ +usr/lib/*/girepository-1.0/Arrow-1.0.typelib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install new file mode 100644 index 000000000..ef0d9f56f --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-cuda-1.0.install @@ -0,0 +1 @@ +usr/lib/*/girepository-1.0/ArrowCUDA-1.0.typelib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install new file mode 100644 index 000000000..27091dab3 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-arrow-dataset-1.0.install @@ -0,0 +1 @@ +usr/lib/*/girepository-1.0/ArrowDataset-1.0.typelib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install new file mode 100644 index 000000000..0433b367a --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-gandiva-1.0.install @@ -0,0 +1 @@ +usr/lib/*/girepository-1.0/Gandiva-1.0.typelib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install new file mode 100644 index 000000000..13fde6681 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-parquet-1.0.install @@ -0,0 +1 @@ +usr/lib/*/girepository-1.0/Parquet-1.0.typelib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install new file mode 100644 index 000000000..7b7ce2158 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/gir1.2-plasma-1.0.install @@ -0,0 +1 @@ +usr/lib/*/girepository-1.0/Plasma-1.0.typelib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install new file mode 100644 index 000000000..77e0b70f6 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-dev.install @@ -0,0 +1,6 @@ +usr/lib/*/cmake/arrow/ArrowCUDAConfig*.cmake +usr/lib/*/cmake/arrow/ArrowCUDATargets*.cmake +usr/lib/*/cmake/arrow/FindArrowCUDA.cmake +usr/lib/*/libarrow_cuda.a +usr/lib/*/libarrow_cuda.so +usr/lib/*/pkgconfig/arrow-cuda.pc diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install new file mode 100644 index 000000000..778ae5fd7 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib-dev.install @@ -0,0 +1,4 @@ +usr/include/arrow-cuda-glib/ +usr/lib/*/libarrow-cuda-glib.so +usr/lib/*/pkgconfig/arrow-cuda-glib.pc +usr/share/gir-1.0/ArrowCUDA-1.0.gir diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib600.install new file mode 100644 index 000000000..a6d637526 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda-glib600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow-cuda-glib.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda600.install new file mode 100644 index 000000000..5ae464687 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-cuda600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow_cuda.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install new file mode 100644 index 000000000..53e727ae0 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-dev.install @@ -0,0 +1,6 @@ +usr/lib/*/cmake/arrow/ArrowDatasetConfig*.cmake +usr/lib/*/cmake/arrow/ArrowDatasetTargets*.cmake +usr/lib/*/cmake/arrow/FindArrowDataset.cmake +usr/lib/*/libarrow_dataset.a +usr/lib/*/libarrow_dataset.so +usr/lib/*/pkgconfig/arrow-dataset.pc diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install new file mode 100644 index 000000000..4c50bde97 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-dev.install @@ -0,0 +1,4 @@ +usr/include/arrow-dataset-glib/ +usr/lib/*/libarrow-dataset-glib.so +usr/lib/*/pkgconfig/arrow-dataset-glib.pc +usr/share/gir-1.0/ArrowDataset-1.0.gir diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base new file mode 100644 index 000000000..5ec8156b0 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.doc-base @@ -0,0 +1,9 @@ +Document: arrow-dataset-glib +Title: Apache Arrow Dataset GLib Reference Manual +Author: The Apache Software Foundation +Abstract: Apache Arrow Dataset GLib provides an API to read and write semantic datasets stored in different locations and formats that uses GLib. +Section: Programming + +Format: HTML +Index: /usr/share/gtk-doc/html/arrow-dataset-glib/index.html +Files: /usr/share/gtk-doc/html/arrow-dataset-glib/*.html diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install new file mode 100644 index 000000000..523bc206e --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.install @@ -0,0 +1 @@ +usr/share/gtk-doc/html/arrow-dataset-glib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links new file mode 100644 index 000000000..3d880362b --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib-doc.links @@ -0,0 +1,3 @@ +usr/share/gtk-doc/html/arrow-dataset-glib usr/share/doc/libarrow-dataset-glib-doc/arrow-dataset-glib +usr/share/doc/libglib2.0-doc/glib usr/share/doc/libarrow-dataset-glib-doc/glib +usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libarrow-dataset-glib-doc/gobject diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib600.install new file mode 100644 index 000000000..10085f3a0 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset-glib600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow-dataset-glib.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset600.install new file mode 100644 index 000000000..014634165 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dataset600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow_dataset.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install new file mode 100644 index 000000000..ccd0c4e5b --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-dev.install @@ -0,0 +1,19 @@ +usr/include/arrow/ +usr/lib/*/cmake/arrow/ArrowConfig*.cmake +usr/lib/*/cmake/arrow/ArrowOptions.cmake +usr/lib/*/cmake/arrow/ArrowTargets*.cmake +usr/lib/*/cmake/arrow/Find*Alt.cmake +usr/lib/*/cmake/arrow/FindArrow.cmake +usr/lib/*/cmake/arrow/FindBrotli.cmake +usr/lib/*/cmake/arrow/FindLz4.cmake +usr/lib/*/cmake/arrow/Find[Suz]*.cmake +usr/lib/*/cmake/arrow/arrow-config.cmake +usr/lib/*/libarrow.a +usr/lib/*/libarrow.so +usr/lib/*/libarrow_bundled_dependencies.a +usr/lib/*/pkgconfig/arrow-compute.pc +usr/lib/*/pkgconfig/arrow-csv.pc +usr/lib/*/pkgconfig/arrow-filesystem.pc +usr/lib/*/pkgconfig/arrow-json.pc +usr/lib/*/pkgconfig/arrow-orc.pc +usr/lib/*/pkgconfig/arrow.pc diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install new file mode 100644 index 000000000..20ca33d84 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-dev.install @@ -0,0 +1,6 @@ +usr/lib/*/cmake/arrow/ArrowFlightConfig*.cmake +usr/lib/*/cmake/arrow/ArrowFlightTargets*.cmake +usr/lib/*/cmake/arrow/FindArrowFlight.cmake +usr/lib/*/libarrow_flight.a +usr/lib/*/libarrow_flight.so +usr/lib/*/pkgconfig/arrow-flight.pc diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-dev.install new file mode 100644 index 000000000..8a8dee3ac --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-dev.install @@ -0,0 +1,4 @@ +usr/include/arrow-flight-glib/ +usr/lib/*/libarrow-flight-glib.so +usr/lib/*/pkgconfig/arrow-flight-glib.pc +usr/share/gir-1.0/ArrowFlight-1.0.gir diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.doc-base b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.doc-base new file mode 100644 index 000000000..94b17c11b --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.doc-base @@ -0,0 +1,9 @@ +Document: arrow-flight-glib +Title: Apache Arrow Flight GLib Reference Manual +Author: The Apache Software Foundation +Abstract: Apache Arrow Flight GLib provides a general-purpose client-server framework to simplify high performance transport of large datasets over network interfaces. +Section: Programming + +Format: HTML +Index: /usr/share/gtk-doc/html/arrow-flight-glib/index.html +Files: /usr/share/gtk-doc/html/arrow-flight-glib/*.html diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.install new file mode 100644 index 000000000..3c95f17ed --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.install @@ -0,0 +1 @@ +usr/share/gtk-doc/html/arrow-flight-glib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.links b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.links new file mode 100644 index 000000000..d55c89a1b --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib-doc.links @@ -0,0 +1,3 @@ +usr/share/gtk-doc/html/arrow-flight-glib usr/share/doc/libarrow-flight-glib-doc/arrow-flight-glib +usr/share/doc/libglib2.0-doc/glib usr/share/doc/libarrow-flight-glib-doc/glib +usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libarrow-flight-glib-doc/gobject diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib600.install new file mode 100644 index 000000000..a6156ed94 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight-glib600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow-flight-glib.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight600.install new file mode 100644 index 000000000..abdb96d4c --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-flight600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow_flight.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install new file mode 100644 index 000000000..f6de7eedb --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-dev.install @@ -0,0 +1,6 @@ +usr/include/arrow-glib/ +usr/lib/*/libarrow-glib.so +usr/lib/*/pkgconfig/arrow-glib.pc +usr/lib/*/pkgconfig/arrow-orc-glib.pc +usr/share/arrow-glib/example/ +usr/share/gir-1.0/Arrow-1.0.gir diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base new file mode 100644 index 000000000..8ae4ffb6d --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.doc-base @@ -0,0 +1,9 @@ +Document: arrow-glib +Title: Apache Arrow GLib Reference Manual +Author: The Apache Software Foundation +Abstract: Apache Arrow GLib is a data processing library for analysis that uses GLib. +Section: Programming + +Format: HTML +Index: /usr/share/gtk-doc/html/arrow-glib/index.html +Files: /usr/share/gtk-doc/html/arrow-glib/*.html diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install new file mode 100644 index 000000000..912a29c58 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.install @@ -0,0 +1,2 @@ +usr/share/doc/arrow-glib/ +usr/share/gtk-doc/html/arrow-glib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links new file mode 100644 index 000000000..556987d0a --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib-doc.links @@ -0,0 +1,3 @@ +usr/share/gtk-doc/html/arrow-glib usr/share/doc/libarrow-glib-doc/arrow-glib +usr/share/doc/libglib2.0-doc/glib usr/share/doc/libarrow-glib-doc/glib +usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libarrow-glib-doc/gobject diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib600.install new file mode 100644 index 000000000..ec369d153 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-glib600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow-glib.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install new file mode 100644 index 000000000..807583f98 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-dev.install @@ -0,0 +1,6 @@ +usr/lib/*/cmake/arrow/ArrowPythonConfig*.cmake +usr/lib/*/cmake/arrow/ArrowPythonTargets*.cmake +usr/lib/*/cmake/arrow/FindArrowPython.cmake +usr/lib/*/libarrow_python.a +usr/lib/*/libarrow_python.so +usr/lib/*/pkgconfig/arrow-python.pc diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install new file mode 100644 index 000000000..6cf96e227 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight-dev.install @@ -0,0 +1,6 @@ +usr/lib/*/cmake/arrow/ArrowPythonFlightConfig*.cmake +usr/lib/*/cmake/arrow/ArrowPythonFlightTargets*.cmake +usr/lib/*/cmake/arrow/FindArrowPythonFlight.cmake +usr/lib/*/libarrow_python_flight.a +usr/lib/*/libarrow_python_flight.so +usr/lib/*/pkgconfig/arrow-python-flight.pc diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight600.install new file mode 100644 index 000000000..b7cbfec1f --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python-flight600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow_python_flight.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python600.install new file mode 100644 index 000000000..eef3e6648 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow-python600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow_python.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow600.install new file mode 100644 index 000000000..98ef2139c --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libarrow600.install @@ -0,0 +1 @@ +usr/lib/*/libarrow.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install new file mode 100644 index 000000000..26e7e76fb --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-dev.install @@ -0,0 +1,7 @@ +usr/include/gandiva/ +usr/lib/*/cmake/arrow/GandivaConfig*.cmake +usr/lib/*/cmake/arrow/GandivaTargets*.cmake +usr/lib/*/cmake/arrow/FindGandiva.cmake +usr/lib/*/libgandiva.a +usr/lib/*/libgandiva.so +usr/lib/*/pkgconfig/gandiva.pc diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install new file mode 100644 index 000000000..fe7d8bb79 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-dev.install @@ -0,0 +1,4 @@ +usr/include/gandiva-glib/ +usr/lib/*/libgandiva-glib.so +usr/lib/*/pkgconfig/gandiva-glib.pc +usr/share/gir-1.0/Gandiva-1.0.gir diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base new file mode 100644 index 000000000..2bf913062 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.doc-base @@ -0,0 +1,9 @@ +Document: gandiva-glib +Title: Gandiva GLib Reference Manual +Author: The Apache Software Foundation +Abstract: Gandiva GLib is a toolset for compiling and evaluating expressions on Arrow Data that uses GLib. +Section: Programming + +Format: HTML +Index: /usr/share/gtk-doc/html/gandiva-glib/index.html +Files: /usr/share/gtk-doc/html/gandiva-glib/*.html diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install new file mode 100644 index 000000000..358e4e5c7 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.install @@ -0,0 +1 @@ +usr/share/gtk-doc/html/gandiva-glib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links new file mode 100644 index 000000000..234794e23 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib-doc.links @@ -0,0 +1,3 @@ +usr/share/gtk-doc/html/gandiva-glib usr/share/doc/libgandiva-glib-doc/gandiva-glib +usr/share/doc/libglib2.0-doc/glib usr/share/doc/libgandiva-glib-doc/glib +usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libgandiva-glib-doc/gobject diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib600.install new file mode 100644 index 000000000..6257fd438 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva-glib600.install @@ -0,0 +1 @@ +usr/lib/*/libgandiva-glib.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva600.install new file mode 100644 index 000000000..1475f49cf --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libgandiva600.install @@ -0,0 +1 @@ +usr/lib/*/libgandiva.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install new file mode 100644 index 000000000..e163115f0 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-dev.install @@ -0,0 +1,7 @@ +usr/include/parquet/ +usr/lib/*/cmake/arrow/ParquetConfig*.cmake +usr/lib/*/cmake/arrow/ParquetTargets*.cmake +usr/lib/*/cmake/arrow/FindParquet.cmake +usr/lib/*/libparquet.a +usr/lib/*/libparquet.so +usr/lib/*/pkgconfig/parquet.pc diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install new file mode 100644 index 000000000..9cce737a7 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-dev.install @@ -0,0 +1,4 @@ +usr/include/parquet-glib/ +usr/lib/*/libparquet-glib.so +usr/lib/*/pkgconfig/parquet-glib.pc +usr/share/gir-1.0/Parquet-1.0.gir diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base new file mode 100644 index 000000000..cc68e2df6 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.doc-base @@ -0,0 +1,9 @@ +Document: parquet-glib +Title: Apache Parquet GLib Reference Manual +Author: The Apache Software Foundation +Abstract: Apache Parquet GLib is a columnar storage format processing library that uses GLib. +Section: Programming + +Format: HTML +Index: /usr/share/gtk-doc/html/parquet-glib/index.html +Files: /usr/share/gtk-doc/html/parquet-glib/*.html diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install new file mode 100644 index 000000000..5843ea3da --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.install @@ -0,0 +1 @@ +usr/share/gtk-doc/html/parquet-glib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links new file mode 100644 index 000000000..c31f346b1 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib-doc.links @@ -0,0 +1,3 @@ +usr/share/gtk-doc/html/parquet-glib usr/share/doc/libparquet-glib-doc/parquet-glib +usr/share/doc/libglib2.0-doc/glib usr/share/doc/libparquet-glib-doc/glib +usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libparquet-glib-doc/gobject diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib600.install new file mode 100644 index 000000000..1c0e44199 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet-glib600.install @@ -0,0 +1 @@ +usr/lib/*/libparquet-glib.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet600.install new file mode 100644 index 000000000..540a91d5e --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libparquet600.install @@ -0,0 +1 @@ +usr/lib/*/libparquet.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install new file mode 100644 index 000000000..c315d4dfc --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-dev.install @@ -0,0 +1,7 @@ +usr/include/plasma/ +usr/lib/*/cmake/arrow/PlasmaConfig*.cmake +usr/lib/*/cmake/arrow/PlasmaTargets*.cmake +usr/lib/*/cmake/arrow/FindPlasma.cmake +usr/lib/*/libplasma.a +usr/lib/*/libplasma.so +usr/lib/*/pkgconfig/plasma.pc diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install new file mode 100644 index 000000000..7800681d2 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-dev.install @@ -0,0 +1,4 @@ +usr/include/plasma-glib/ +usr/lib/*/libplasma-glib.so +usr/lib/*/pkgconfig/plasma-glib.pc +usr/share/gir-1.0/Plasma-1.0.gir diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base new file mode 100644 index 000000000..a9d306d8b --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.doc-base @@ -0,0 +1,9 @@ +Document: plasma-glib +Title: Plasma GLib Reference Manual +Author: The Apache Software Foundation +Abstract: Plasma GLib is an in-memory object store and cache for big data that uses GLib. +Section: Programming + +Format: HTML +Index: /usr/share/gtk-doc/html/plasma-glib/index.html +Files: /usr/share/gtk-doc/html/plasma-glib/*.html diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install new file mode 100644 index 000000000..ad13b94cd --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.install @@ -0,0 +1 @@ +usr/share/gtk-doc/html/plasma-glib diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links new file mode 100644 index 000000000..193262f9b --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib-doc.links @@ -0,0 +1,3 @@ +usr/share/gtk-doc/html/plasma-glib usr/share/doc/libplasma-glib-doc/plasma-glib +usr/share/doc/libglib2.0-doc/glib usr/share/doc/libplasma-glib-doc/glib +usr/share/doc/libglib2.0-doc/gobject usr/share/doc/libplasma-glib-doc/gobject diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib600.install new file mode 100644 index 000000000..339bcca3e --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma-glib600.install @@ -0,0 +1 @@ +usr/lib/*/libplasma-glib.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma600.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma600.install new file mode 100644 index 000000000..f8a744b65 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/libplasma600.install @@ -0,0 +1 @@ +usr/lib/*/libplasma.so.* diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/patches/series b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/patches/series new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/patches/series diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install new file mode 100644 index 000000000..bd13b0e81 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/plasma-store-server.install @@ -0,0 +1 @@ +usr/bin/plasma-store-server diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/rules b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/rules new file mode 100755 index 000000000..2de533615 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/rules @@ -0,0 +1,104 @@ +#!/usr/bin/make -f +# -*- makefile-gmake -*- +# +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 +# This has to be exported to make some magic below work. +export DH_OPTIONS + +export DEB_BUILD_MAINT_OPTIONS=reproducible=-timeless + +BUILD_TYPE=release + +%: + dh $@ --with gir + +override_dh_auto_configure: + if dpkg -l nvidia-cuda-toolkit > /dev/null 2>&1; then \ + ARROW_CUDA=ON; \ + ARROW_PLASMA=ON; \ + else \ + ARROW_CUDA=OFF; \ + ARROW_PLASMA=OFF; \ + fi; \ + dh_auto_configure \ + --sourcedirectory=cpp \ + --builddirectory=cpp_build \ + --buildsystem=cmake+ninja \ + -- \ + -DARROW_CUDA=$${ARROW_CUDA} \ + -DARROW_FLIGHT=ON \ + -DARROW_GANDIVA=ON \ + -DARROW_GANDIVA_JAVA=OFF \ + -DARROW_MIMALLOC=ON \ + -DARROW_ORC=ON \ + -DARROW_PACKAGE_KIND=deb \ + -DARROW_PARQUET=ON \ + -DARROW_PLASMA=$${ARROW_PLASMA} \ + -DARROW_PYTHON=ON \ + -DARROW_S3=ON \ + -DARROW_USE_CCACHE=OFF \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DCMAKE_BUILD_TYPE=$(BUILD_TYPE) \ + -DCMAKE_UNITY_BUILD=ON \ + -DPARQUET_REQUIRE_ENCRYPTION=ON \ + -DPythonInterp_FIND_VERSION=ON \ + -DPythonInterp_FIND_VERSION_MAJOR=3 + +override_dh_auto_build: + dh_auto_build \ + --sourcedirectory=cpp \ + --builddirectory=cpp_build + dh_auto_configure \ + --sourcedirectory=c_glib \ + --builddirectory=c_glib_build \ + --buildsystem=meson+ninja \ + -- \ + -Darrow_cpp_build_type=$(BUILD_TYPE) \ + -Darrow_cpp_build_dir=../cpp_build \ + -Dgtk_doc=true + env \ + LD_LIBRARY_PATH=$(CURDIR)/cpp_build/$(BUILD_TYPE) \ + dh_auto_build \ + --sourcedirectory=c_glib \ + --builddirectory=c_glib_build \ + --buildsystem=meson+ninja + +override_dh_auto_install: + dh_auto_install \ + --sourcedirectory=c_glib \ + --builddirectory=c_glib_build \ + --buildsystem=meson+ninja + # Remove built files to reduce disk usage + dh_auto_clean \ + --sourcedirectory=c_glib \ + --builddirectory=c_glib_build \ + --buildsystem=meson+ninja + + dh_auto_install \ + --sourcedirectory=cpp \ + --builddirectory=cpp_build + # Remove built files to reduce disk usage + dh_auto_clean \ + --sourcedirectory=cpp \ + --builddirectory=cpp_build + +override_dh_auto_test: + # TODO: We need Boost 1.64 or later to build tests for + # Apache Arrow Flight. + # git clone --depth 1 https://github.com/apache/arrow-testing.git + # git clone --depth 1 https://github.com/apache/parquet-testing.git + # cd cpp_build && \ + # env \ + # ARROW_TEST_DATA=$(CURDIR)/arrow-testing/data \ + # PARQUET_TEST_DATA=$(CURDIR)/parquet-testing/data \ + # ctest --exclude-regex 'arrow-cuda-test|plasma-client_tests' + +# skip file failing with "Unknown DWARF DW_OP_255" (see bug#949296) +override_dh_dwz: + dh_dwz --exclude=libgandiva.so diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/source/format b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/source/format new file mode 100644 index 000000000..163aaf8d8 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/watch b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/watch new file mode 100644 index 000000000..5cb3f0091 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/debian/watch @@ -0,0 +1,2 @@ +version=3 +https://dist.apache.org/repos/dist/release/arrow/arrow-(.+)/apache-arrow-(.+).tar.gz diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8-aarch64/from b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8-aarch64/from new file mode 100644 index 000000000..b570587e0 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8-aarch64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/almalinux:8 diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile new file mode 100644 index 000000000..003b26747 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/almalinux-8/Dockerfile @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=almalinux:8 +FROM ${FROM} + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + dnf install -y ${quiet} epel-release && \ + dnf install --enablerepo=powertools -y ${quiet} \ + bison \ + boost-devel \ + brotli-devel \ + bzip2-devel \ + c-ares-devel \ + ccache \ + clang \ + cmake \ + curl-devel \ + flex \ + gcc-c++ \ + gflags-devel \ + git \ + glog-devel \ + gobject-introspection-devel \ + gtk-doc \ + libarchive \ + libzstd-devel \ + llvm-devel \ + llvm-static \ + lz4-devel \ + make \ + ncurses-devel \ + ninja-build \ + openssl-devel \ + pkg-config \ + python3 \ + python3-devel \ + python3-numpy \ + python3-pip \ + re2-devel \ + # rapidjson-devel \ + rpmdevtools \ + snappy-devel \ + tar \ + # utf8proc-devel \ + zlib-devel && \ + dnf clean ${quiet} all diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile new file mode 100644 index 000000000..6ada89729 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/Dockerfile @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=amazonlinux:2 +FROM ${FROM} + +COPY qemu-* /usr/bin/ + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + yum update -y ${quiet} && \ + amazon-linux-extras install -y epel && \ + yum install -y ${quiet} \ + bison \ + brotli-devel \ + bzip2-devel \ + ccache \ + cmake3 \ + flex \ + gcc-c++ \ + git \ + glog-devel \ + gobject-introspection-devel \ + gtk-doc \ + lz4-devel \ + make \ + openssl-devel \ + pkg-config \ + rapidjson-devel \ + rpmdevtools \ + snappy-devel \ + tar \ + utf8proc-devel \ + zlib-devel && \ + # Install ninja-build dependencies in amzn2-core + yum install -y ${quiet} ninja-build && \ + # Install ninja-build from EPEL because ninja-build in amzn2-core is old. + yum install -y ${quiet} --disablerepo=amzn2-core ninja-build && \ + yum clean ${quiet} all diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/qemu-dummy-static b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/qemu-dummy-static new file mode 100755 index 000000000..279d89545 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/amazon-linux-2/qemu-dummy-static @@ -0,0 +1,33 @@ +#!/usr/bin/env sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Do nothing. This exists only for not requiring qemu-aarch64-static copy. +# Recent Debian (buster or later) and Ubuntu (18.10 or later) on amd64 hosts or +# arm64 host don't require qemu-aarch64-static in Docker image. But old Debian +# and Ubuntu hosts on amd64 require qemu-aarch64-static in Docker image. +# +# We use "COPY qemu* /usr/bin/" in Dockerfile. If we don't put any "qemnu*", +# the "COPY" is failed. It means that we always require "qemu*" even if we +# use recent Debian/Ubuntu or arm64 host. If we have this dummy "qemu*" file, +# the "COPY" isn't failed. It means that we can copy "qemu*" only when we +# need. +# +# See also "script" in dev/tasks/linux-packages/azure.linux.arm64.yml. +# Azure Pipelines uses old Ubuntu (18.04). +# So we need to put "qemu-aarch64-static" into this directory. diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in new file mode 100644 index 000000000..3a8f71ecc --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/arrow.spec.in @@ -0,0 +1,892 @@ +# -*- sh-shell: rpm -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +%define _amzn %{?amzn:%{amzn}}%{!?amzn:0} +%define is_amazon_linux (%{_amzn} != 0) + +%define boost_version %( \ + if [ %{rhel} -eq 7 ]; then \ + echo 169; \ + fi) +%define cmake_version %( \ + if [ %{rhel} -lt 8 ]; then \ + echo 3; \ + fi) +%define python_version %( \ + if [ %{rhel} -eq 7 ]; then \ + echo 36; \ + else \ + echo 3; \ + fi) + +%define lz4_requirement %( \ + if [ %{_amzn} -eq 0 ]; then \ + echo ">= 1.8.0"; \ + fi) + +%define use_boost (!%{is_amazon_linux}) +%define use_flight (%{rhel} >= 8) +%define use_gandiva (%{rhel} >= 8 && %{_arch} != "aarch64") +%define use_gflags (!%{is_amazon_linux}) +%define use_mimalloc (%{rhel} >= 8) +%define use_python (!%{is_amazon_linux}) +# TODO: Enable this. This works on local but is fragile on GitHub Actions and +# Travis CI. +# %define use_s3 (%{rhel} >= 8) +%define use_s3 0 + +%define have_rapidjson (%{rhel} == 7) +%define have_re2 (%{rhel} >= 8) +# EPEL ships utf8proc but it's old. +# %define have_utf8proc (%{rhel} == 7) +%define have_utf8proc 0 +%define have_zstd (!%{is_amazon_linux}) + +Name: @PACKAGE@ +Version: @VERSION@ +Release: @RELEASE@%{?dist} +Summary: Apache Arrow is a data processing library for analysis + +License: Apache-2.0 +URL: https://arrow.apache.org/ +Source0: https://dist.apache.org/repos/dist/release/@PACKAGE@/@PACKAGE@-%{version}/apache-@PACKAGE@-%{version}.tar.gz + +BuildRequires: bison +%if %{use_boost} +BuildRequires: boost%{boost_version}-devel +%endif +BuildRequires: brotli-devel +BuildRequires: bzip2-devel +%if %{use_flight} +BuildRequires: c-ares-devel +%endif +BuildRequires: cmake%{cmake_version} +%if %{use_s3} +BuildRequires: curl-devel +%endif +BuildRequires: flex +BuildRequires: gcc-c++ +%if %{use_gflags} +BuildRequires: gflags-devel +%endif +BuildRequires: git +BuildRequires: glog-devel +%if %{have_zstd} +BuildRequires: libzstd-devel +%endif +BuildRequires: lz4-devel %{lz4_requirement} +BuildRequires: ninja-build +BuildRequires: openssl-devel +BuildRequires: pkgconfig +%if %{use_python} +BuildRequires: python%{python_version}-devel +BuildRequires: python%{python_version}-numpy +%endif +%if %{have_rapidjson} +BuildRequires: rapidjson-devel +%endif +%if %{have_re2} +BuildRequires: re2-devel +%endif +BuildRequires: snappy-devel +%if %{have_utf8proc} +BuildRequires: utf8proc-devel +%endif +BuildRequires: zlib-devel + +%if %{use_gandiva} +BuildRequires: llvm-devel +BuildRequires: ncurses-devel +%endif + +BuildRequires: gobject-introspection-devel +BuildRequires: gtk-doc + +%description +Apache Arrow is a data processing library for analysis. + +%prep +%setup -q -n apache-@PACKAGE@-%{version} + +%build +cpp_build_type=release +mkdir cpp/build +cd cpp/build +%cmake3 .. \ + -DARROW_CSV=ON \ + -DARROW_DATASET=ON \ +%if %{use_flight} + -DARROW_FLIGHT=ON \ +%endif +%if %{use_gandiva} + -DARROW_GANDIVA=ON \ +%endif + -DARROW_HDFS=ON \ + -DARROW_JSON=ON \ +%if %{use_mimalloc} + -DARROW_MIMALLOC=ON \ +%endif + -DARROW_ORC=ON \ + -DARROW_PACKAGE_KIND=rpm \ + -DARROW_PARQUET=ON \ + -DARROW_PLASMA=ON \ +%if %{use_python} + -DARROW_PYTHON=ON \ +%endif +%if %{use_s3} + -DARROW_S3=ON \ +%endif + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DCMAKE_BUILD_TYPE=$cpp_build_type \ + -DARROW_USE_CCACHE=OFF \ + -DPARQUET_REQUIRE_ENCRYPTION=ON \ +%if %{use_python} + -DPythonInterp_FIND_VERSION=ON \ + -DPythonInterp_FIND_VERSION_MAJOR=3 \ +%endif + -GNinja + +ninja %{?_smp_mflags} +cd - + +cd c_glib +pip3 install meson +meson setup build \ + --default-library=both \ + --libdir=%{_libdir} \ + --prefix=%{_prefix} \ + -Darrow_cpp_build_dir=../cpp/build \ + -Darrow_cpp_build_type=$cpp_build_type \ + -Dgtk_doc=true +LD_LIBRARY_PATH=$PWD/../cpp/build/$cpp_build_type \ + ninja -C build %{?_smp_mflags} +cd - + +%install +cpp_build_type=release + +cd c_glib +DESTDIR=$RPM_BUILD_ROOT ninja -C build install +ninja -C build clean +cd - + +cd cpp/build +DESTDIR=$RPM_BUILD_ROOT ninja install +ninja clean +cd - + +%package libs +Summary: Runtime libraries for Apache Arrow C++ +License: Apache-2.0 +Requires: brotli +%if %{use_gflags} +Requires: gflags +%endif +Requires: glog +%if %{have_zstd} +Requires: libzstd +%endif +Requires: lz4 %{lz4_requirement} +%if %{have_re2} +Requires: re2 +%endif +Requires: snappy +%if %{have_utf8proc} +Requires: utf8proc +%endif +Requires: zlib + +%description libs +This package contains the libraries for Apache Arrow C++. + +%files libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_docdir}/arrow/ +%{_libdir}/libarrow.so.* + +%package devel +Summary: Libraries and header files for Apache Arrow C++ +License: Apache-2.0 +Requires: %{name}-libs = %{version}-%{release} +Requires: brotli-devel +Requires: bzip2-devel +%if %{use_flight} +Requires: c-ares-devel +%endif +%if %{have_zstd} +Requires: libzstd-devel +%endif +Requires: lz4-devel %{lz4_requirement} +Requires: openssl-devel +%if %{have_rapidjson} +Requires: rapidjson-devel +%endif +%if %{have_re2} +Requires: re2-devel +%endif +Requires: snappy-devel +%if %{have_utf8proc} +Requires: utf8proc-devel +%endif +Requires: zlib-devel + +%description devel +Libraries and header files for Apache Arrow C++. + +%files devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/arrow/ +%exclude %{_includedir}/arrow/dataset/ +%if %{use_flight} +%exclude %{_includedir}/arrow/flight/ +%endif +%exclude %{_includedir}/arrow/python/ +%{_libdir}/cmake/arrow/ArrowConfig*.cmake +%{_libdir}/cmake/arrow/ArrowOptions.cmake +%{_libdir}/cmake/arrow/ArrowTargets*.cmake +%{_libdir}/cmake/arrow/FindArrow.cmake +%{_libdir}/cmake/arrow/FindBrotli.cmake +%{_libdir}/cmake/arrow/FindLz4.cmake +%{_libdir}/cmake/arrow/FindSnappy.cmake +%if %{use_flight} +%{_libdir}/cmake/arrow/Findc-aresAlt.cmake +%endif +%if %{have_re2} +%{_libdir}/cmake/arrow/Findre2Alt.cmake +%endif +%if %{have_utf8proc} +%{_libdir}/cmake/arrow/Findutf8proc.cmake +%endif +%if %{have_zstd} +%{_libdir}/cmake/arrow/Findzstd.cmake +%endif +%{_libdir}/cmake/arrow/arrow-config.cmake +%{_libdir}/libarrow.a +%{_libdir}/libarrow.so +%{_libdir}/libarrow_bundled_dependencies.a +%{_libdir}/pkgconfig/arrow-compute.pc +%{_libdir}/pkgconfig/arrow-csv.pc +%{_libdir}/pkgconfig/arrow-filesystem.pc +%{_libdir}/pkgconfig/arrow-json.pc +%{_libdir}/pkgconfig/arrow-orc.pc +%{_libdir}/pkgconfig/arrow.pc + +%package dataset-libs +Summary: C++ library to read and write semantic datasets stored in different locations and formats +License: Apache-2.0 +Requires: %{name}-libs = %{version}-%{release} + +%description dataset-libs +This package contains the libraries for Apache Arrow dataset. + +%files dataset-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libarrow_dataset.so.* + +%package dataset-devel +Summary: Libraries and header files for Apache Arrow dataset. +License: Apache-2.0 +Requires: %{name}-dataset-libs = %{version}-%{release} + +%description dataset-devel +Libraries and header files for Apache Arrow dataset. + +%files dataset-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/arrow/dataset/ +%{_libdir}/cmake/arrow/ArrowDatasetConfig*.cmake +%{_libdir}/cmake/arrow/ArrowDatasetTargets*.cmake +%{_libdir}/cmake/arrow/FindArrowDataset.cmake +%{_libdir}/libarrow_dataset.a +%{_libdir}/libarrow_dataset.so +%{_libdir}/pkgconfig/arrow-dataset.pc + +%if %{use_flight} +%package flight-libs +Summary: C++ library for fast data transport. +License: Apache-2.0 +Requires: %{name}-libs = %{version}-%{release} +%if %{use_flight} +Requires: c-ares +%endif +Requires: openssl + +%description flight-libs +This package contains the libraries for Apache Arrow Flight. + +%files flight-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libarrow_flight.so.* + +%package flight-devel +Summary: Libraries and header files for Apache Arrow Flight. +License: Apache-2.0 +Requires: %{name}-flight-libs = %{version}-%{release} + +%description flight-devel +Libraries and header files for Apache Arrow Flight. + +%files flight-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/arrow/flight/ +%{_libdir}/cmake/arrow/ArrowFlightConfig*.cmake +%{_libdir}/cmake/arrow/ArrowFlightTargets*.cmake +%{_libdir}/cmake/arrow/FindArrowFlight.cmake +%{_libdir}/libarrow_flight.a +%{_libdir}/libarrow_flight.so +%{_libdir}/pkgconfig/arrow-flight.pc +%endif + +%if %{use_gandiva} +%package -n gandiva-libs +Summary: C++ library for compiling and evaluating expressions on Apache Arrow data. +License: Apache-2.0 +Requires: %{name}-libs = %{version}-%{release} +Requires: ncurses-libs + +%description -n gandiva-libs +This package contains the libraries for Gandiva. + +%files -n gandiva-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libgandiva.so.* + +%package -n gandiva-devel +Summary: Libraries and header files for Gandiva. +License: Apache-2.0 +Requires: gandiva-libs = %{version}-%{release} +Requires: llvm-devel + +%description -n gandiva-devel +Libraries and header files for Gandiva. + +%files -n gandiva-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/gandiva/ +%{_libdir}/cmake/arrow/GandivaConfig*.cmake +%{_libdir}/cmake/arrow/GandivaTargets*.cmake +%{_libdir}/cmake/arrow/FindGandiva.cmake +%{_libdir}/libgandiva.a +%{_libdir}/libgandiva.so +%{_libdir}/pkgconfig/gandiva.pc +%endif + +%if %{use_python} +%package python-libs +Summary: Python integration library for Apache Arrow +License: Apache-2.0 +Requires: %{name}-libs = %{version}-%{release} +Requires: python%{python_version}-numpy + +%description python-libs +This package contains the Python integration library for Apache Arrow. + +%files python-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libarrow_python.so.* + +%package python-devel +Summary: Libraries and header files for Python integration library for Apache Arrow +License: Apache-2.0 +Requires: %{name}-devel = %{version}-%{release} +Requires: %{name}-libs = %{version}-%{release} +Requires: python%{python_version}-devel + +%description python-devel +Libraries and header files for Python integration library for Apache Arrow. + +%files python-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/arrow/python/ +%exclude %{_includedir}/arrow/python/flight.h +%{_libdir}/cmake/arrow/ArrowPythonConfig*.cmake +%{_libdir}/cmake/arrow/ArrowPythonTargets*.cmake +%{_libdir}/cmake/arrow/FindArrowPython.cmake +%{_libdir}/libarrow_python.a +%{_libdir}/libarrow_python.so +%{_libdir}/pkgconfig/arrow-python.pc + +%if %{use_flight} +%package python-flight-libs +Summary: Python integration library for Apache Arrow Flight +License: Apache-2.0 +Requires: %{name}-flight-libs = %{version}-%{release} +Requires: %{name}-python-libs = %{version}-%{release} + +%description python-flight-libs +This package contains the Python integration library for Apache Arrow Flight. + +%files python-flight-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libarrow_python_flight.so.* + +%package python-flight-devel +Summary: Libraries and header files for Python integration library for Apache Arrow Flight. +License: Apache-2.0 +Requires: %{name}-flight-devel = %{version}-%{release} +Requires: %{name}-python-devel = %{version}-%{release} +Requires: %{name}-python-flight-libs = %{version}-%{release} + +%description python-flight-devel +Libraries and header files for Python integration library for +Apache Arrow Flight. + +%files python-flight-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/arrow/python/flight.h +%{_libdir}/cmake/arrow/ArrowPythonFlightConfig*.cmake +%{_libdir}/cmake/arrow/ArrowPythonFlightTargets*.cmake +%{_libdir}/cmake/arrow/FindArrowPythonFlight.cmake +%{_libdir}/libarrow_python_flight.a +%{_libdir}/libarrow_python_flight.so +%{_libdir}/pkgconfig/arrow-python-flight.pc +%endif +%endif + +%package -n plasma-libs +Summary: Runtime libraries for Plasma in-memory object store +License: Apache-2.0 +Requires: %{name}-libs = %{version}-%{release} + +%description -n plasma-libs +This package contains the libraries for Plasma in-memory object store. + +%files -n plasma-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libplasma.so.* + +%package -n plasma-store-server +Summary: Server for Plasma in-memory object store +License: Apache-2.0 +Requires: plasma-libs = %{version}-%{release} + +%description -n plasma-store-server +This package contains the server for Plasma in-memory object store. + +%files -n plasma-store-server +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_bindir}/plasma-store-server + +%package -n plasma-devel +Summary: Libraries and header files for Plasma in-memory object store +License: Apache-2.0 +Requires: plasma-libs = %{version}-%{release} + +%description -n plasma-devel +Libraries and header files for Plasma in-memory object store. + +%files -n plasma-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/plasma/ +%{_libdir}/cmake/arrow/PlasmaConfig*.cmake +%{_libdir}/cmake/arrow/PlasmaTargets*.cmake +%{_libdir}/cmake/arrow/FindPlasma.cmake +%{_libdir}/libplasma.a +%{_libdir}/libplasma.so +%{_libdir}/pkgconfig/plasma*.pc + +%package -n parquet-libs +Summary: Runtime libraries for Apache Parquet C++ +License: Apache-2.0 +Requires: %{name}-libs = %{version}-%{release} +Requires: openssl + +%description -n parquet-libs +This package contains the libraries for Apache Parquet C++. + +%files -n parquet-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libparquet.so.* + +%package -n parquet-devel +Summary: Libraries and header files for Apache Parquet C++ +License: Apache-2.0 +Requires: parquet-libs = %{version}-%{release} +Requires: zlib-devel + +%description -n parquet-devel +Libraries and header files for Apache Parquet C++. + +%files -n parquet-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/parquet/ +%{_libdir}/cmake/arrow/ParquetConfig*.cmake +%{_libdir}/cmake/arrow/ParquetTargets*.cmake +%{_libdir}/cmake/arrow/FindParquet.cmake +%{_libdir}/libparquet.a +%{_libdir}/libparquet.so +%{_libdir}/pkgconfig/parquet*.pc + +%package glib-libs +Summary: Runtime libraries for Apache Arrow GLib +License: Apache-2.0 +Requires: %{name}-libs = %{version}-%{release} +Requires: glib2 + +%description glib-libs +This package contains the libraries for Apache Arrow GLib. + +%files glib-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libarrow-glib.so.* +%{_datadir}/gir-1.0/Arrow-1.0.gir + +%package glib-devel +Summary: Libraries and header files for Apache Arrow GLib +License: Apache-2.0 +Requires: %{name}-devel = %{version}-%{release} +Requires: glib2-devel +Requires: gobject-introspection-devel + +%description glib-devel +Libraries and header files for Apache Arrow GLib. + +%files glib-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/arrow-glib/ +%{_libdir}/libarrow-glib.a +%{_libdir}/libarrow-glib.so +%{_libdir}/pkgconfig/arrow-glib.pc +%{_libdir}/pkgconfig/arrow-orc-glib.pc +%{_libdir}/girepository-1.0/Arrow-1.0.typelib +%{_datadir}/arrow-glib/example/ + +%package glib-doc +Summary: Documentation for Apache Arrow GLib +License: Apache-2.0 + +%description glib-doc +Documentation for Apache Arrow GLib. + +%files glib-doc +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_docdir}/arrow-glib/ +%{_datadir}/gtk-doc/html/arrow-glib/ + +%package dataset-glib-libs +Summary: Runtime libraries for Apache Arrow Dataset GLib +License: Apache-2.0 +Requires: %{name}-dataset-libs = %{version}-%{release} +Requires: %{name}-glib-libs = %{version}-%{release} + +%description dataset-glib-libs +This package contains the libraries for Apache Arrow Dataset GLib. + +%files dataset-glib-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libarrow-dataset-glib.so.* +%{_datadir}/gir-1.0/ArrowDataset-1.0.gir + +%package dataset-glib-devel +Summary: Libraries and header files for Apache Arrow Dataset GLib +License: Apache-2.0 +Requires: %{name}-dataset-devel = %{version}-%{release} +Requires: %{name}-glib-devel = %{version}-%{release} + +%description dataset-glib-devel +Libraries and header files for Apache Arrow Dataset GLib. + +%files dataset-glib-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/arrow-dataset-glib/ +%{_libdir}/libarrow-dataset-glib.a +%{_libdir}/libarrow-dataset-glib.so +%{_libdir}/pkgconfig/arrow-dataset-glib.pc +%{_libdir}/girepository-1.0/ArrowDataset-1.0.typelib + +%package dataset-glib-doc +Summary: Documentation for Apache Arrow Dataset GLib +License: Apache-2.0 + +%description dataset-glib-doc +Documentation for Apache Arrow dataset GLib. + +%files dataset-glib-doc +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_datadir}/gtk-doc/html/arrow-dataset-glib/ + +%if %{use_flight} +%package flight-glib-libs +Summary: Runtime libraries for Apache Arrow Flight GLib +License: Apache-2.0 +Requires: %{name}-flight-libs = %{version}-%{release} +Requires: %{name}-glib-libs = %{version}-%{release} + +%description flight-glib-libs +This package contains the libraries for Apache Arrow Flight GLib. + +%files flight-glib-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libarrow-flight-glib.so.* +%{_datadir}/gir-1.0/ArrowFlight-1.0.gir + +%package flight-glib-devel +Summary: Libraries and header files for Apache Arrow Flight GLib +License: Apache-2.0 +Requires: %{name}-flight-devel = %{version}-%{release} +Requires: %{name}-glib-devel = %{version}-%{release} + +%description flight-glib-devel +Libraries and header files for Apache Arrow Flight GLib. + +%files flight-glib-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/arrow-flight-glib/ +%{_libdir}/libarrow-flight-glib.a +%{_libdir}/libarrow-flight-glib.so +%{_libdir}/pkgconfig/arrow-flight-glib.pc +%{_libdir}/girepository-1.0/ArrowFlight-1.0.typelib + +%package flight-glib-doc +Summary: Documentation for Apache Arrow Flight GLib +License: Apache-2.0 + +%description flight-glib-doc +Documentation for Apache Arrow Flight GLib. + +%files flight-glib-doc +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_datadir}/gtk-doc/html/arrow-flight-glib/ +%endif + +%if %{use_gandiva} +%package -n gandiva-glib-libs +Summary: Runtime libraries for Gandiva GLib +License: Apache-2.0 +Requires: gandiva-libs = %{version}-%{release} +Requires: %{name}-glib-libs = %{version}-%{release} + +%description -n gandiva-glib-libs +This package contains the libraries for Gandiva GLib. + +%files -n gandiva-glib-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libgandiva-glib.so.* +%{_datadir}/gir-1.0/Gandiva-1.0.gir + +%package -n gandiva-glib-devel +Summary: Libraries and header files for Gandiva GLib +License: Apache-2.0 +Requires: gandiva-devel = %{version}-%{release} +Requires: %{name}-glib-devel = %{version}-%{release} + +%description -n gandiva-glib-devel +Libraries and header files for Gandiva GLib. + +%files -n gandiva-glib-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/gandiva-glib/ +%{_libdir}/libgandiva-glib.a +%{_libdir}/libgandiva-glib.so +%{_libdir}/pkgconfig/gandiva-glib.pc +%{_libdir}/girepository-1.0/Gandiva-1.0.typelib + +%package -n gandiva-glib-doc +Summary: Documentation for Gandiva GLib +License: Apache-2.0 + +%description -n gandiva-glib-doc +Documentation for Gandiva GLib. + +%files -n gandiva-glib-doc +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_datadir}/gtk-doc/html/gandiva-glib/ +%endif + +%package -n plasma-glib-libs +Summary: Runtime libraries for Plasma GLib +License: Apache-2.0 +Requires: plasma-libs = %{version}-%{release} +Requires: %{name}-glib-libs = %{version}-%{release} + +%description -n plasma-glib-libs +This package contains the libraries for Plasma GLib. + +%files -n plasma-glib-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libplasma-glib.so.* +%{_datadir}/gir-1.0/Plasma-1.0.gir + +%package -n plasma-glib-devel +Summary: Libraries and header files for Plasma GLib +License: Apache-2.0 +Requires: plasma-devel = %{version}-%{release} +Requires: %{name}-glib-devel = %{version}-%{release} + +%description -n plasma-glib-devel +Libraries and header files for Plasma GLib. + +%files -n plasma-glib-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/plasma-glib/ +%{_libdir}/libplasma-glib.a +%{_libdir}/libplasma-glib.so +%{_libdir}/pkgconfig/plasma-glib.pc +%{_libdir}/girepository-1.0/Plasma-1.0.typelib + +%package -n plasma-glib-doc +Summary: Documentation for Plasma GLib +License: Apache-2.0 + +%description -n plasma-glib-doc +Documentation for Plasma GLib. + +%files -n plasma-glib-doc +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_datadir}/gtk-doc/html/plasma-glib/ + +%package -n parquet-glib-libs +Summary: Runtime libraries for Apache Parquet GLib +License: Apache-2.0 +Requires: parquet-libs = %{version}-%{release} +Requires: %{name}-glib-libs = %{version}-%{release} + +%description -n parquet-glib-libs +This package contains the libraries for Apache Parquet GLib. + +%files -n parquet-glib-libs +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_libdir}/libparquet-glib.so.* +%{_datadir}/gir-1.0/Parquet-1.0.gir + +%package -n parquet-glib-devel +Summary: Libraries and header files for Apache Parquet GLib +License: Apache-2.0 +Requires: parquet-devel = %{version}-%{release} +Requires: %{name}-glib-devel = %{version}-%{release} + +%description -n parquet-glib-devel +Libraries and header files for Apache Parquet GLib. + +%files -n parquet-glib-devel +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_includedir}/parquet-glib/ +%{_libdir}/libparquet-glib.a +%{_libdir}/libparquet-glib.so +%{_libdir}/pkgconfig/parquet-glib.pc +%{_libdir}/girepository-1.0/Parquet-1.0.typelib + +%package -n parquet-glib-doc +Summary: Documentation for Apache Parquet GLib +License: Apache-2.0 + +%description -n parquet-glib-doc +Documentation for Apache Parquet GLib. + +%files -n parquet-glib-doc +%defattr(-,root,root,-) +%doc README.md LICENSE.txt NOTICE.txt +%{_datadir}/gtk-doc/html/parquet-glib/ + +%changelog +* Wed Nov 10 2021 Sutou Kouhei <kou@clear-code.com> - 6.0.1-1 +- New upstream release. + +* Thu Oct 21 2021 Krisztián Szűcs <szucs.krisztian@gmail.com> - 6.0.0-1 +- New upstream release. + +* Mon Jan 18 2021 Krisztián Szűcs <szucs.krisztian@gmail.com> - 3.0.0-1 +- New upstream release. + +* Mon Oct 12 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 2.0.0-1 +- New upstream release. + +* Mon Jul 20 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 1.0.0-1 +- New upstream release. + +* Thu Apr 16 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 0.17.0-1 +- New upstream release. + +* Thu Jan 30 2020 Krisztián Szűcs <szucs.krisztian@gmail.com> - 0.16.0-1 +- New upstream release. + +* Mon Sep 30 2019 Krisztián Szűcs <szucs.krisztian@gmail.com> - 0.15.0-1 +- New upstream release. + +* Fri Jun 28 2019 Sutou Kouhei <kou@clear-code.com> - 0.14.0-1 +- New upstream release. + +* Thu Mar 28 2019 Kouhei Sutou <kou@clear-code.com> - 0.13.0-1 +- New upstream release. + +* Wed Jan 16 2019 Krisztián Szűcs <szucs.krisztian@gmail.com> - 0.12.0-1 +- New upstream release. + +* Thu Oct 04 2018 Kouhei Sutou <kou@clear-code.com> - 0.11.0-1 +- New upstream release. + +* Thu Aug 02 2018 Phillip Cloud <cpcloud@gmail.com> - 0.10.0-1 +- New upstream release. + +* Fri Mar 16 2018 Kouhei Sutou <kou@clear-code.com> - 0.9.0-1 +- New upstream release. + +* Sun Dec 17 2017 Uwe Korn <uwelk@xhochy.com> - 0.8.0-1 +- New upstream release. + +* Wed Sep 27 2017 Kouhei Sutou <kou@clear-code.com> - 0.7.1-1 +- New upstream release. + +* Tue Sep 12 2017 Wes McKinney <wes.mckinney@twosigma.com> - 0.7.0-1 +- New upstream release. + +* Fri Aug 11 2017 Kouhei Sutou <kou@clear-code.com> - 0.6.0-1 +- New upstream release. + +* Wed Aug 02 2017 Kouhei Sutou <kou@clear-code.com> - 0.6.0.20170802-1 +- New upstream release. diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile new file mode 100644 index 000000000..6856e3854 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/centos-7/Dockerfile @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=centos:7 +FROM ${FROM} + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + yum update -y ${quiet} && \ + yum install -y ${quiet} epel-release && \ + yum install -y ${quiet} \ + bison \ + boost169-devel \ + brotli-devel \ + bzip2-devel \ + ccache \ + cmake3 \ + flex \ + gcc-c++ \ + gflags-devel \ + git \ + glog-devel \ + gobject-introspection-devel \ + gtk-doc \ + libzstd-devel \ + lz4-devel \ + make \ + ninja-build \ + openssl-devel \ + pkg-config \ + python36 \ + python36-devel \ + python36-numpy \ + rapidjson-devel \ + rpmdevtools \ + snappy-devel \ + tar \ + zlib-devel && \ + yum clean ${quiet} all + +ENV \ + BOOST_INCLUDEDIR=/usr/include/boost169 \ + BOOST_LIBRARYDIR=/usr/lib64/boost169 diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/centos-8-aarch64/from b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/centos-8-aarch64/from new file mode 100644 index 000000000..587ce9d4a --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/centos-8-aarch64/from @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +arm64v8/centos:8 diff --git a/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile new file mode 100644 index 000000000..ad145c4ee --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apache-arrow/yum/centos-8/Dockerfile @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG FROM=centos:8 +FROM ${FROM} + +ARG DEBUG + +RUN \ + quiet=$([ "${DEBUG}" = "yes" ] || echo "--quiet") && \ + dnf install -y ${quiet} epel-release && \ + dnf install --enablerepo=powertools -y ${quiet} \ + bison \ + boost-devel \ + brotli-devel \ + bzip2-devel \ + c-ares-devel \ + ccache \ + clang \ + cmake \ + curl-devel \ + flex \ + gcc-c++ \ + gflags-devel \ + git \ + glog-devel \ + gobject-introspection-devel \ + gtk-doc \ + libarchive \ + libzstd-devel \ + llvm-devel \ + llvm-static \ + lz4-devel \ + make \ + ncurses-devel \ + ninja-build \ + openssl-devel \ + pkg-config \ + python3 \ + python3-devel \ + python3-numpy \ + python3-pip \ + re2-devel \ + # rapidjson-devel \ + rpmdevtools \ + snappy-devel \ + tar \ + # utf8proc-devel \ + zlib-devel && \ + dnf clean ${quiet} all diff --git a/src/arrow/dev/tasks/linux-packages/apt/build.sh b/src/arrow/dev/tasks/linux-packages/apt/build.sh new file mode 100755 index 000000000..a54567a5c --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/apt/build.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# -*- sh-indentation: 2; sh-basic-offset: 2 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +LANG=C + +set -u + +run() +{ + "$@" + if test $? -ne 0; then + echo "Failed $@" + exit 1 + fi +} + +. /host/env.sh + +distribution=$(lsb_release --id --short | tr 'A-Z' 'a-z') +code_name=$(lsb_release --codename --short) +case "${distribution}" in + debian) + component=main + ;; + ubuntu) + component=universe + ;; +esac +architecture=$(dpkg-architecture -q DEB_BUILD_ARCH) + +debuild_options=() +dpkg_buildpackage_options=(-us -uc) + +run mkdir -p /build +run cd /build +find . -not -path ./ccache -a -not -path "./ccache/*" -delete +if which ccache > /dev/null 2>&1; then + export CCACHE_COMPILERCHECK=content + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=6 + export CCACHE_DIR="${PWD}/ccache" + export CCACHE_MAXSIZE=500M + ccache --show-stats + debuild_options+=(-eCCACHE_COMPILERCHECK) + debuild_options+=(-eCCACHE_COMPRESS) + debuild_options+=(-eCCACHE_COMPRESSLEVEL) + debuild_options+=(-eCCACHE_DIR) + debuild_options+=(-eCCACHE_MAXSIZE) + if [ -d /usr/lib/ccache ] ;then + debuild_options+=(--prepend-path=/usr/lib/ccache) + fi +fi +run cp /host/tmp/${PACKAGE}-${VERSION}.tar.gz \ + ${PACKAGE}_${VERSION}.orig.tar.gz +run tar xfz ${PACKAGE}_${VERSION}.orig.tar.gz +case "${VERSION}" in + *~dev*) + run mv ${PACKAGE}-$(echo $VERSION | sed -e 's/~dev/-dev/') \ + ${PACKAGE}-${VERSION} + ;; + *~rc*) + run mv ${PACKAGE}-$(echo $VERSION | sed -r -e 's/~rc[0-9]+//') \ + ${PACKAGE}-${VERSION} + ;; +esac +run cd ${PACKAGE}-${VERSION}/ +platform="${distribution}-${code_name}" +if [ -d "/host/tmp/debian.${platform}-${architecture}" ]; then + run cp -rp "/host/tmp/debian.${platform}-${architecture}" debian +elif [ -d "/host/tmp/debian.${platform}" ]; then + run cp -rp "/host/tmp/debian.${platform}" debian +else + run cp -rp "/host/tmp/debian" debian +fi +: ${DEB_BUILD_OPTIONS:="parallel=$(nproc)"} +# DEB_BUILD_OPTIONS="${DEB_BUILD_OPTIONS} noopt" +export DEB_BUILD_OPTIONS +if [ "${DEBUG:-no}" = "yes" ]; then + run debuild "${debuild_options[@]}" "${dpkg_buildpackage_options[@]}" +else + run debuild "${debuild_options[@]}" "${dpkg_buildpackage_options[@]}" > /dev/null +fi +if which ccache > /dev/null 2>&1; then + ccache --show-stats +fi +run cd - + +repositories="/host/repositories" +package_initial=$(echo "${PACKAGE}" | sed -e 's/\(.\).*/\1/') +pool_dir="${repositories}/${distribution}/pool/${code_name}/${component}/${package_initial}/${PACKAGE}" +run mkdir -p "${pool_dir}/" +run \ + find . \ + -maxdepth 1 \ + -type f \ + -not -path '*.build' \ + -not -path '*.buildinfo' \ + -exec cp '{}' "${pool_dir}/" ';' + +run chown -R "$(stat --format "%u:%g" "${repositories}")" "${repositories}" diff --git a/src/arrow/dev/tasks/linux-packages/github.linux.amd64.yml b/src/arrow/dev/tasks/linux-packages/github.linux.amd64.yml new file mode 100644 index 000000000..557c4ab41 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/github.linux.amd64.yml @@ -0,0 +1,138 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + package: + name: Package + runs-on: ubuntu-20.04 + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_login_dockerhub()|indent }} + + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.0' + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + - name: Cache ccache + uses: actions/cache@v2 + with: + path: arrow/dev/tasks/linux-packages/apache-arrow/{{ task_namespace }}/build/{{ target }}/ccache + key: linux-{{ task_namespace }}-ccache-{{ target }}-{{ "${{ hashFiles('arrow/cpp/**') }}" }} + restore-keys: linux-{{ task_namespace }}-ccache-{{ target }}- + - name: Build + run: | + set -e + pushd arrow/dev/tasks/linux-packages + rake version:update + rake docker:pull || : + rake --trace {{ task_namespace }}:build BUILD_DIR=build + sudo rm -rf */*/build + popd + env: + APT_TARGETS: {{ target }} + ARROW_VERSION: {{ arrow.version }} + REPO: {{ '${{ secrets.REPO }}' }} + YUM_TARGETS: {{ target }} + - uses: actions/upload-artifact@v2 + with: + name: packages + path: packages/*/{{ task_namespace }}/repositories/ + - name: Docker Push + continue-on-error: true + shell: bash + run: | + pushd arrow/dev/tasks/linux-packages + rake docker:push + popd + env: + APT_TARGETS: {{ target }} + REPO: {{ '${{ secrets.REPO }}' }} + YUM_TARGETS: {{ target }} + - name: Set up test + run: | + set -e + sudo apt update + # We can install createrepo_c by package with Ubuntu 22.04. + # createrepo_c \ + sudo apt install -y \ + apt-utils \ + devscripts \ + gpg \ + rpm + gem install apt-dists-merge + (echo "Key-Type: RSA"; \ + echo "Key-Length: 4096"; \ + echo "Name-Real: Test"; \ + echo "Name-Email: test@example.com"; \ + echo "%no-protection") | \ + gpg --full-generate-key --batch + GPG_KEY_ID=$(gpg --list-keys --with-colon test@example.com | grep fpr | cut -d: -f10) + echo "GPG_KEY_ID=${GPG_KEY_ID}" >> ${GITHUB_ENV} + gpg --export --armor test@example.com > arrow/dev/tasks/linux-packages/KEYS + # We can install createrepo_c by package with Ubuntu 22.04. + # This is workaround: + - name: Install createrepo_c + run: | + sudo apt install -y \ + cmake \ + libbz2-dev \ + libcurl4-openssl-dev \ + libglib2.0-dev \ + liblzma-dev \ + libmagic-dev \ + librpm-dev \ + libsqlite3-dev \ + libssl-dev \ + libxml2-dev \ + libzstd-dev \ + pkg-config \ + zlib1g-dev + git clone --depth 1 https://github.com/rpm-software-management/createrepo_c.git + pushd createrepo_c + /usr/bin/cmake \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DENABLE_BASHCOMP=OFF \ + -DENABLE_DRPM=OFF \ + -DENABLE_PYTHON=OFF \ + -DWITH_LIBMODULEMD=OFF \ + -DWITH_ZCHUNK=OFF \ + . + make -j$(nproc) + sudo make install + popd + rm -rf createrepo_c + - name: Test + run: | + set -e + pushd arrow/dev/tasks/linux-packages + rake --trace {{ task_namespace }}:test + rm -rf {{ task_namespace }}/repositories + popd + env: + APT_TARGETS: {{ target }} + ARROW_VERSION: {{ arrow.version }} + YUM_TARGETS: {{ target }} + + {% set patterns = upload_extensions | format_all("arrow/dev/tasks/linux-packages/**/*{}") %} + {{ macros.github_upload_releases(patterns)|indent }} diff --git a/src/arrow/dev/tasks/linux-packages/helper.rb b/src/arrow/dev/tasks/linux-packages/helper.rb new file mode 100644 index 000000000..30ac3b898 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/helper.rb @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +module Helper + module ApacheArrow + private + def detect_release_time + release_time_env = ENV["ARROW_RELEASE_TIME"] + if release_time_env + Time.parse(release_time_env).utc + else + latest_commit_time(arrow_source_dir) || Time.now.utc + end + end + + def arrow_source_dir + File.join(__dir__, "..", "..", "..") + end + + def detect_version(release_time) + version_env = ENV["ARROW_VERSION"] + return version_env if version_env + + pom_xml_path = File.join(arrow_source_dir, "java", "pom.xml") + pom_xml_content = File.read(pom_xml_path) + version = pom_xml_content[/^ <version>(.+?)<\/version>/, 1] + formatted_release_time = release_time.strftime("%Y%m%d") + version.gsub(/-SNAPSHOT\z/) {"-dev#{formatted_release_time}"} + end + + def detect_env(name) + value = ENV[name] + return value if value and not value.empty? + + dot_env_path = File.join(arrow_source_dir, ".env") + File.open(dot_env_path) do |dot_env| + dot_env.each_line do |line| + case line.chomp + when /\A#{Regexp.escape(name)}=(.*)/ + return $1 + end + end + end + raise "Failed to detect #{name} environment variable" + end + + def detect_repo + detect_env("REPO") + end + + def docker_image(os, architecture) + architecture ||= "amd64" + "#{detect_repo}:#{architecture}-#{os}-package-#{@package}" + end + end +end diff --git a/src/arrow/dev/tasks/linux-packages/package-task.rb b/src/arrow/dev/tasks/linux-packages/package-task.rb new file mode 100644 index 000000000..9dd1b2d93 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/package-task.rb @@ -0,0 +1,645 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "English" +require "open-uri" +require "time" + +class PackageTask + include Rake::DSL + + def initialize(package, version, release_time, options={}) + @package = package + @version = version + @release_time = release_time + + @archive_base_name = "#{@package}-#{@version}" + @archive_name = "#{@archive_base_name}.tar.gz" + @full_archive_name = File.expand_path(@archive_name) + + @rpm_package = @package + case @version + when /-((dev|rc)\d+)\z/ + base_version = $PREMATCH + sub_version = $1 + type = $2 + if type == "rc" and options[:rc_build_type] == :release + @deb_upstream_version = base_version + @deb_archive_base_name_version = base_version + @rpm_version = base_version + @rpm_release = "1" + else + @deb_upstream_version = "#{base_version}~#{sub_version}" + @deb_archive_base_name_version = @version + @rpm_version = base_version + @rpm_release = "0.#{sub_version}" + end + else + @deb_upstream_version = @version + @deb_archive_base_name_version = @version + @rpm_version = @version + @rpm_release = "1" + end + @deb_release = ENV["DEB_RELEASE"] || "1" + end + + def define + define_dist_task + define_apt_task + define_yum_task + define_version_task + define_docker_tasks + end + + private + def env_value(name) + value = ENV[name] + raise "Specify #{name} environment variable" if value.nil? + value + end + + def debug_build? + ENV["DEBUG"] != "no" + end + + def git_directory?(directory) + candidate_paths = [".git", "HEAD"] + candidate_paths.any? do |candidate_path| + File.exist?(File.join(directory, candidate_path)) + end + end + + def latest_commit_time(git_directory) + return nil unless git_directory?(git_directory) + cd(git_directory) do + return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc + end + end + + def download(url, output_path) + if File.directory?(output_path) + base_name = url.split("/").last + output_path = File.join(output_path, base_name) + end + absolute_output_path = File.expand_path(output_path) + + unless File.exist?(absolute_output_path) + mkdir_p(File.dirname(absolute_output_path)) + rake_output_message "Downloading... #{url}" + open_url(url) do |downloaded_file| + File.open(absolute_output_path, "wb") do |output_file| + IO.copy_stream(downloaded_file, output_file) + end + end + end + + absolute_output_path + end + + def open_url(url, &block) + URI(url).open(&block) + end + + def substitute_content(content) + content.gsub(/@(.+?)@/) do |matched| + yield($1, matched) + end + end + + def docker_image(os, architecture) + image = "#{@package}-#{os}" + image << "-#{architecture}" if architecture + image + end + + def docker_run(os, architecture, console: false) + id = os + id = "#{id}-#{architecture}" if architecture + image = docker_image(os, architecture) + build_command_line = [ + "docker", + "build", + "--cache-from", image, + "--tag", image, + ] + run_command_line = [ + "docker", + "run", + "--rm", + "--log-driver", "none", + "--volume", "#{Dir.pwd}:/host:rw", + ] + if $stdin.tty? + run_command_line << "--interactive" + run_command_line << "--tty" + else + run_command_line.concat(["--attach", "STDOUT"]) + run_command_line.concat(["--attach", "STDERR"]) + end + build_dir = ENV["BUILD_DIR"] + if build_dir + build_dir = "#{File.expand_path(build_dir)}/#{id}" + mkdir_p(build_dir) + run_command_line.concat(["--volume", "#{build_dir}:/build:rw"]) + end + if debug_build? + build_command_line.concat(["--build-arg", "DEBUG=yes"]) + run_command_line.concat(["--env", "DEBUG=yes"]) + end + pass_through_env_names = [ + "DEB_BUILD_OPTIONS", + "RPM_BUILD_NCPUS", + ] + pass_through_env_names.each do |name| + value = ENV[name] + next unless value + run_command_line.concat(["--env", "#{name}=#{value}"]) + end + if File.exist?(File.join(id, "Dockerfile")) + docker_context = id + else + from = File.readlines(File.join(id, "from")).find do |line| + /^[a-z]/i =~ line + end + build_command_line.concat(["--build-arg", "FROM=#{from.chomp}"]) + docker_context = os + end + build_command_line.concat(docker_build_options(os, architecture)) + run_command_line.concat(docker_run_options(os, architecture)) + build_command_line << docker_context + run_command_line << image + run_command_line << "/host/build.sh" unless console + + sh(*build_command_line) + sh(*run_command_line) + end + + def docker_build_options(os, architecture) + [] + end + + def docker_run_options(os, architecture) + [] + end + + def docker_pull(os, architecture) + image = docker_image(os, architecture) + command_line = [ + "docker", + "pull", + image, + ] + command_line.concat(docker_pull_options(os, architecture)) + sh(*command_line) + end + + def docker_pull_options(os, architecture) + [] + end + + def docker_push(os, architecture) + image = docker_image(os, architecture) + command_line = [ + "docker", + "push", + image, + ] + command_line.concat(docker_push_options(os, architecture)) + sh(*command_line) + end + + def docker_push_options(os, architecture) + [] + end + + def define_dist_task + define_archive_task + desc "Create release package" + task :dist => [@archive_name] + end + + def split_target(target) + components = target.split("-") + if components[0, 2] == ["amazon", "linux"] + components[0, 2] = components[0, 2].join("-") + end + if components.size >= 3 + components[2..-1] = components[2..-1].join("-") + end + components + end + + def enable_apt? + true + end + + def apt_targets + return [] unless enable_apt? + + targets = (ENV["APT_TARGETS"] || "").split(",") + targets = apt_targets_default if targets.empty? + + targets.find_all do |target| + Dir.exist?(File.join(apt_dir, target)) + end + end + + def apt_targets_default + # Disable arm64 targets by default for now + # because they require some setups on host. + [ + "debian-buster", + # "debian-buster-arm64", + "debian-bullseye", + # "debian-bullseye-arm64", + "debian-bookworm", + # "debian-bookworm-arm64", + "ubuntu-bionic", + # "ubuntu-bionic-arm64", + "ubuntu-focal", + # "ubuntu-focal-arm64", + "ubuntu-hirsute", + # "ubuntu-hirsute-arm64", + "ubuntu-impish", + # "ubuntu-impish-arm64", + ] + end + + def deb_archive_base_name + "#{@package}-#{@deb_archive_base_name_version}" + end + + def deb_archive_name + "#{@package}-#{@deb_upstream_version}.tar.gz" + end + + def apt_dir + "apt" + end + + def apt_prepare_debian_dir(tmp_dir, target) + source_debian_dir = nil + specific_debian_dir = "debian.#{target}" + distribution, code_name, _architecture = split_target(target) + platform = [distribution, code_name].join("-") + platform_debian_dir = "debian.#{platform}" + if File.exist?(specific_debian_dir) + source_debian_dir = specific_debian_dir + elsif File.exist?(platform_debian_dir) + source_debian_dir = platform_debian_dir + else + source_debian_dir = "debian" + end + + prepared_debian_dir = "#{tmp_dir}/debian.#{target}" + cp_r(source_debian_dir, prepared_debian_dir) + control_in_path = "#{prepared_debian_dir}/control.in" + if File.exist?(control_in_path) + control_in = File.read(control_in_path) + rm_f(control_in_path) + File.open("#{prepared_debian_dir}/control", "w") do |control| + prepared_control = apt_prepare_debian_control(control_in, target) + control.print(prepared_control) + end + end + end + + def apt_prepare_debian_control(control_in, target) + message = "#{__method__} must be defined to use debian/control.in" + raise NotImplementedError, message + end + + def apt_build(console: false) + tmp_dir = "#{apt_dir}/tmp" + rm_rf(tmp_dir) + mkdir_p(tmp_dir) + cp(deb_archive_name, + File.join(tmp_dir, deb_archive_name)) + apt_targets.each do |target| + apt_prepare_debian_dir(tmp_dir, target) + end + + env_sh = "#{apt_dir}/env.sh" + File.open(env_sh, "w") do |file| + file.puts(<<-ENV) +PACKAGE=#{@package} +VERSION=#{@deb_upstream_version} + ENV + end + + apt_targets.each do |target| + cd(apt_dir) do + distribution, version, architecture = split_target(target) + os = "#{distribution}-#{version}" + docker_run(os, architecture, console: console) + end + end + end + + def define_apt_task + namespace :apt do + source_build_sh = "#{__dir__}/apt/build.sh" + build_sh = "#{apt_dir}/build.sh" + repositories_dir = "#{apt_dir}/repositories" + + file build_sh => source_build_sh do + cp(source_build_sh, build_sh) + end + + directory repositories_dir + + desc "Build deb packages" + if enable_apt? + build_dependencies = [ + deb_archive_name, + build_sh, + repositories_dir, + ] + else + build_dependencies = [] + end + task :build => build_dependencies do + apt_build if enable_apt? + end + + namespace :build do + desc "Open console" + task :console => build_dependencies do + apt_build(console: true) if enable_apt? + end + end + end + + desc "Release APT repositories" + apt_tasks = [ + "apt:build", + ] + task :apt => apt_tasks + end + + def enable_yum? + true + end + + def yum_targets + return [] unless enable_yum? + + targets = (ENV["YUM_TARGETS"] || "").split(",") + targets = yum_targets_default if targets.empty? + + targets.find_all do |target| + Dir.exist?(File.join(yum_dir, target)) + end + end + + def yum_targets_default + # Disable aarch64 targets by default for now + # because they require some setups on host. + [ + "almalinux-8", + # "almalinux-8-arch64", + "amazon-linux-2", + # "amazon-linux-2-arch64", + "centos-7", + # "centos-7-aarch64", + "centos-8", + # "centos-8-aarch64", + ] + end + + def rpm_archive_base_name + "#{@package}-#{@rpm_version}" + end + + def rpm_archive_name + "#{rpm_archive_base_name}.tar.gz" + end + + def yum_dir + "yum" + end + + def yum_build_sh + "#{yum_dir}/build.sh" + end + + def yum_expand_variable(key) + case key + when "PACKAGE" + @rpm_package + when "VERSION" + @rpm_version + when "RELEASE" + @rpm_release + else + nil + end + end + + def yum_spec_in_path + "#{yum_dir}/#{@rpm_package}.spec.in" + end + + def yum_build(console: false) + tmp_dir = "#{yum_dir}/tmp" + rm_rf(tmp_dir) + mkdir_p(tmp_dir) + cp(rpm_archive_name, + File.join(tmp_dir, rpm_archive_name)) + + env_sh = "#{yum_dir}/env.sh" + File.open(env_sh, "w") do |file| + file.puts(<<-ENV) +SOURCE_ARCHIVE=#{rpm_archive_name} +PACKAGE=#{@rpm_package} +VERSION=#{@rpm_version} +RELEASE=#{@rpm_release} + ENV + end + + spec = "#{tmp_dir}/#{@rpm_package}.spec" + spec_in_data = File.read(yum_spec_in_path) + spec_data = substitute_content(spec_in_data) do |key, matched| + yum_expand_variable(key) || matched + end + File.open(spec, "w") do |spec_file| + spec_file.print(spec_data) + end + + yum_targets.each do |target| + cd(yum_dir) do + distribution, version, architecture = split_target(target) + os = "#{distribution}-#{version}" + docker_run(os, architecture, console: console) + end + end + end + + def define_yum_task + namespace :yum do + source_build_sh = "#{__dir__}/yum/build.sh" + file yum_build_sh => source_build_sh do + cp(source_build_sh, yum_build_sh) + end + + repositories_dir = "#{yum_dir}/repositories" + directory repositories_dir + + desc "Build RPM packages" + if enable_yum? + build_dependencies = [ + repositories_dir, + rpm_archive_name, + yum_build_sh, + yum_spec_in_path, + ] + else + build_dependencies = [] + end + task :build => build_dependencies do + yum_build if enable_yum? + end + + namespace :build do + desc "Open console" + task :console => build_dependencies do + yum_build(console: true) if enable_yum? + end + end + end + + desc "Release Yum repositories" + yum_tasks = [ + "yum:build", + ] + task :yum => yum_tasks + end + + def define_version_task + namespace :version do + desc "Update versions" + task :update do + update_debian_changelog + update_spec + end + end + end + + def package_changelog_message + "New upstream release." + end + + def packager_name + ENV["DEBFULLNAME"] || ENV["NAME"] || guess_packager_name_from_git + end + + def guess_packager_name_from_git + name = `git config --get user.name`.chomp + return name unless name.empty? + `git log -n 1 --format=%aN`.chomp + end + + def packager_email + ENV["DEBEMAIL"] || ENV["EMAIL"] || guess_packager_email_from_git + end + + def guess_packager_email_from_git + email = `git config --get user.email`.chomp + return email unless email.empty? + `git log -n 1 --format=%aE`.chomp + end + + def update_content(path) + if File.exist?(path) + content = File.read(path) + else + content = "" + end + content = yield(content) + File.open(path, "w") do |file| + file.puts(content) + end + end + + def update_debian_changelog + return unless enable_apt? + + Dir.glob("debian*") do |debian_dir| + update_content("#{debian_dir}/changelog") do |content| + <<-CHANGELOG.rstrip +#{@package} (#{@deb_upstream_version}-#{@deb_release}) unstable; urgency=low + + * New upstream release. + + -- #{packager_name} <#{packager_email}> #{@release_time.rfc2822} + +#{content} + CHANGELOG + end + end + end + + def update_spec + return unless enable_yum? + + release_time = @release_time.strftime("%a %b %d %Y") + update_content(yum_spec_in_path) do |content| + content = content.sub(/^(%changelog\n)/, <<-CHANGELOG) +%changelog +* #{release_time} #{packager_name} <#{packager_email}> - #{@rpm_version}-#{@rpm_release} +- #{package_changelog_message} + + CHANGELOG + content = content.sub(/^(Release:\s+)\d+/, "\\11") + content.rstrip + end + end + + def define_docker_tasks + namespace :docker do + pull_tasks = [] + push_tasks = [] + + (apt_targets + yum_targets).each do |target| + distribution, version, architecture = split_target(target) + os = "#{distribution}-#{version}" + + namespace :pull do + desc "Pull built image for #{target}" + task target do + docker_pull(os, architecture) + end + pull_tasks << "docker:pull:#{target}" + end + + namespace :push do + desc "Push built image for #{target}" + task target do + docker_push(os, architecture) + end + push_tasks << "docker:push:#{target}" + end + end + + desc "Pull built images" + task :pull => pull_tasks + + desc "Push built images" + task :push => push_tasks + end + end +end diff --git a/src/arrow/dev/tasks/linux-packages/travis.linux.arm64.yml b/src/arrow/dev/tasks/linux-packages/travis.linux.arm64.yml new file mode 100644 index 000000000..3703f4c46 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/travis.linux.arm64.yml @@ -0,0 +1,155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +arch: arm64-graviton2 +virt: vm +os: linux +dist: focal +group: edge +language: minimal + +addons: + apt: + packages: + - apt-utils + # We need Ubuntu 20.10 or later + # - createrepo_c + - devscripts + - gpg + - libgit2-dev + - python3-pip + - rake + - rpm + + # https://bugs.launchpad.net/ubuntu/+source/glibc/+bug/1916485 + # We need to use runc 1.0.0~rc93 or later from focal-updated. + - runc + + # To build createrepo_c from source. + # We can remove them when we can install createrepo_c package + - cmake + - libbz2-dev + - libcurl4-openssl-dev + - libglib2.0-dev + - liblzma-dev + - libmagic-dev + - librpm-dev + - libsqlite3-dev + - libssl-dev + - libxml2-dev + - libzstd-dev + - pkg-config + - zlib1g-dev + update: true + +services: + - docker + +# don't build twice +if: tag IS blank + +env: + global: + - APT_TARGETS={{ target }} + - ARROW_VERSION={{ arrow.version }} + - BUILD_REF={{ arrow.head }} + - TRAVIS_TAG={{ task.tag }} + - YUM_TARGETS={{ target }} + +before_script: + - set -e + {{ macros.travis_checkout_arrow() }} + {{ macros.travis_docker_login() }} + + # Build createrepo_c from source. + # We can remove them when we can install createrepo_c package + - git clone --depth 1 https://github.com/rpm-software-management/createrepo_c.git + - pushd createrepo_c + - | + /usr/bin/cmake \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DENABLE_BASHCOMP=OFF \ + -DENABLE_DRPM=OFF \ + -DENABLE_PYTHON=OFF \ + -DWITH_LIBMODULEMD=OFF \ + -DWITH_ZCHUNK=OFF \ + . + - make -j$(nproc) + - sudo make install + - popd + - rm -rf createrepo_c + +script: + # Build packages + - pushd arrow/dev/tasks/linux-packages + - rake version:update + - | + rake docker:pull || : + - pushd apache-arrow-apt-source/apt + - | + for target in debian-* ubuntu-*; do + cp -a ${target} ${target}-arm64 + done + - popd + - pushd apache-arrow-release/yum + - | + for target in almalinux-* centos-*; do + cp -a ${target} ${target}-aarch64 + done + - popd + - | + rake \ + --trace \ + {{ task_namespace }}:build \ + BUILD_DIR=build \ + DEB_BUILD_OPTIONS=parallel=2 \ + RPM_BUILD_NCPUS=2 + - sudo rm -rf */*/build + - popd + # Push Docker image + - pushd arrow/dev/tasks/linux-packages + - | + docker login -u "${DOCKERHUB_USER}" \ + -p "${DOCKERHUB_TOKEN}" || : + - | + rake docker:push || : + - popd + # Test built packages + - sudo gem install apt-dists-merge + - | + (echo "Key-Type: RSA"; \ + echo "Key-Length: 4096"; \ + echo "Name-Real: Test"; \ + echo "Name-Email: test@example.com"; \ + echo "%no-protection") | \ + gpg --full-generate-key --batch + - | + GPG_KEY_ID=$(gpg --list-keys --with-colon test@example.com | grep fpr | cut -d: -f10) + - gpg --export --armor test@example.com > arrow/dev/tasks/linux-packages/KEYS + - pushd arrow/dev/tasks/linux-packages + - | + rake --trace {{ task_namespace }}:test \ + CREATEREPO=createrepo_c \ + GPG_KEY_ID=${GPG_KEY_ID} + - rm -rf {{ task_namespace }}/repositories + - popd + +after_success: + {% set patterns = upload_extensions | format_all("arrow/dev/tasks/linux-packages/**/*{}") %} + {{ macros.travis_upload_releases(patterns) }} diff --git a/src/arrow/dev/tasks/linux-packages/yum/build.sh b/src/arrow/dev/tasks/linux-packages/yum/build.sh new file mode 100755 index 000000000..5224f23e9 --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/yum/build.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# -*- sh-indentation: 2; sh-basic-offset: 2 -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -u + +run() +{ + "$@" + if test $? -ne 0; then + echo "Failed $@" + exit 1 + fi +} + +rpmbuild_options= + +. /host/env.sh + +if grep -q amazon /etc/system-release-cpe; then + distribution=$(cut -d ":" -f 5 /etc/system-release-cpe | tr '_' '-') + distribution_version=$(cut -d ":" -f 6 /etc/system-release-cpe) +else + distribution=$(cut -d ":" -f 4 /etc/system-release-cpe) + distribution_version=$(cut -d ":" -f 5 /etc/system-release-cpe) +fi +distribution_version=$(echo ${distribution_version} | sed -e 's/\..*$//g') + +architecture="$(arch)" +lib_directory=/usr/lib64 +case "${architecture}" in + i*86) + architecture=i386 + lib_directory=/usr/lib + ;; +esac + +run mkdir -p /build +run cd /build +find . -not -path ./ccache -a -not -path "./ccache/*" -delete +if which ccache > /dev/null 2>&1; then + export CCACHE_COMPILERCHECK=content + export CCACHE_COMPRESS=1 + export CCACHE_COMPRESSLEVEL=6 + export CCACHE_MAXSIZE=500M + export CCACHE_DIR="${PWD}/ccache" + ccache --show-stats + if [ -d "${lib_directory}/ccache" ]; then + PATH="${lib_directory}/ccache:$PATH" + fi +fi + +run mkdir -p rpmbuild +run cd +rm -rf rpmbuild +run ln -fs /build/rpmbuild ./ +if [ -x /usr/bin/rpmdev-setuptree ]; then + rm -rf .rpmmacros + run rpmdev-setuptree +else + run cat <<RPMMACROS > ~/.rpmmacros +%_topdir ${HOME}/rpmbuild +RPMMACROS + run mkdir -p rpmbuild/SOURCES + run mkdir -p rpmbuild/SPECS + run mkdir -p rpmbuild/BUILD + run mkdir -p rpmbuild/RPMS + run mkdir -p rpmbuild/SRPMS +fi + +repositories="/host/repositories" +repository="${repositories}/${distribution}/${distribution_version}" +rpm_dir="${repository}/${architecture}/Packages" +srpm_dir="${repository}/source/SRPMS" +run mkdir -p "${rpm_dir}" "${srpm_dir}" + +# for debug +# rpmbuild_options="$rpmbuild_options --define 'optflags -O0 -g3'" + +if [ -n "${SOURCE_ARCHIVE}" ]; then + case "${RELEASE}" in + 0.dev*) + source_archive_base_name=$( \ + echo ${SOURCE_ARCHIVE} | sed -e 's/\.tar\.gz$//') + run tar xf /host/tmp/${SOURCE_ARCHIVE} \ + --transform="s,^[^/]*,${PACKAGE}," + run mv \ + ${PACKAGE} \ + ${source_archive_base_name} + run tar czf \ + rpmbuild/SOURCES/${SOURCE_ARCHIVE} \ + ${source_archive_base_name} + run rm -rf ${source_archive_base_name} + ;; + *) + run cp /host/tmp/${SOURCE_ARCHIVE} rpmbuild/SOURCES/ + ;; + esac +else + run cp /host/tmp/${PACKAGE}-${VERSION}.* rpmbuild/SOURCES/ +fi +run cp \ + /host/tmp/${PACKAGE}.spec \ + rpmbuild/SPECS/ + +run cat <<BUILD > build.sh +#!/usr/bin/env bash + +rpmbuild -ba ${rpmbuild_options} rpmbuild/SPECS/${PACKAGE}.spec +BUILD +run chmod +x build.sh +if [ -n "${SCL:-}" ]; then + run cat <<WHICH_STRIP > which-strip.sh +#!/usr/bin/env bash + +which strip +WHICH_STRIP + run chmod +x which-strip.sh + run cat <<USE_SCL_STRIP >> ~/.rpmmacros +%__strip $(run scl enable ${SCL} ./which-strip.sh) +USE_SCL_STRIP + if [ "${DEBUG:-no}" = "yes" ]; then + run scl enable ${SCL} ./build.sh + else + run scl enable ${SCL} ./build.sh > /dev/null + fi +else + if [ "${DEBUG:-no}" = "yes" ]; then + run ./build.sh + else + run ./build.sh > /dev/null + fi +fi + +if which ccache > /dev/null 2>&1; then + ccache --show-stats +fi + +run mv rpmbuild/RPMS/*/* "${rpm_dir}/" +run mv rpmbuild/SRPMS/* "${srpm_dir}/" + +run chown -R "$(stat --format "%u:%g" "${repositories}")" "${repositories}" diff --git a/src/arrow/dev/tasks/macros.jinja b/src/arrow/dev/tasks/macros.jinja new file mode 100644 index 000000000..be265caa4 --- /dev/null +++ b/src/arrow/dev/tasks/macros.jinja @@ -0,0 +1,198 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{%- macro github_header() -%} +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow +on: + push: + branches: + - "*-github-*" +{% endmacro %} + +{%- macro github_checkout_arrow() -%} + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow config core.symlinks true + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Fetch Submodules and Tags + shell: bash + run: cd arrow && ci/scripts/util_checkout.sh +{% endmacro %} + +{%- macro github_login_dockerhub() -%} + - name: Login to Dockerhub + uses: docker/login-action@v1 + with: + username: {{ '${{ secrets.DOCKERHUB_USER }}' }} + password: {{ '${{ secrets.DOCKERHUB_TOKEN }}' }} +{% endmacro %} + +{%- macro github_login_ghcr() -%} + - name: Login to GitHub Container Registry + shell: bash + run: docker login ghcr.io -u {{ '${{ github.repository_owner }}' }} -p {{ '${{ secrets.CROSSBOW_GHCR_TOKEN }}' }} +{% endmacro %} + +{%- macro github_install_archery() -%} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install Archery + shell: bash + run: pip install -e arrow/dev/archery[all] +{% endmacro %} + +{%- macro github_upload_releases(pattern) -%} + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Setup Crossbow + shell: bash + run: pip install -e arrow/dev/archery[crossbow-upload] + - name: Upload artifacts + shell: bash + run: | + archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + {% if pattern is string %} + "{{ pattern }}" + {% elif pattern is iterable %} + {% for p in pattern %} + "{{ p }}" {{ "\\" if not loop.last else "" }} + {% endfor %} + {% endif %} + env: + CROSSBOW_GITHUB_TOKEN: {{ '${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}' }} +{% endmacro %} + +{%- macro github_upload_gemfury(pattern) -%} + {%- if arrow.branch == 'master' -%} + - name: Upload package to Gemfury + shell: bash + run: | + path=$(ls {{ pattern }}) + curl -F "package=@${path}" https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/ + env: + CROSSBOW_GEMFURY_TOKEN: {{ '${{ secrets.CROSSBOW_GEMFURY_TOKEN }}' }} + CROSSBOW_GEMFURY_ORG: {{ '${{ secrets.CROSSBOW_GEMFURY_ORG }}' }} + {% endif %} +{% endmacro %} + +{%- macro azure_checkout_arrow() -%} + - script: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + displayName: Clone arrow +{% endmacro %} + +{%- macro azure_upload_releases(pattern) -%} + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.8' + - script: pip install -e arrow/dev/archery[crossbow-upload] + displayName: Install Crossbow + - bash: | + archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + {% if pattern is string %} + "{{ pattern }}" + {% elif pattern is iterable %} + {% for p in pattern %} + "{{ p }}" {{ "\\" if not loop.last else "" }} + {% endfor %} + {% endif %} + env: + CROSSBOW_GITHUB_TOKEN: $(CROSSBOW_GITHUB_TOKEN) + displayName: Upload packages as a GitHub release +{% endmacro %} + +{%- macro azure_upload_anaconda(pattern) -%} + {%- if arrow.branch == 'master' -%} + - task: CondaEnvironment@1 + inputs: + packageSpecs: 'anaconda-client' + installOptions: '-c conda-forge' + updateConda: no + - script: | + conda install -y anaconda-client + anaconda -t $(CROSSBOW_ANACONDA_TOKEN) upload --force {{ pattern }} + displayName: Upload packages to Anaconda + {% endif %} +{% endmacro %} + +{%- macro travis_checkout_arrow() -%} + - git clone --no-checkout {{ arrow.remote }} arrow + - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + - git -C arrow checkout FETCH_HEAD + - git -C arrow submodule update --init --recursive +{% endmacro %} + +{%- macro travis_install_archery() -%} + - sudo -H pip3 install --upgrade pip + - sudo -H pip3 install docker-compose + - sudo -H pip3 install -e arrow/dev/archery[docker] +{% endmacro %} + +{%- macro travis_docker_login() -%} + - echo "${DOCKERHUB_TOKEN}" | docker login --username "${DOCKERHUB_USER}" --password-stdin +{% endmacro %} + +{%- macro travis_upload_releases(pattern) -%} + - sudo -H pip3 install pygit2==1.0 + - sudo -H pip3 install -e arrow/dev/archery[crossbow-upload] + - | + archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + {% if pattern is string %} + "{{ pattern }}" + {% elif pattern is iterable %} + {% for p in pattern %} + "{{ p }}" {{ "\\" if not loop.last else "" }} + {% endfor %} + {% endif %} +{% endmacro %} + +{%- macro travis_upload_gemfury(pattern) -%} + {%- if arrow.branch == 'master' -%} + - | + WHEEL_PATH=$(echo arrow/python/repaired_wheels/*.whl) + curl \ + -F "package=@${WHEEL_PATH}" \ + "https://${CROSSBOW_GEMFURY_TOKEN}@push.fury.io/${CROSSBOW_GEMFURY_ORG}/" + {% endif %} +{% endmacro %} diff --git a/src/arrow/dev/tasks/nightlies.sample.yml b/src/arrow/dev/tasks/nightlies.sample.yml new file mode 100644 index 000000000..710f7c0ad --- /dev/null +++ b/src/arrow/dev/tasks/nightlies.sample.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# this travis configuration can be used to submit cron scheduled tasks +# 1. copy this file to one of crossbow's branch (master for example) with +# filename .travis.yml +# 2. setup daily cron jobs for that particular branch, see travis' +# documentation https://docs.travis-ci.com/user/cron-jobs/ + +branches: + # don't attempt to build branches intended for windows builds + except: + - /.*win.*/ + +os: linux +dist: trusty +language: generic + +before_install: + # Install Miniconda. + - echo `pwd` + - | + echo "" + echo "Installing a fresh version of Miniconda." + MINICONDA_URL="https://repo.continuum.io/miniconda" + MINICONDA_FILE="Miniconda3-latest-Linux-x86_64.sh" + curl -L -O "${MINICONDA_URL}/${MINICONDA_FILE}" + bash $MINICONDA_FILE -b + + # Configure conda. + - | + echo "" + echo "Configuring conda." + source /home/travis/miniconda3/bin/activate root + conda config --remove channels defaults + conda config --add channels defaults + conda config --add channels conda-forge + conda config --set show_channel_urls true + +install: + - pushd .. + # to build against a specific branch of a fork + # git clone -b <branch> https://github.com/<user>/arrow + - git clone https://github.com/apache/arrow + - pip install dev/archery[crossbow] + +script: + # submit packaging tasks + - | + if [ $TRAVIS_EVENT_TYPE = "cron" ]; then + archery crossbow submit -g conda -g wheel -g linux + else + archery crossbow submit --dry-run -g conda -g wheel -g linux + fi diff --git a/src/arrow/dev/tasks/nuget-packages/github.linux.yml b/src/arrow/dev/tasks/nuget-packages/github.linux.yml new file mode 100644 index 000000000..cd03a7bfe --- /dev/null +++ b/src/arrow/dev/tasks/nuget-packages/github.linux.yml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + package: + name: Package + runs-on: ubuntu-latest + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + + - name: Prepare version + run: | + sed -i'' -E -e \ + "s/^ <Version>.+<\/Version>/ <Version>{{ arrow.no_rc_semver_version }}<\/Version>/" \ + arrow/csharp/Directory.Build.props + - name: Build package + run: | + pushd arrow + archery docker run {{ run }} + popd + + {% set patterns = ["arrow/csharp/artifacts/**/*.nupkg", + "arrow/csharp/artifacts/**/*.snupkg"] %} + {{ macros.github_upload_releases(patterns)|indent }} diff --git a/src/arrow/dev/tasks/python-sdist/github.yml b/src/arrow/dev/tasks/python-sdist/github.yml new file mode 100644 index 000000000..68371876a --- /dev/null +++ b/src/arrow/dev/tasks/python-sdist/github.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + build: + name: "Build sdist" + runs-on: ubuntu-20.04 + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + + - name: Build sdist + run: | + archery docker run python-sdist + {% if arrow.branch == 'master' %} + archery docker push python-sdist || : + {% endif %} + env: + PYARROW_VERSION: {{ arrow.no_rc_version }} + + - name: Test sdist + run: archery docker run ubuntu-python-sdist-test + env: + PYARROW_VERSION: {{ arrow.no_rc_version }} + + {{ macros.github_upload_releases("arrow/python/dist/*.tar.gz")|indent }} + {{ macros.github_upload_gemfury("arrow/python/dist/*.tar.gz")|indent }} diff --git a/src/arrow/dev/tasks/python-wheels/github.linux.amd64.yml b/src/arrow/dev/tasks/python-wheels/github.linux.amd64.yml new file mode 100644 index 000000000..dc2386482 --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/github.linux.amd64.yml @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + build: + name: "Build wheel for Manylinux {{ manylinux_version }}" + runs-on: ubuntu-latest + env: + # archery uses these environment variables + ARCH: amd64 + PYTHON: "{{ python_version }}" + + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_install_archery()|indent }} + {{ macros.github_login_dockerhub()|indent }} + + - name: Build wheel + shell: bash + run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} + + # TODO(kszucs): auditwheel show + - name: Test wheel + shell: bash + run: | + archery docker run python-wheel-manylinux-test-imports + archery docker run python-wheel-manylinux-test-unittests + + {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} + + {% if arrow.branch == 'master' %} + - name: Push Docker Image + shell: bash + run: | + archery docker push python-wheel-manylinux-{{ manylinux_version }} + archery docker push python-wheel-manylinux-test-unittests + {% endif %} diff --git a/src/arrow/dev/tasks/python-wheels/github.osx.amd64.yml b/src/arrow/dev/tasks/python-wheels/github.osx.amd64.yml new file mode 100644 index 000000000..8078abfd5 --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/github.osx.amd64.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +env: + ARROW_S3: {{ arrow_s3 }} + CC: "clang" + CXX: "clang++" + MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}" + PYARROW_BUILD_VERBOSE: 1 + PYARROW_VERSION: "{{ arrow.no_rc_version }}" + PYTHON_VERSION: "{{ python_version }}" + PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}" + VCPKG_DEFAULT_TRIPLET: x64-osx-static-release + VCPKG_FEATURE_FLAGS: "-manifests" + VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} + VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }} + VCPKG_VERSION: "{{ vcpkg_version }}" + +jobs: + build: + name: Build wheel for OS X + runs-on: macos-10.15 + steps: + {{ macros.github_checkout_arrow()|indent }} + + - name: Install System Dependencies + run: brew install bash bison coreutils ninja cmake + + - uses: actions/cache@v2 + id: vcpkg-cache + with: + path: vcpkg + key: vcpkg-{{ macos_deployment_target }}-{{ vcpkg_version }}-{{ "${{ hashFiles('arrow/ci/vcpkg/*.patch', 'arrow/ci/vcpkg/*osx*.cmake') }}" }} + + - name: Install Vcpkg + if: steps.vcpkg-cache.outputs.cache-hit != 'true' + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: "10.15" + run: arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT + + - name: Install Packages + run: | + $VCPKG_ROOT/vcpkg install \ + abseil \ + boost-filesystem \ + brotli \ + bzip2 \ + c-ares \ + curl \ + flatbuffers \ + gflags \ + glog \ + grpc \ + lz4 \ + openssl \ + orc \ + protobuf \ + rapidjson \ + re2 \ + snappy \ + thrift \ + utf8proc \ + zlib \ + zstd + + {% if arrow_s3 == "ON" %} + - name: Install AWS SDK C++ + run: $VCPKG_ROOT/vcpkg install aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] + {% endif %} + + - name: Install Python {{ python_version }} + shell: bash + run: sudo arrow/ci/scripts/install_python.sh macos {{ python_version }} + + - name: Build Wheel + shell: bash + run: | + $PYTHON -m virtualenv build-env + source build-env/bin/activate + pip install --upgrade pip wheel + arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build + + - name: Test Wheel + shell: bash + run: | + $PYTHON -m virtualenv test-env + source test-env/bin/activate + pip install --upgrade pip wheel + arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + + {{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }} + {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/src/arrow/dev/tasks/python-wheels/github.osx.arm64.yml b/src/arrow/dev/tasks/python-wheels/github.osx.arm64.yml new file mode 100644 index 000000000..e5456dbfc --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/github.osx.arm64.yml @@ -0,0 +1,157 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prerequisits on the host: +# - brew install bash bison coreutils ninja cmake +# - sudo arrow/ci/scripts/install_python.sh macos 3.9 + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +env: + ARROW_FLIGHT: OFF + ARROW_JEMALLOC: OFF + ARROW_SIMD_LEVEL: "{{ arrow_simd_level }}" + CC: "clang" + CMAKE_BUILD_TYPE: release + CMAKE_CXX_COMPILER_LAUNCHER: "ccache" + CXX: "clang++" + MACOSX_DEPLOYMENT_TARGET: "{{ macos_deployment_target }}" + PYARROW_BUILD_VERBOSE: 1 + PYARROW_VERSION: "{{ arrow.no_rc_version }}" + PYTHON_VERSION: "{{ python_version }}" + PYTHON: "/Library/Frameworks/Python.framework/Versions/{{ python_version }}/bin/python{{ python_version }}" + VCPKG_DEFAULT_TRIPLET: {{ arch }}-osx-static-release + VCPKG_FEATURE_FLAGS: "-manifests" + VCPKG_OVERLAY_TRIPLETS: {{ "${{ github.workspace }}/arrow/ci/vcpkg" }} + VCPKG_ROOT: {{ "${{ github.workspace }}/vcpkg" }} + VCPKG_VERSION: "{{ vcpkg_version }}" + +jobs: + build: + name: Build wheel for OS X + runs-on: self-hosted + steps: + - name: Cleanup + shell: bash + run: rm -rf arrow vcpkg build crossbow-env build-env test-*-env + + {{ macros.github_checkout_arrow()|indent }} + + - name: Add Brew's Bison to PATH + shell: bash + run: echo "/opt/homebrew/opt/bison/bin" >> $GITHUB_PATH + + - name: Install Vcpkg + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: "11.0" + run: arch -arm64 arrow/ci/scripts/install_vcpkg.sh $VCPKG_VERSION $VCPKG_ROOT + + - name: Install OpenSSL + shell: bash + run: arch -arm64 $VCPKG_ROOT/vcpkg install openssl + + {% if arch == "universal2" %} + # OpenSSL doesn't provide an universal2 configuration yet, so vcpkg is + # unable to propagate the list of architectures from VCPKG_OSX_ARCHIETCTURES. + # In order to prevent link time warnings (which may turn out to be errors) + # we compile OpenSSL separately for the two architectures and merge the + # binaries into universal2 ones using `lipo`. + - name: Create universal binaries for OpenSSL + shell: bash + run: | + for arch in arm64 x64; do + VCPKG_DEFAULT_TRIPLET=${arch}-osx-static-release arch -arm64 $VCPKG_ROOT/vcpkg install openssl + done + for lib in libcrypto libssl; do + lipo -create $VCPKG_ROOT/installed/arm64-osx-static-release/lib/${lib}.a \ + $VCPKG_ROOT/installed/x64-osx-static-release/lib/${lib}.a \ + -output $VCPKG_ROOT/installed/universal2-osx-static-release/lib/${lib}.a + done + {% endif %} + + - name: Install Packages + run: | + arch -arm64 $VCPKG_ROOT/vcpkg install \ + aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ + boost-filesystem \ + brotli \ + bzip2 \ + c-ares \ + curl \ + flatbuffers \ + gflags \ + glog \ + lz4 \ + orc \ + protobuf \ + rapidjson \ + re2 \ + snappy \ + thrift \ + utf8proc \ + zlib \ + zstd + + - name: Build Wheel + shell: bash + run: | + $PYTHON -m virtualenv build-env + source build-env/bin/activate + pip install --upgrade pip wheel + arch -arm64 arrow/ci/scripts/python_wheel_macos_build.sh {{ arch }} $(pwd)/arrow $(pwd)/build + + - name: Test Wheel on ARM64 + shell: bash + env: + PYTEST_ADDOPTS: "-k 'not test_cancellation'" + run: | + $PYTHON -m virtualenv test-arm64-env + source test-arm64-env/bin/activate + pip install --upgrade pip wheel + arch -arm64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + + {% if arch == "universal2" %} + - name: Test Wheel on AMD64 + shell: bash + env: + PYTEST_ADDOPTS: "-k 'not test_cancellation'" + run: | + $PYTHON -m virtualenv test-amd64-env + source test-amd64-env/bin/activate + pip install --upgrade pip wheel + arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow + {% endif %} + + - name: Upload artifacts + shell: bash + run: | + $PYTHON -m virtualenv crossbow-env + source crossbow-env/bin/activate + arch -x86_64 pip install -e arrow/dev/archery[crossbow-upload] + arch -x86_64 archery crossbow \ + --queue-path $(pwd) \ + --queue-remote {{ queue_remote_url }} \ + upload-artifacts \ + --sha {{ task.branch }} \ + --tag {{ task.tag }} \ + "arrow/python/repaired_wheels/*.whl" + env: + CROSSBOW_GITHUB_TOKEN: {{ "${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" }} + + {{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }} diff --git a/src/arrow/dev/tasks/python-wheels/github.windows.yml b/src/arrow/dev/tasks/python-wheels/github.windows.yml new file mode 100644 index 000000000..f9989aed0 --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/github.windows.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + build: + name: "Build wheel for Windows" + runs-on: windows-2019 + env: + # archery uses this environment variable + PYTHON: "{{ python_version }}" + # this is a private repository at the moment (mostly because of licensing + # consideration of windows images with visual studio), but anyone can + # recreate the image by manually building it via: + # `archery build python-wheel-windows-vs2017` + # note that we don't run docker build since there wouldn't be a cache hit + # and rebuilding the dependencies takes a fair amount of time + REPO: ghcr.io/ursacomputing/arrow + # prefer the docker cli over docker-compose + ARCHERY_USE_DOCKER_CLI: 1 + + steps: + {{ macros.github_checkout_arrow()|indent }} + {{ macros.github_login_ghcr()|indent }} + {{ macros.github_install_archery()|indent }} + + - name: Build wheel + shell: cmd + run: archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-windows-vs2017 + + # Two layers of the official python 3.6 windows image are not available for download. + # Docker pull returns with unexpected status resolving reader: 403 Forbidden. + # See https://issues.apache.org/jira/browse/ARROW-14424 + {% if python_version != "3.6" %} + - name: Test wheel + shell: cmd + run: archery docker run python-wheel-windows-test + {% endif %} + + {{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }} + {{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }} diff --git a/src/arrow/dev/tasks/python-wheels/travis.linux.arm64.yml b/src/arrow/dev/tasks/python-wheels/travis.linux.arm64.yml new file mode 100644 index 000000000..d32d89d83 --- /dev/null +++ b/src/arrow/dev/tasks/python-wheels/travis.linux.arm64.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +arch: arm64-graviton2 +virt: vm +os: linux +dist: focal +group: edge +language: minimal + +addons: + apt: + packages: + - libgit2-dev + - python3-pip + +services: + - docker + +# don't build twice +if: tag IS blank + +env: + global: + - BUILD_REF={{ arrow.head }} + - TRAVIS_TAG={{ task.tag }} + # archery uses these environment variables + - ARCH=arm64v8 + - PYTHON="{{ python_version }}" + +before_script: + - set -e + {{ macros.travis_checkout_arrow() }} + {{ macros.travis_docker_login() }} + +script: + # Install Archery and Crossbow dependencies + {{ macros.travis_install_archery() }} + + # Build and Test packages + # output something every minutes to prevent travis from killing the build + - while sleep 1m; do echo "=====[ $SECONDS seconds still running ]====="; done & + - archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }} + - archery docker run python-wheel-manylinux-test-imports + - archery docker run python-wheel-manylinux-test-unittests + - kill %1 + +after_success: + # Upload wheel as github artifact + {{ macros.travis_upload_releases("arrow/python/repaired_wheels/*.whl") }} + {{ macros.travis_upload_gemfury("arrow/python/repaired_wheels/*.whl") }} + + {% if arrow.branch == 'master' %} + # Push the docker image to dockerhub + - archery docker push python-wheel-manylinux-{{ manylinux_version }} + - archery docker push python-wheel-manylinux-test-unittests + {% endif %} diff --git a/src/arrow/dev/tasks/r/azure.linux.yml b/src/arrow/dev/tasks/r/azure.linux.yml new file mode 100644 index 000000000..92e725f68 --- /dev/null +++ b/src/arrow/dev/tasks/r/azure.linux.yml @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +jobs: + - job: linux + pool: + vmImage: ubuntu-latest + timeoutInMinutes: 360 + steps: + - script: | + set -ex + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + displayName: Clone arrow + + - script: | + set -ex + docker -v + docker-compose -v + cd arrow + export R_ORG={{ r_org }} + export R_IMAGE={{ r_image }} + export R_TAG={{ r_tag }} + export DEVTOOLSET_VERSION={{ devtoolset_version|default("-1") }} + docker-compose pull --ignore-pull-failures r + docker-compose build r + displayName: Docker build + + - script: | + set -ex + cd arrow + export R_ORG={{ r_org }} + export R_IMAGE={{ r_image }} + export R_TAG={{ r_tag }} + export ARROW_R_DEV={{ not_cran|default("TRUE") }} + # Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true + # if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off. + docker-compose run {{ flags|default("") }} r + displayName: Docker run + + - script: | + set -ex + cat arrow/r/check/arrow.Rcheck/00install.out + displayName: Dump install logs + condition: succeededOrFailed() + - script: | + set -ex + cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* + displayName: Dump test logs + condition: succeededOrFailed() diff --git a/src/arrow/dev/tasks/r/github.devdocs.yml b/src/arrow/dev/tasks/r/github.devdocs.yml new file mode 100644 index 000000000..5591e6587 --- /dev/null +++ b/src/arrow/dev/tasks/r/github.devdocs.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + devdocs: + name: 'R devdocs {{ "${{ matrix.os }}" }}' + runs-on: {{ "${{ matrix.os }}" }} + strategy: + fail-fast: false + matrix: + os: [macOS-latest, ubuntu-20.04] + + steps: + {{ macros.github_checkout_arrow()|indent }} + + - uses: r-lib/actions/setup-r@v1 + - uses: r-lib/actions/setup-pandoc@v1 + - name: Install knitr, rmarkdown + run: | + install.packages(c("rmarkdown", "knitr", "sessioninfo")) + shell: Rscript {0} + - name: Session info + run: | + options(width = 100) + pkgs <- installed.packages()[, "Package"] + sessioninfo::session_info(pkgs, include_base = TRUE) + shell: Rscript {0} + - name: Remove system gfortran so that brew can install gcc successfully + run: rm -f /usr/local/bin/gfortran + - name: Write the install script + env: + RUN_DEVDOCS: TRUE + DEVDOCS_MACOS: {{ "${{contains(matrix.os, 'macOS')}}" }} + DEVDOCS_UBUNTU: {{ "${{contains(matrix.os, 'ubuntu')}}" }} + run: | + # This isn't actually rendering the docs, but will save arrow/r/vignettes/script.sh + # which can be sourced to install arrow. + rmarkdown::render("arrow/r/vignettes/developing.Rmd") + shell: Rscript {0} + - name: Install from the devdocs + env: + LIBARROW_BINARY: FALSE + ARROW_R_DEV: TRUE + run: bash arrow/r/vignettes/script.sh + shell: bash + - name: Ensure that the Arrow package is loadable and we have the correct one + run: | + echo $LD_LIBRARY_PATH + R --no-save <<EOF + Sys.getenv("LD_LIBRARY_PATH") + library(arrow) + arrow_info() + EOF + shell: bash -l {0} + - name: Save the install script + uses: actions/upload-artifact@v2 + with: + name: {{ "devdocs-script_os-${{ matrix.os }}_sysinstall-${{ matrix.system-install }}" }} + path: arrow/r/vignettes/script.sh + if: always() diff --git a/src/arrow/dev/tasks/r/github.linux.arrow.version.back.compat.yml b/src/arrow/dev/tasks/r/github.linux.arrow.version.back.compat.yml new file mode 100644 index 000000000..e48b67ac6 --- /dev/null +++ b/src/arrow/dev/tasks/r/github.linux.arrow.version.back.compat.yml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push + +jobs: + write-files: + name: "Write files" + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + env: + ARROW_R_DEV: "TRUE" + RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + - name: Fetch Submodules and Tags + shell: bash + run: cd arrow && ci/scripts/util_checkout.sh + - uses: r-lib/actions/setup-r@v1 + - name: Install dependencies + run: | + install.packages(c("remotes", "glue", "sys")) + remotes::install_deps("arrow/r", dependencies = TRUE) + shell: Rscript {0} + - name: Install Arrow + run: | + cd arrow/r + R CMD INSTALL . + shell: bash + - name: Write files + run: | + cd arrow/r + R -f extra-tests/write-files.R + shell: bash + + - name: Upload the parquet artifacts + uses: actions/upload-artifact@v2 + with: + name: files + path: arrow/r/extra-tests/files + + read-files: + name: "Read files with Arrow {{ '${{ matrix.config.old_arrow_version }}' }}" + needs: [write-files] + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + config: + # We use the R version that was released at the time of the arrow release in order + # to make sure we can download binaries from RSPM. + - { old_arrow_version: '5.0.0', r: '4.1' } + - { old_arrow_version: '4.0.0', r: '4.0' } + - { old_arrow_version: '3.0.0', r: '4.0' } + - { old_arrow_version: '2.0.0', r: '4.0' } + - { old_arrow_version: '1.0.1', r: '4.0' } + env: + ARROW_R_DEV: "TRUE" + RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" + OLD_ARROW_VERSION: {{ '${{ matrix.config.old_arrow_version }}' }} + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - uses: r-lib/actions/setup-r@v1 + with: + r-version: {{ '${{ matrix.config.r }}' }} + - name: Install old Arrow + run: | + install.packages(c("remotes", "testthat")) + remotes::install_version("arrow", "{{ '${{ matrix.config.old_arrow_version }}' }}") + shell: Rscript {0} + - name: Setup our testing directory, copy only the tests to it. + run: | + mkdir -p extra-tests/files + cp arrow/r/extra-tests/helper*.R extra-tests/ + cp arrow/r/extra-tests/test-*.R extra-tests/ + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: files + path: extra-tests/files + - name: Test reading + run: | + testthat::test_dir("extra-tests") + shell: Rscript {0} diff --git a/src/arrow/dev/tasks/r/github.linux.cran.yml b/src/arrow/dev/tasks/r/github.linux.cran.yml new file mode 100644 index 000000000..03d22dcbf --- /dev/null +++ b/src/arrow/dev/tasks/r/github.linux.cran.yml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + as-cran: + name: "rhub/{{ MATRIX }}" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # See https://hub.docker.com/r/rhub + r_image: + - debian-gcc-devel + - debian-gcc-patched + - debian-gcc-release + - fedora-gcc-devel + - fedora-clang-devel + env: + R_ORG: "rhub" + R_IMAGE: {{ MATRIX }} + R_TAG: "latest" + ARROW_R_DEV: "FALSE" + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + - name: Fetch Submodules and Tags + shell: bash + run: cd arrow && ci/scripts/util_checkout.sh + - name: Docker Pull + shell: bash + run: cd arrow && docker-compose pull --ignore-pull-failures r + - name: Docker Build + shell: bash + run: cd arrow && docker-compose build r + - name: Docker Run + shell: bash + run: cd arrow && docker-compose run r + - name: Dump install logs + run: cat arrow/r/check/arrow.Rcheck/00install.out + if: always() + - name: Dump test logs + run: cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* + if: always() + - name: Save the test output + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-output + path: arrow/r/check/arrow.Rcheck/tests/testthat.Rout* diff --git a/src/arrow/dev/tasks/r/github.linux.offline.build.yml b/src/arrow/dev/tasks/r/github.linux.offline.build.yml new file mode 100644 index 000000000..60685b18c --- /dev/null +++ b/src/arrow/dev/tasks/r/github.linux.offline.build.yml @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + grab-dependencies: + name: "Download thirdparty dependencies" + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + env: + ARROW_R_DEV: "TRUE" + RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + - name: Fetch Submodules and Tags + shell: bash + run: cd arrow && ci/scripts/util_checkout.sh + - uses: r-lib/actions/setup-r@v1 + - name: Pull Arrow dependencies + run: | + cd arrow/r + # This is `make build`, but with no vignettes and not running `make doc` + cp ../NOTICE.txt inst/NOTICE.txt + rsync --archive --delete ../cpp tools/ + cp -p ../.env tools/ + cp -p ../NOTICE.txt tools/ + cp -p ../LICENSE.txt tools/ + R CMD build --no-build-vignettes --no-manual . + built_tar=$(ls -1 arrow*.tar.gz | head -n 1) + R -e "source('R/install-arrow.R'); create_package_with_all_dependencies(dest_file = 'arrow_with_deps.tar.gz', source_file = \"${built_tar}\")" + shell: bash + - name: Upload the third party dependency artifacts + uses: actions/upload-artifact@v2 + with: + name: thirdparty_deps + path: arrow/r/arrow_with_deps.tar.gz + + intall-offline: + name: "Install offline" + needs: [grab-dependencies] + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + env: + ARROW_R_DEV: TRUE + RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - uses: r-lib/actions/setup-r@v1 + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: thirdparty_deps + path: arrow/r/ + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt install libcurl4-openssl-dev libssl-dev + - name: Install dependencies + run: | + install.packages(c("remotes", "glue", "sys")) + remotes::install_deps("arrow/r", dependencies = TRUE) + shell: Rscript {0} + - name: Install + env: + TEST_OFFLINE_BUILD: true + LIBARROW_MINIMAL: false + run: | + cd arrow/r + R CMD INSTALL --install-tests --no-test-load --no-docs --no-help --no-byte-compile arrow_with_deps.tar.gz + - name: Run the tests + run: R -e 'if(tools::testInstalledPackage("arrow") != 0L) stop("There was a test failure.")' + - name: Dump test logs + run: cat arrow-tests/testthat.Rout* + if: always() + - name: Save the test output + uses: actions/upload-artifact@v2 + with: + name: test-output + path: arrow-tests/testthat.Rout* + if: always() diff --git a/src/arrow/dev/tasks/r/github.linux.rchk.yml b/src/arrow/dev/tasks/r/github.linux.rchk.yml new file mode 100644 index 000000000..72ff26969 --- /dev/null +++ b/src/arrow/dev/tasks/r/github.linux.rchk.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + as-cran: + name: "rchk" + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + env: + ARROW_R_DEV: "FALSE" + RSPM: "https://packagemanager.rstudio.com/cran/__linux__/focal/latest" + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + - uses: r-lib/actions/setup-r@v1 + - uses: r-lib/actions/setup-pandoc@v1 + - name: Install dependencies + run: | + install.packages("remotes") + remotes::install_deps("arrow/r", dependencies = TRUE) + shell: Rscript {0} + - name: Build arrow package + run: | + R CMD build --no-build-vignettes arrow/r + mkdir packages + mv arrow_*.tar.gz packages + - name: rchk + run: | + docker run -v `pwd`/packages:/rchk/packages kalibera/rchk:latest /rchk/packages/arrow_*.tar.gz |& tee rchk.out + - name: Confirm that rchk has no errors + # Suspicious call, [UP], and [PB] are all of the error types currently at + # https://github.com/kalibera/cran-checks/tree/master/rchk/results + # though this might not be exhaustive, there does not appear to be a way to have rchk return an error code + # CRAN also will remove some of the outputs (especially those related to Rcpp and strptime, e.g. + # ERROR: too many states (abstraction error?)) + # https://github.com/kalibera/rchk + run: | + if [ $(grep -c "Suspicious call" rchk.out) -gt 0 ] || [ $(grep -c "\[UP\]" rchk.out) -gt 0 ] || [ $(grep -c "\[PB\]" rchk.out) -gt 0 ]; then + echo "Found rchk errors" + cat rchk.out + exit 1 + fi + if: always() + - name: Dump rchk output logs + run: cat rchk.out + if: always() diff --git a/src/arrow/dev/tasks/r/github.linux.revdepcheck.yml b/src/arrow/dev/tasks/r/github.linux.revdepcheck.yml new file mode 100644 index 000000000..80071171b --- /dev/null +++ b/src/arrow/dev/tasks/r/github.linux.revdepcheck.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + r-versions: + name: "rstudio/r-base:latest-focal" + runs-on: ubuntu-latest + strategy: + fail-fast: false + env: + R_ORG: "rstudio" + R_IMAGE: "r-base" + R_TAG: "latest-focal" + ARROW_R_DEV: "TRUE" + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + - name: Fetch Submodules and Tags + shell: bash + run: cd arrow && ci/scripts/util_checkout.sh + - name: Docker Pull + shell: bash + run: cd arrow && docker-compose pull --ignore-pull-failures r + - name: Docker Build + shell: bash + run: cd arrow && docker-compose build r-revdepcheck + - name: Docker Run + shell: bash + run: cd arrow && docker-compose run r-revdepcheck + - name: revdepcheck CRAN report + if: always() + shell: bash + run: cat arrow/r/revdep/cran.md + - name: revdepcheck failures + if: always() + shell: bash + run: cat arrow/r/revdep/failures.md + - name: revdepcheck problems + if: always() + shell: bash + run: cat arrow/r/revdep/problems.md + - name: Save the revdep output + if: always() + uses: actions/upload-artifact@v2 + with: + name: revdepcheck-folder + path: arrow/r/revdep diff --git a/src/arrow/dev/tasks/r/github.linux.versions.yml b/src/arrow/dev/tasks/r/github.linux.versions.yml new file mode 100644 index 000000000..f383fe8d0 --- /dev/null +++ b/src/arrow/dev/tasks/r/github.linux.versions.yml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + r-versions: + name: "rstudio/r-base:{{ MATRIX }}-bionic" + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # See https://hub.docker.com/r/rstudio/r-base + r_version: + # We test devel, release, and oldrel in regular CI. + # This is for older versions + # rlang and vctrs depend on R >= 3.3 + - "3.3" + - "3.4" + - "3.5" + - "3.6" + env: + R_ORG: "rstudio" + R_IMAGE: "r-base" + R_TAG: "{{ MATRIX }}-bionic" + ARROW_R_DEV: "TRUE" + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Free Up Disk Space + shell: bash + run: arrow/ci/scripts/util_cleanup.sh + - name: Fetch Submodules and Tags + shell: bash + run: cd arrow && ci/scripts/util_checkout.sh + - name: Docker Pull + shell: bash + run: cd arrow && docker-compose pull --ignore-pull-failures r + - name: Docker Build + shell: bash + run: cd arrow && docker-compose build r + - name: Docker Run + shell: bash + run: cd arrow && docker-compose run r + - name: Dump install logs + run: cat arrow/r/check/arrow.Rcheck/00install.out + if: always() + - name: Dump test logs + run: cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* + if: always() + - name: Save the test output + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-output + path: arrow/r/check/arrow.Rcheck/tests/testthat.Rout* diff --git a/src/arrow/dev/tasks/r/github.macos-linux.local.yml b/src/arrow/dev/tasks/r/github.macos-linux.local.yml new file mode 100644 index 000000000..79e3332af --- /dev/null +++ b/src/arrow/dev/tasks/r/github.macos-linux.local.yml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + autobrew: + name: "install from local source" + runs-on: {{ "${{ matrix.os }}" }} + strategy: + fail-fast: false + matrix: + os: [macOS-latest, ubuntu-20.04] + + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Configure non-autobrew dependencies (macos) + run: | + cd arrow/r + brew install openssl + if: contains(matrix.os, 'macOS') + - name: Configure non-autobrew dependencies (linux) + run: | + cd arrow/r + sudo apt-get update + sudo apt install libcurl4-openssl-dev libssl-dev + if: contains(matrix.os, 'ubuntu') + - uses: r-lib/actions/setup-r@v1 + - name: Install dependencies + run: | + install.packages("remotes") + remotes::install_deps("arrow/r", dependencies = TRUE) + remotes::install_cran(c("rcmdcheck", "sys", "sessioninfo")) + shell: Rscript {0} + - name: Session info + run: | + options(width = 100) + pkgs <- installed.packages()[, "Package"] + sessioninfo::session_info(pkgs, include_base = TRUE) + shell: Rscript {0} + - name: Install + env: + _R_CHECK_CRAN_INCOMING_: false + ARROW_USE_PKG_CONFIG: false + FORCE_BUNDLED_BUILD: true + LIBARROW_MINIMAL: false + ARROW_R_DEV: TRUE + run: | + cd arrow/r + R CMD INSTALL . --install-tests + - name: Run the tests + run: R -e 'if(tools::testInstalledPackage("arrow") != 0L) stop("There was a test failure.")' + - name: Dump test logs + run: cat arrow-tests/testthat.Rout* + if: failure() + - name: Save the test output + uses: actions/upload-artifact@v2 + with: + name: test-output + path: arrow-tests/testthat.Rout* + if: always() diff --git a/src/arrow/dev/tasks/r/github.macos.autobrew.yml b/src/arrow/dev/tasks/r/github.macos.autobrew.yml new file mode 100644 index 000000000..1b8500f64 --- /dev/null +++ b/src/arrow/dev/tasks/r/github.macos.autobrew.yml @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + autobrew: + name: "Autobrew" + runs-on: macOS-latest + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Configure autobrew script + run: | + cd arrow/r + # Put the formula inside r/ so that it's included in the package build + cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb tools/apache-arrow.rb + # Pin the current commit in the formula to test so that we're not always pulling from master + sed -i.bak -E -e 's@https://github.com/apache/arrow.git"$@{{ arrow.remote }}.git", :revision => "{{ arrow.head }}"@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak + # Sometimes crossbow gives a remote URL with .git and sometimes not. Make sure there's only one + sed -i.bak -E -e 's@.git.git@.git@' tools/apache-arrow.rb && rm -f tools/apache-arrow.rb.bak + # Get minio for S3 testing + brew install minio + - uses: r-lib/actions/setup-r@v1 + - name: Install dependencies + run: | + install.packages("remotes") + remotes::install_deps("arrow/r", dependencies = TRUE) + remotes::install_cran(c("rcmdcheck", "sys", "sessioninfo")) + shell: Rscript {0} + - name: Session info + run: | + options(width = 100) + pkgs <- installed.packages()[, "Package"] + sessioninfo::session_info(pkgs, include_base = TRUE) + shell: Rscript {0} + - name: Check + env: + _R_CHECK_CRAN_INCOMING_: false + ARROW_USE_PKG_CONFIG: false + run: arrow/ci/scripts/r_test.sh arrow + - name: Dump install logs + run: cat arrow/r/check/arrow.Rcheck/00install.out + if: always() + - name: Dump test logs + run: cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout* + if: always() + - name: Save the test output + if: always() + uses: actions/upload-artifact@v2 + with: + name: test-output + path: arrow/r/check/arrow.Rcheck/tests/testthat.Rout* diff --git a/src/arrow/dev/tasks/tasks.yml b/src/arrow/dev/tasks/tasks.yml new file mode 100644 index 000000000..8e67d3116 --- /dev/null +++ b/src/arrow/dev/tasks/tasks.yml @@ -0,0 +1,1308 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +groups: + # these groups are just for convenience + # makes it easier to submit related tasks + +{############################# Packaging tasks ###############################} + + conda: + - conda-* + + wheel: + - wheel-* + + linux: + - almalinux-* + - amazon-linux-* + - centos-* + - debian-* + - ubuntu-* + + linux-amd64: + - almalinux-*-amd64 + - amazon-linux-*-amd64 + - centos-*-amd64 + - debian-*-amd64 + - ubuntu-*-amd64 + + linux-arm64: + - almalinux-*-arm64 + - centos-*-arm64 + - debian-*-arm64 + - ubuntu-*-arm64 + + homebrew: + - homebrew-* + + packaging: + - almalinux-* + - amazon-linux-* + - centos-* + - debian-* + - java-jars + - nuget + - python-sdist + - ubuntu-* + - wheel-* + +{############################# Testing tasks #################################} + + test: + - test-* + + cpp: + - test-*cpp* + + c-glib: + - test-*c-glib* + + python: + - test-*python* + + r: + - test*-r-* + - homebrew-r-autobrew + # r-conda tasks + - conda-linux-gcc-py*-cpu-r* + - conda-osx-clang-py*-r* + - conda-win-vs2017-py*-r* + + ruby: + - test-*ruby* + + vcpkg: + - test-*vcpkg* + + integration: + - test-*dask* + - test-*hdfs* + - test-*kartothek* + - test-*pandas* + - test-*spark* + # TEMP disable because those are failing due to needing upstream fix (ARROW-13594) + # - test-*turbodbc* + + example: + - example-* + + example-cpp: + - example-*cpp* + + verify-rc: + - verify-rc-* + + verify-rc-binaries: + - verify-rc-binaries-* + + verify-rc-wheels: + - verify-rc-wheels-* + + verify-rc-source: + - verify-rc-source-* + + verify-rc-source-macos: + - verify-rc-source-*-macos-* + + verify-rc-source-linux: + - verify-rc-source-*-linux-* + +{######################## Tasks to run regularly #############################} + + nightly: + - almalinux-* + - amazon-linux-* + - debian-* + - ubuntu-* + - centos-* + - conda-* + - java-jars + # List the homebrews explicitly because we don't care about running homebrew-cpp-autobrew + - homebrew-cpp + - homebrew-r-autobrew + - nuget + - test-* + - example-* + - wheel-* + - python-sdist + +tasks: + # arbitrary_task_name: + # template: path of jinja2 templated yml + # params: optional extra parameters + # artifacts: list of regex patterns, each needs to match a single github + # release asset, version variable is replaced in the pattern + # e.g.: + # - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0-linux-64.tar.bz2 + +{############################## Conda Linux ##################################} + + conda-clean: + ci: azure + template: conda-recipes/azure.clean.yml + + # Important notes on the conda setup here: + # + # * On conda-forge the `pyarrow` and `arrow-cpp` packages are built in + # the same feedstock as the dependency matrix is the same for them as + # Python and the OS are the main dimension. The R package `r-arrow` is + # an independent feedstock as it doesn't have the Python but the + # R dimension. To limit the number of CI jobs, we are building `r-arrow` + # for R 4.0 with the Python 3.6 jobs and for R 4.1 with the Python 3.7 jobs. + # * The files in `dev/tasks/conda-recipes/.ci_support/` are automatically + # generated and to be synced regularly from the feedstock. We have no way + # yet to generate them inside the arrow repository automatically. + + conda-linux-gcc-py36-cpu-r40: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython + r_config: linux_64_r_base4.0 + artifacts: + - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-linux-gcc-py37-cpu-r41: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython + r_config: linux_64_r_base4.1 + artifacts: + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-linux-gcc-py38-cpu: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-linux-gcc-py39-cpu: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + +{% for python_version, numpy_version in [("3.6", "1.17"), + ("3.7", "1.17"), + ("3.8", "1.17"), + ("3.9", "1.19"), + ("3.10", "1.21")] %} + {% set pyver = python_version | replace(".", "") %} + + conda-linux-gcc-py{{ pyver }}-cuda: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_64_cuda_compiler_version10.2numpy{{ numpy_version }}python{{ python_version }}.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cuda.tar.bz2 + - pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cuda.tar.bz2 + + conda-linux-gcc-py{{ pyver }}-arm64: + ci: azure + template: conda-recipes/azure.linux.yml + params: + config: linux_aarch64_numpy{{ numpy_version }}python{{ python_version }}.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2 + +{% endfor %} + + ############################## Conda OSX #################################### + + conda-osx-clang-py36-r40: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_64_numpy1.17python3.6.____cpython + r_config: osx_64_r_base4.0 + artifacts: + - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-osx-clang-py37-r41: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_64_numpy1.17python3.7.____cpython + r_config: osx_64_r_base4.1 + artifacts: + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-osx-clang-py38: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_64_numpy1.17python3.8.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-osx-clang-py39: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_64_numpy1.19python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-osx-arm64-clang-py38: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_arm64_python3.8.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-osx-arm64-clang-py39: + ci: azure + template: conda-recipes/azure.osx.yml + params: + config: osx_arm64_python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2 + + ############################## Conda Windows ################################ + + conda-win-vs2017-py36-r40: + ci: azure + template: conda-recipes/azure.win.yml + params: + config: win_64_cuda_compiler_versionNonenumpy1.17python3.6.____cpython + r_config: win_64_r_base4.0 + artifacts: + - arrow-cpp-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py36(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-win-vs2017-py37-r41: + ci: azure + template: conda-recipes/azure.win.yml + params: + config: win_64_cuda_compiler_versionNonenumpy1.17python3.7.____cpython + r_config: win_64_r_base4.1 + artifacts: + - arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-win-vs2017-py38: + ci: azure + template: conda-recipes/azure.win.yml + params: + config: win_64_cuda_compiler_versionNonenumpy1.17python3.8.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + + conda-win-vs2017-py39: + ci: azure + template: conda-recipes/azure.win.yml + params: + config: win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython + artifacts: + - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2 + + +{% for python_version, python_tag, abi_tag in [("3.6", "cp36", "cp36m"), + ("3.7", "cp37", "cp37m"), + ("3.8", "cp38", "cp38"), + ("3.9", "cp39", "cp39"), + ("3.10", "cp310", "cp310")] %} + +{############################## Wheel Linux ##################################} + +{% for ci, arch, arch_alias, x_y, manylinux in [("github", "amd64", "x86_64", "2_12", "2010"), + ("github", "amd64", "x86_64", "2_17", "2014"), + ("travis", "arm64", "aarch64", "2_17", "2014")] %} + wheel-manylinux{{ manylinux }}-{{ python_tag }}-{{ arch }}: + ci: {{ ci }} + template: python-wheels/{{ ci }}.linux.{{ arch }}.yml + params: + python_version: "{{ python_version }}" + manylinux_version: {{ manylinux }} + artifacts: + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-manylinux_{{ x_y }}_{{ arch_alias }}.manylinux{{ manylinux }}_{{ arch_alias }}.whl +{% endfor %} + +{############################## Wheel OSX ####################################} + +# enable S3 support from macOS 10.13 so we don't need to bundle curl, crypt and ssl +{% for macos_version, macos_codename, arrow_s3 in [("10.9", "mavericks", "OFF"), + ("10.13", "high-sierra", "ON")] %} + {% set platform_tag = "macosx_{}_x86_64".format(macos_version.replace('.', '_')) %} + + wheel-macos-{{ macos_codename }}-{{ python_tag }}-amd64: + ci: github + template: python-wheels/github.osx.amd64.yml + params: + vcpkg_version: "2021.04.30" + python_version: "{{ python_version }}" + macos_deployment_target: {{ macos_version }} + arrow_s3: {{ arrow_s3 }} + artifacts: + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl + +{% endfor %} + +{############################## Wheel Windows ################################} + + wheel-windows-{{ python_tag }}-amd64: + ci: github + template: python-wheels/github.windows.yml + params: + python_version: "{{ python_version }}" + artifacts: + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl + +{% endfor %} + +{############################## Wheel OSX M1 #################################} + + # The python 3.8 universal2 installer has been built with macos deployment + # target 11.0, so we cannot build binaries with earlier deployment target + # otherwise distutils will raise a deployment target version mismatch error. + wheel-macos-big-sur-cp38-arm64: + ci: github + template: python-wheels/github.osx.arm64.yml + params: + arch: arm64 + arrow_simd_level: "DEFAULT" + vcpkg_version: "2021.04.30" + python_version: "3.8" + macos_deployment_target: "11.0" + + artifacts: + - pyarrow-{no_rc_version}-cp38-cp38-macosx_11_0_arm64.whl + +{% for python_version, python_tag in [("3.9", "cp39"), ("3.10", "cp310")] %} + wheel-macos-big-sur-{{ python_tag }}-arm64: + ci: github + template: python-wheels/github.osx.arm64.yml + params: + arch: arm64 + arrow_simd_level: "DEFAULT" + vcpkg_version: "2021.04.30" + python_version: "{{ python_version }}" + macos_deployment_target: "11.0" + artifacts: + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_11_0_arm64.whl + + wheel-macos-big-sur-{{ python_tag }}-universal2: + ci: github + template: python-wheels/github.osx.arm64.yml + params: + arch: universal2 + # Universal2 builds for both rosetta and native, but we currently can't + # configure SIMD for both architectures at the same time + arrow_simd_level: "NONE" + vcpkg_version: "2021.04.30" + python_version: "{{ python_version }}" + macos_deployment_target: "10.13" + artifacts: + - pyarrow-{no_rc_version}-{{ python_tag }}-{{ python_tag }}-macosx_10_13_universal2.whl +{% endfor %} + +{############################ Python sdist ####################################} + + python-sdist: + ci: github + template: python-sdist/github.yml + artifacts: + - pyarrow-{no_rc_version}.tar.gz + +{############################## Linux PKGS ####################################} + +{% for target in ["debian-buster", + "debian-bullseye", + "debian-bookworm", + "ubuntu-bionic", + "ubuntu-focal", + "ubuntu-hirsute", + "ubuntu-impish"] %} + {% for architecture in ["amd64", "arm64"] %} + {{ target }}-{{ architecture }}: + {% if architecture == "amd64" %} + ci: github + template: linux-packages/github.linux.amd64.yml + {% else %} + ci: travis + template: linux-packages/travis.linux.arm64.yml + {% endif %} + params: + {% if architecture == "amd64" %} + target: "{{ target }}" + {% else %} + target: "{{ target }}-arm64" + {% endif %} + task_namespace: "apt" + upload_extensions: + - .ddeb + - .deb + - .debian.tar.xz + - .dsc + - .orig.tar.gz + artifacts: + {% if architecture == "amd64" %} + - apache-arrow-apt-source_{no_rc_version}-1.debian.tar.xz + - apache-arrow-apt-source_{no_rc_version}-1.dsc + - apache-arrow-apt-source_{no_rc_version}-1_all.deb + - apache-arrow-apt-source_{no_rc_version}.orig.tar.gz + - apache-arrow_{no_rc_version}-1.debian.tar.xz + - apache-arrow_{no_rc_version}-1.dsc + - apache-arrow_{no_rc_version}.orig.tar.gz + {% endif %} + - gir1.2-arrow-1.0_{no_rc_version}-1_[a-z0-9]+.deb + - gir1.2-arrow-dataset-1.0_{no_rc_version}-1_[a-z0-9]+.deb + - gir1.2-arrow-flight-1.0_{no_rc_version}-1_[a-z0-9]+.deb + - gir1.2-gandiva-1.0_{no_rc_version}-1_[a-z0-9]+.deb + - gir1.2-parquet-1.0_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-dataset-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-dataset-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-dataset-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-dataset-glib600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-dataset-glib600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-dataset600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-dataset600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-flight-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-flight-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-flight-glib600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-flight-glib600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-flight600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-flight600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-glib600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-glib600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-python-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-python-flight-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-python-flight600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-python-flight600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-python600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-python600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow600_{no_rc_version}-1_[a-z0-9]+.deb + - libgandiva-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libgandiva-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libgandiva-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb + - libgandiva-glib600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libgandiva-glib600_{no_rc_version}-1_[a-z0-9]+.deb + - libgandiva600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libgandiva600_{no_rc_version}-1_[a-z0-9]+.deb + - libparquet-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libparquet-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libparquet-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb + - libparquet-glib600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libparquet-glib600_{no_rc_version}-1_[a-z0-9]+.deb + - libparquet600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libparquet600_{no_rc_version}-1_[a-z0-9]+.deb + {% if architecture == "amd64" %} + - gir1.2-arrow-cuda-1.0_{no_rc_version}-1_[a-z0-9]+.deb + - gir1.2-plasma-1.0_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-cuda-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-cuda-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-cuda-glib600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-cuda-glib600_{no_rc_version}-1_[a-z0-9]+.deb + - libarrow-cuda600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libarrow-cuda600_{no_rc_version}-1_[a-z0-9]+.deb + - libplasma-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libplasma-glib-dev_{no_rc_version}-1_[a-z0-9]+.deb + - libplasma-glib-doc_{no_rc_version}-1_[a-z0-9]+.deb + - libplasma-glib600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libplasma-glib600_{no_rc_version}-1_[a-z0-9]+.deb + - libplasma600-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - libplasma600_{no_rc_version}-1_[a-z0-9]+.deb + - plasma-store-server-dbgsym_{no_rc_version}-1_[a-z0-9]+.d?deb + - plasma-store-server_{no_rc_version}-1_[a-z0-9]+.deb + {% endif %} + {% endfor %} +{% endfor %} + +{% for target in ["almalinux-8", + "amazon-linux-2", + "centos-7", + "centos-8"] %} + {% set is_rhel8_based = (target == "almalinux-8" or target == "centos-8") %} + {% for architecture in ["amd64", "arm64"] %} + {% if not (target in ["amazon-linux-2", "centos-7"] and architecture == "arm64") %} + {{ target }}-{{ architecture }}: + {% if architecture == "amd64" %} + ci: github + template: linux-packages/github.linux.amd64.yml + {% else %} + ci: travis + template: linux-packages/travis.linux.arm64.yml + {% endif %} + params: + {% if architecture == "amd64" %} + target: "{{ target }}" + {% else %} + target: "{{ target }}-aarch64" + {% endif %} + task_namespace: "yum" + upload_extensions: + - .rpm + artifacts: + {% if architecture == "amd64" %} + - apache-arrow-release-{no_rc_version}-1.[a-z0-9]+.noarch.rpm + - apache-arrow-release-{no_rc_version}-1.[a-z0-9]+.src.rpm + {% endif %} + - arrow-dataset-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-dataset-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-dataset-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - arrow-dataset-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - arrow-dataset-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - arrow-dataset-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - arrow-dataset-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - arrow-debugsource-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - arrow-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - arrow-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-flight-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-flight-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-flight-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-flight-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-flight-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-flight-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - arrow-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - arrow-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - arrow-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - arrow-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - arrow-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if target != "amazon-linux-2" %} + - arrow-python-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - arrow-python-flight-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-python-flight-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-python-flight-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - arrow-python-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - arrow-python-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + {% if architecture == "amd64" %} + - arrow-{no_rc_version}-1.[a-z0-9]+.src.rpm + {% endif %} + {% if is_rhel8_based and architecture == "amd64" %} + - gandiva-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - gandiva-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - gandiva-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - gandiva-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - gandiva-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - gandiva-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - gandiva-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - parquet-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - parquet-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - parquet-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - parquet-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - parquet-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - parquet-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - parquet-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - plasma-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - plasma-glib-devel-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + - plasma-glib-doc-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - plasma-glib-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - plasma-glib-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - plasma-libs-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - plasma-libs-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% if is_rhel8_based %} + - plasma-store-server-debuginfo-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + - plasma-store-server-{no_rc_version}-1.[a-z0-9]+.[a-z0-9_]+.rpm + {% endif %} + {% endfor %} +{% endfor %} + + ############################## Homebrew Tasks ################################ + + homebrew-cpp: + ci: github + template: homebrew-formulae/github.macos.yml + params: + formula: apache-arrow.rb + + homebrew-cpp-autobrew: + ci: github + template: homebrew-formulae/github.macos.yml + params: + formula: autobrew/apache-arrow.rb + + homebrew-r-autobrew: + # This tests that the autobrew formula + script work in practice + ci: github + template: r/github.macos.autobrew.yml + + ############################## Arrow JAR's ################################## + + java-jars: + # Build jar's that contains cpp libraries dependencies + ci: github + template: java-jars/github.yml + artifacts: + - arrow-algorithm-{no_rc_version}-tests.jar + - arrow-algorithm-{no_rc_version}.jar + - arrow-algorithm-{no_rc_version}.pom + - arrow-avro-{no_rc_version}-tests.jar + - arrow-avro-{no_rc_version}.jar + - arrow-avro-{no_rc_version}.pom + - arrow-c-data-{no_rc_version}-tests.jar + - arrow-c-data-{no_rc_version}.jar + - arrow-c-data-{no_rc_version}.pom + - arrow-compression-{no_rc_version}-tests.jar + - arrow-compression-{no_rc_version}.jar + - arrow-compression-{no_rc_version}.pom + - arrow-dataset-{no_rc_version}-tests.jar + - arrow-dataset-{no_rc_version}.jar + - arrow-dataset-{no_rc_version}.pom + - arrow-format-{no_rc_version}-tests.jar + - arrow-format-{no_rc_version}.jar + - arrow-format-{no_rc_version}.pom + - arrow-gandiva-{no_rc_version}-tests.jar + - arrow-gandiva-{no_rc_version}.jar + - arrow-gandiva-{no_rc_version}.pom + - arrow-java-root-{no_rc_version}.pom + - arrow-jdbc-{no_rc_version}-tests.jar + - arrow-jdbc-{no_rc_version}.jar + - arrow-jdbc-{no_rc_version}.pom + - arrow-memory-{no_rc_version}.pom + - arrow-memory-core-{no_rc_version}-tests.jar + - arrow-memory-core-{no_rc_version}.jar + - arrow-memory-core-{no_rc_version}.pom + - arrow-memory-netty-{no_rc_version}-tests.jar + - arrow-memory-netty-{no_rc_version}.jar + - arrow-memory-netty-{no_rc_version}.pom + - arrow-memory-unsafe-{no_rc_version}-tests.jar + - arrow-memory-unsafe-{no_rc_version}.jar + - arrow-memory-unsafe-{no_rc_version}.pom + - arrow-orc-{no_rc_version}-tests.jar + - arrow-orc-{no_rc_version}.jar + - arrow-orc-{no_rc_version}.pom + - arrow-performance-{no_rc_version}-tests.jar + - arrow-performance-{no_rc_version}.jar + - arrow-performance-{no_rc_version}.pom + - arrow-plasma-{no_rc_version}-tests.jar + - arrow-plasma-{no_rc_version}.jar + - arrow-plasma-{no_rc_version}.pom + - arrow-tools-{no_rc_version}-jar-with-dependencies.jar + - arrow-tools-{no_rc_version}-tests.jar + - arrow-tools-{no_rc_version}.jar + - arrow-tools-{no_rc_version}.pom + - arrow-vector-{no_rc_version}-shade-format-flatbuffers.jar + - arrow-vector-{no_rc_version}-tests.jar + - arrow-vector-{no_rc_version}.jar + - arrow-vector-{no_rc_version}.pom + - flight-core-{no_rc_version}-jar-with-dependencies.jar + - flight-core-{no_rc_version}-shaded-ext.jar + - flight-core-{no_rc_version}-shaded.jar + - flight-core-{no_rc_version}-tests.jar + - flight-core-{no_rc_version}.jar + - flight-core-{no_rc_version}.pom + - flight-grpc-{no_rc_version}-tests.jar + - flight-grpc-{no_rc_version}.jar + - flight-grpc-{no_rc_version}.pom + + ############################## NuGet packages ############################### + + nuget: + ci: github + template: nuget-packages/github.linux.yml + params: + run: ubuntu-csharp + artifacts: + - Apache.Arrow.Flight.AspNetCore.{no_rc_version}.nupkg + - Apache.Arrow.Flight.AspNetCore.{no_rc_version}.snupkg + - Apache.Arrow.Flight.{no_rc_version}.nupkg + - Apache.Arrow.Flight.{no_rc_version}.snupkg + - Apache.Arrow.{no_rc_version}.nupkg + - Apache.Arrow.{no_rc_version}.snupkg + + ########################### Release verification ############################ + +{% for target in ["binary", "yum", "apt"] %} + verify-rc-binaries-{{ target }}-amd64: + ci: github + template: verify-rc/github.linux.amd64.yml + params: + env: + TEST_DEFAULT: 0 + TEST_{{ target|upper }}: 1 + artifact: "binaries" +{% endfor %} + +{% for platform, arch, runner in [("linux", "amd64", "ubuntu-20.04"), + ("macos", "amd64", "macos-10.15")] %} + {% for target in ["cpp", + "csharp", + "go", + "integration", + "java", + "js", + "python", + "ruby"] %} + + verify-rc-source-{{ target }}-{{ platform }}-{{ arch }}: + ci: github + template: verify-rc/github.{{ platform }}.{{ arch }}.yml + params: + env: + INSTALL_NODE: 0 + TEST_DEFAULT: 0 + TEST_{{ target|upper }}: 1 + artifact: "source" + github_runner: {{ runner }} + {% endfor %} +{% endfor %} + +{% for platform, arch, runner in [("macos", "arm64", "self-hosted")] %} + {% for target in ["cpp", + "csharp", + "go", + "integration", + "js", + "python", + "ruby"] %} + + verify-rc-source-{{ target }}-{{ platform }}-{{ arch }}: + ci: github + template: verify-rc/github.{{ platform }}.{{ arch }}.yml + params: + env: + ARROW_FLIGHT: 0 + ARROW_GANDIVA: 0 + INSTALL_NODE: 0 + TEST_DEFAULT: 0 + TEST_INTEGRATION_JAVA: 0 + TEST_{{ target|upper }}: 1 + artifact: "source" + github_runner: {{ runner }} + {% endfor %} +{% endfor %} + + verify-rc-wheels-linux-amd64: + ci: github + template: verify-rc/github.linux.amd64.yml + params: + env: + TEST_DEFAULT: 0 + artifact: "wheels" + + verify-rc-wheels-macos-10.15-amd64: + ci: github + template: verify-rc/github.macos.amd64.yml + params: + github_runner: "macos-10.15" + env: + TEST_DEFAULT: 0 + artifact: "wheels" + + # The github hosted macos-11 runners are in preview only, but should be switched once they are generally available: + # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources + verify-rc-wheels-macos-11-amd64: + ci: github + template: verify-rc/github.macos.arm64.yml + params: + github_runner: "self-hosted" + arch_emulation: "x86_64" + env: + TEST_DEFAULT: 0 + artifact: "wheels" + + verify-rc-wheels-macos-11-arm64: + ci: github + template: verify-rc/github.macos.arm64.yml + params: + github_runner: "self-hosted" + arch_emulation: "arm64" + env: + TEST_DEFAULT: 0 + artifact: "wheels" + + verify-rc-source-windows: + ci: github + template: verify-rc/github.win.yml + params: + script: "verify-release-candidate.bat" + + verify-rc-wheels-windows: + ci: github + template: verify-rc/github.win.yml + params: + script: "verify-release-candidate-wheels.bat" + +{############################## Docker tests #################################} + +{% for image in ["conda-cpp", + "debian-c-glib", + "ubuntu-c-glib", + "debian-ruby", + "ubuntu-ruby"] %} + test-{{ image }}: + ci: github + template: docker-tests/github.linux.yml + params: + image: {{ image }} +{% endfor %} + + # Use azure to run valgrind tests to prevent OOM + test-conda-cpp-valgrind: + ci: azure + template: docker-tests/azure.linux.yml + params: + run: conda-cpp-valgrind + +{% for ubuntu_version in ["18.04", "20.04"] %} + test-ubuntu-{{ ubuntu_version }}-cpp: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: {{ ubuntu_version }} + image: ubuntu-cpp +{% endfor %} + + test-ubuntu-20.04-cpp-bundled: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 20.04 + image: ubuntu-cpp-bundled + + test-debian-11-cpp: + ci: github + template: docker-tests/github.linux.yml + params: + env: + DEBIAN: 11 + image: debian-cpp + + test-fedora-33-cpp: + ci: github + template: docker-tests/github.linux.yml + params: + env: + FEDORA: 33 + image: fedora-cpp + + test-ubuntu-18.04-cpp-release: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 18.04 + flags: "-e ARROW_BUILD_TYPE=release" + image: ubuntu-cpp + + test-ubuntu-18.04-cpp-static: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 18.04 + flags: "-e ARROW_BUILD_SHARED=OFF -e ARROW_BUILD_STATIC=ON -e ARROW_TEST_LINKAGE=static" + image: ubuntu-cpp + +{% for cpp_standard in [14, 17] %} + test-ubuntu-20.04-cpp-{{ cpp_standard }}: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 20.04 + flags: "-e CMAKE_ARGS=-DCMAKE_CXX_STANDARD={{ cpp_standard }}" + image: ubuntu-cpp +{% endfor %} + + test-ubuntu-20.04-cpp-thread-sanitizer: + ci: github + template: docker-tests/github.linux.yml + params: + env: + # clang-tools and llvm version need to be synchronized so as + # to have the right llvm-symbolizer version + CLANG_TOOLS: 11 + LLVM: 11 + UBUNTU: 20.04 + image: ubuntu-cpp-thread-sanitizer + +{% for python_version in ["3.6", "3.7", "3.8", "3.9", "3.10"] %} + test-conda-python-{{ python_version }}: + ci: github + template: docker-tests/github.linux.yml + params: + env: + PYTHON: "{{ python_version }}" + image: conda-python +{% endfor %} + + test-conda-python-3.8-hypothesis: + ci: github + template: docker-tests/github.linux.yml + params: + env: + HYPOTHESIS_PROFILE: ci + PYARROW_TEST_HYPOTHESIS: ON + PYTHON: 3.8 + # limit to execute hypothesis tests only + PYTEST_ARGS: "-m hypothesis" + image: conda-python-pandas + + test-debian-11-python-3: + ci: azure + template: docker-tests/azure.linux.yml + params: + env: + DEBIAN: 11 + run: debian-python + + test-ubuntu-18.04-python-3: + ci: azure + template: docker-tests/azure.linux.yml + params: + env: + UBUNTU: 18.04 + run: ubuntu-python + + test-fedora-33-python-3: + ci: azure + template: docker-tests/azure.linux.yml + params: + env: + FEDORA: 33 + run: fedora-python + + test-r-linux-valgrind: + ci: azure + template: docker-tests/azure.linux.yml + params: + env: + ARROW_R_DEV: "TRUE" + UBUNTU: 18.04 + run: ubuntu-r-valgrind + + test-r-linux-rchk: + ci: github + template: r/github.linux.rchk.yml + + test-r-linux-as-cran: + ci: github + template: r/github.linux.cran.yml + params: + MATRIX: {{ "${{ matrix.r_image }}" }} + + test-r-arrow-backwards-compatibility: + ci: github + template: r/github.linux.arrow.version.back.compat.yml + + test-r-versions: + ci: github + template: r/github.linux.versions.yml + params: + MATRIX: {{ "${{ matrix.r_version }}" }} + + test-r-install-local: + ci: github + template: r/github.macos-linux.local.yml + + test-r-devdocs: + ci: github + template: r/github.devdocs.yml + + test-r-depsource-auto: + ci: azure + template: r/azure.linux.yml + params: + r_org: rocker + r_image: r-base + r_tag: latest + flags: '-e ARROW_DEPENDENCY_SOURCE=AUTO' + + test-r-depsource-system: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 21.04 + CLANG_TOOLS: 9 # can remove this when >=9 is the default + flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE -e ARROW_DEPENDENCY_SOURCE=SYSTEM' + image: ubuntu-r-only-r + + test-r-offline-minimal: + ci: azure + template: r/azure.linux.yml + params: + r_org: rocker + r_image: r-base + r_tag: latest + flags: '-e TEST_OFFLINE_BUILD=true' + + test-r-offline-maximal: + ci: github + template: r/github.linux.offline.build.yml + + +{% for r_org, r_image, r_tag in [("rhub", "ubuntu-gcc-release", "latest"), + ("rocker", "r-base", "latest"), + ("rstudio", "r-base", "3.6-bionic"), + ("rstudio", "r-base", "3.6-centos8"), + ("rstudio", "r-base", "3.6-opensuse15"), + ("rstudio", "r-base", "3.6-opensuse42")] %} + test-r-{{ r_org }}-{{ r_image }}-{{ r_tag }}: + ci: azure + template: r/azure.linux.yml + params: + r_org: {{ r_org }} + r_image: {{ r_image }} + r_tag: {{ r_tag }} +{% endfor %} + + # This is with R built with --enable-lto + # CRAN also does R CMD INSTALL --use-LTO + # which overrides the UseLTO field in r/DESCRIPTION + test-r-rhub-debian-gcc-devel-lto-latest: + ci: azure + template: r/azure.linux.yml + params: + r_org: rhub + r_image: debian-gcc-devel-lto + r_tag: latest + flags: '-e NOT_CRAN=false -e INSTALL_ARGS=--use-LTO' + + # This one has -flto=auto + test-r-ubuntu-21.04: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 21.04 + CLANG_TOOLS: 9 # can remove this when >=9 is the default + flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE' + image: ubuntu-r-only-r + + # This also has -flto=auto + test-r-gcc-11: + ci: github + template: docker-tests/github.linux.yml + params: + env: + UBUNTU: 21.04 + CLANG_TOOLS: 9 # can remove this when >=9 is the default + GCC_VERSION: 11 + # S3 support is not buildable with gcc11 right now + flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE -e ARROW_S3=OFF' + image: ubuntu-r-only-r + + test-r-rstudio-r-base-3.6-centos7-devtoolset-8: + ci: azure + template: r/azure.linux.yml + params: + r_org: rstudio + r_image: r-base + r_tag: 3.6-centos7 + devtoolset_version: 8 + + test-r-minimal-build: + ci: azure + template: r/azure.linux.yml + params: + r_org: rocker + r_image: r-base + r_tag: latest + flags: "-e LIBARROW_MINIMAL=TRUE" + + test-ubuntu-18.04-r-sanitizer: + ci: azure + template: docker-tests/azure.linux.yml + params: + env: + UBUNTU: 18.04 + run: ubuntu-r-sanitizer + + revdep-r-check: + ci: github + template: r/github.linux.revdepcheck.yml + + test-debian-11-go-1.15: + ci: azure + template: docker-tests/azure.linux.yml + params: + env: + DEBIAN: 11 + GO: 1.15 + run: debian-go + + test-ubuntu-20.10-docs: + ci: azure + template: docker-tests/azure.linux.yml + params: + env: + UBUNTU: "20.10" + run: ubuntu-docs + + test-ubuntu-default-docs: + ci: azure + template: docker-tests/azure.linux.yml + params: + run: ubuntu-docs + + ############################## vcpkg tests ################################## + + test-build-vcpkg-win: + ci: github + template: vcpkg-tests/github.windows.yml + + ############################## Integration tests ############################ + +{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.6", "0.23", "1.16", True), + ("3.7", "0.24", "1.19", True), + ("3.7", "latest", "latest", False), + ("3.8", "latest", "latest", False), + ("3.8", "nightly", "nightly", False), + ("3.9", "master", "nightly", False)] %} + test-conda-python-{{ python_version }}-pandas-{{ pandas_version }}: + ci: github + template: docker-tests/github.linux.yml + params: + env: + PYTHON: {{ python_version }} + PANDAS: {{ pandas_version }} + NUMPY: {{ numpy_version }} + {% if cache_leaf %} + # use the latest pandas release, so prevent reusing any cached layers + flags: --no-leaf-cache + {% endif %} + image: conda-python-pandas +{% endfor %} + +{% for dask_version in ["latest", "master"] %} + test-conda-python-3.9-dask-{{ dask_version }}: + ci: github + template: docker-tests/github.linux.yml + params: + env: + PYTHON: 3.9 + DASK: {{ dask_version }} + # use the latest dask release, so prevent reusing any cached layers + flags: --no-leaf-cache + image: conda-python-dask +{% endfor %} + +# TEMP disable because those are failing due to needing upstream fix (ARROW-13594) +# {% for turbodbc_version in ["latest", "master"] %} +# test-conda-python-3.7-turbodbc-{{ turbodbc_version }}: +# ci: github +# template: docker-tests/github.linux.yml +# params: +# env: +# PYTHON: 3.7 +# TURBODBC: {{ turbodbc_version }} +# # use the latest turbodbc release, so prevent reusing any cached layers +# flags: --no-leaf-cache +# image: conda-python-turbodbc +# {% endfor %} + +{% for kartothek_version in ["latest", "master"] %} + test-conda-python-3.7-kartothek-{{ kartothek_version }}: + ci: github + template: docker-tests/github.linux.yml + params: + env: + PYTHON: 3.7 + KARTOTHEK: {{ kartothek_version }} + flags: --no-leaf-cache + image: conda-python-kartothek +{% endfor %} + +{% for hdfs_version in ["2.9.2", "3.2.1"] %} + test-conda-python-3.7-hdfs-{{ hdfs_version }}: + ci: github + template: docker-tests/github.linux.yml + params: + env: + PYTHON: 3.7 + HDFS: {{ hdfs_version }} + image: conda-python-hdfs +{% endfor %} + +{% for python_version, spark_version, test_pyarrow_only in [("3.7", "v3.1.2", "false"), + ("3.8", "v3.2.0", "false"), + ("3.9", "master", "false")] %} + test-conda-python-{{ python_version }}-spark-{{ spark_version }}: + ci: github + template: docker-tests/github.linux.yml + params: + env: + PYTHON: {{ python_version }} + SPARK: {{ spark_version }} + TEST_PYARROW_ONLY: {{ test_pyarrow_only }} + # use the branch-3.0 of spark, so prevent reusing any layers + flags: --no-leaf-cache + image: conda-python-spark +{% endfor %} + + # Remove the "skipped-" prefix in ARROW-8475 + skipped-test-conda-cpp-hiveserver2: + ci: github + template: docker-tests/github.linux.yml + params: + image: conda-cpp-hiveserver2 + +{% for kind in ["static", "static-system-dependency"] %} + example-cpp-minimal-build-{{ kind }}: + ci: github + template: cpp-examples/github.linux.yml + params: + type: minimal_build + run: {{ kind }} +{% endfor %} diff --git a/src/arrow/dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat b/src/arrow/dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat new file mode 100644 index 000000000..6423720c2 --- /dev/null +++ b/src/arrow/dev/tasks/vcpkg-tests/cpp-build-vcpkg.bat @@ -0,0 +1,86 @@ +@rem Licensed to the Apache Software Foundation (ASF) under one +@rem or more contributor license agreements. See the NOTICE file +@rem distributed with this work for additional information +@rem regarding copyright ownership. The ASF licenses this file +@rem to you under the Apache License, Version 2.0 (the +@rem "License"); you may not use this file except in compliance +@rem with the License. You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, +@rem software distributed under the License is distributed on an +@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@rem KIND, either express or implied. See the License for the +@rem specific language governing permissions and limitations +@rem under the License. + +@rem Run VsDevCmd.bat to set Visual Studio environment variables for building +@rem on the command line. This is the path for Visual Studio Enterprise 2019 + +call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\Common7\Tools\VsDevCmd.bat" -arch=amd64 + + +@rem Install build dependencies with vcpkg + +@rem TODO(ianmcook): change --x-manifest-root to --manifest-root after it +@rem changes in vcpkg + +vcpkg install ^ + --triplet x64-windows ^ + --x-manifest-root cpp ^ + --feature-flags=versions ^ + --clean-after-build ^ + || exit /B 1 + + +@rem Set environment variables + +set ARROW_TEST_DATA=%cd%\testing\data +set PARQUET_TEST_DATA=%cd%\cpp\submodules\parquet-testing\data + + +@rem Build Arrow C++ library + +mkdir cpp\build +pushd cpp\build + +@rem TODO(ianmcook): test using --parallel %NUMBER_OF_PROCESSORS% with +@rem cmake --build instead of specifying -DARROW_CXXFLAGS="/MP" here +@rem (see https://gitlab.kitware.com/cmake/cmake/-/issues/20564) + +@rem TODO(ianmcook): Add -DARROW_BUILD_BENCHMARKS=ON after the issue described +@rem at https://github.com/google/benchmark/issues/1046 is resolved + +cmake -G "Visual Studio 16 2019" -A x64 ^ + -DARROW_BOOST_USE_SHARED=ON ^ + -DARROW_BUILD_SHARED=ON ^ + -DARROW_BUILD_STATIC=OFF ^ + -DARROW_BUILD_TESTS=ON ^ + -DARROW_CXXFLAGS="/MP" ^ + -DARROW_DATASET=ON ^ + -DARROW_DEPENDENCY_SOURCE=VCPKG ^ + -DARROW_FLIGHT=OFF ^ + -DARROW_MIMALLOC=ON ^ + -DARROW_PARQUET=ON ^ + -DARROW_PYTHON=OFF ^ + -DARROW_WITH_BROTLI=ON ^ + -DARROW_WITH_BZ2=ON ^ + -DARROW_WITH_LZ4=ON ^ + -DARROW_WITH_SNAPPY=ON ^ + -DARROW_WITH_ZLIB=ON ^ + -DARROW_WITH_ZSTD=ON ^ + -DCMAKE_BUILD_TYPE=release ^ + -DCMAKE_UNITY_BUILD=ON ^ + .. || exit /B 1 + +cmake --build . --target INSTALL --config Release || exit /B 1 + + +@rem Test Arrow C++ library + +ctest --output-on-failure ^ + --parallel %NUMBER_OF_PROCESSORS% ^ + --timeout 300 || exit /B 1 + +popd diff --git a/src/arrow/dev/tasks/vcpkg-tests/github.windows.yml b/src/arrow/dev/tasks/vcpkg-tests/github.windows.yml new file mode 100644 index 000000000..ad3e793a6 --- /dev/null +++ b/src/arrow/dev/tasks/vcpkg-tests/github.windows.yml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE: must set "Crossbow" as name to have the badge links working in the +# github comment reports! +name: Crossbow + +on: + push: + branches: + - "*-github-*" + +jobs: + test-vcpkg-win: + name: Install build deps with vcpkg and build Arrow C++ + runs-on: windows-2019 + steps: + - name: Checkout Arrow + run: | + git clone --no-checkout {{ arrow.remote }} arrow + git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} + git -C arrow checkout FETCH_HEAD + git -C arrow submodule update --init --recursive + - name: Remove and Reinstall vcpkg + # When running vcpkg in Github Actions on Windows, remove the + # preinstalled vcpkg and install the newest version from source. + # Versions of vcpkg rapidly stop working until updated, and + # the safest and most reliable way to update vcpkg is simply + # to remove and reinstall it. + shell: cmd + run: | + CALL vcpkg integrate remove 2>NUL + CALL C: + CALL cd \ + CALL rmdir /s /q vcpkg 2>NUL + CALL git clone https://github.com/microsoft/vcpkg.git vcpkg + CALL cd vcpkg + CALL bootstrap-vcpkg.bat -win64 -disableMetrics + CALL vcpkg integrate install + CALL setx PATH "%PATH%;C:\vcpkg" + - name: Install Dependencies with vcpkg and Build Arrow C++ + shell: cmd + run: | + CALL cd arrow + CALL dev\tasks\vcpkg-tests\cpp-build-vcpkg.bat diff --git a/src/arrow/dev/tasks/verify-rc/github.linux.amd64.yml b/src/arrow/dev/tasks/verify-rc/github.linux.amd64.yml new file mode 100644 index 000000000..8a4613a49 --- /dev/null +++ b/src/arrow/dev/tasks/verify-rc/github.linux.amd64.yml @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + verify: + name: "Verify release candidate Ubuntu {{ artifact }}" + runs-on: {{ github_runner|default("ubuntu-20.04") }} + {% if env is defined %} + env: + {% for key, value in env.items() %} + {{ key }}: {{ value }} + {% endfor %} + {% endif %} + + steps: + {{ macros.github_checkout_arrow()|indent }} + + - name: Install System Dependencies + run: | + # TODO: don't require removing newer llvms + sudo apt-get --purge remove -y llvm-9 clang-9 + sudo apt-get update -y + sudo apt-get install -y \ + autoconf-archive \ + binfmt-support \ + bison \ + curl \ + flex \ + gtk-doc-tools \ + jq \ + libboost-all-dev \ + libgirepository1.0-dev \ + ninja-build \ + qemu-user-static \ + wget + + if [ "$TEST_JAVA" = "1" ]; then + # Maven + MAVEN_VERSION=3.6.3 + wget https://downloads.apache.org/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.zip + unzip apache-maven-$MAVEN_VERSION-bin.zip + mkdir -p $HOME/java + mv apache-maven-$MAVEN_VERSION $HOME/java + export PATH=$HOME/java/apache-maven-$MAVEN_VERSION/bin:$PATH + fi + + if [ "$TEST_RUBY" = "1" ]; then + ruby --version + sudo gem install bundler + fi + - uses: actions/setup-node@v2-beta + with: + node-version: '14' + - name: Run verification + shell: bash + run: | + arrow/dev/release/verify-release-candidate.sh \ + {{ artifact }} \ + {{ release|default("1.0.0") }} {{ rc|default("0") }} diff --git a/src/arrow/dev/tasks/verify-rc/github.macos.amd64.yml b/src/arrow/dev/tasks/verify-rc/github.macos.amd64.yml new file mode 100644 index 000000000..d39cda382 --- /dev/null +++ b/src/arrow/dev/tasks/verify-rc/github.macos.amd64.yml @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + verify: + name: "Verify release candidate macOS {{ artifact }}" + runs-on: {{ github_runner|default("macos-latest") }} + {% if env is defined %} + env: + {% for key, value in env.items() %} + {{ key }}: {{ value }} + {% endfor %} + {% endif %} + + steps: + {{ macros.github_checkout_arrow()|indent }} + + - name: Install System Dependencies + shell: bash + run: | + brew update + brew bundle --file=arrow/cpp/Brewfile + brew bundle --file=arrow/c_glib/Brewfile + - uses: actions/setup-node@v2-beta + with: + node-version: '14' + - name: Run verification + shell: bash + run: | + arrow/dev/release/verify-release-candidate.sh \ + {{ artifact }} \ + {{ release|default("1.0.0") }} {{ rc|default("0") }} diff --git a/src/arrow/dev/tasks/verify-rc/github.macos.arm64.yml b/src/arrow/dev/tasks/verify-rc/github.macos.arm64.yml new file mode 100644 index 000000000..26139ed60 --- /dev/null +++ b/src/arrow/dev/tasks/verify-rc/github.macos.arm64.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + verify: + name: "Verify release candidate macOS {{ artifact }}" + runs-on: {{ github_runner }} + {% if env is defined %} + env: + {% for key, value in env.items() %} + {{ key }}: {{ value }} + {% endfor %} + {% endif %} + + steps: + - name: Cleanup + shell: bash + run: rm -rf arrow + + {{ macros.github_checkout_arrow()|indent }} + + - name: Run verification + shell: bash + run: | + export PATH="$(brew --prefix node@14)/bin:$PATH" + export PATH="$(brew --prefix ruby)/bin:$PATH" + export PKG_CONFIG_PATH="$(brew --prefix ruby)/lib/pkgconfig" + arch -{{ arch_emulation|default("arm64") }} arrow/dev/release/verify-release-candidate.sh \ + {{ artifact }} \ + {{ release|default("1.0.0") }} {{ rc|default("0") }} diff --git a/src/arrow/dev/tasks/verify-rc/github.win.yml b/src/arrow/dev/tasks/verify-rc/github.win.yml new file mode 100644 index 000000000..5406327e8 --- /dev/null +++ b/src/arrow/dev/tasks/verify-rc/github.win.yml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{% import 'macros.jinja' as macros with context %} + +{{ macros.github_header() }} + +jobs: + verify: + name: "Verify release candidate Windows source" + runs-on: windows-2016 + {% if env is defined %} + env: + {% for key, value in env.items() %} + {{ key }}: {{ value }} + {% endfor %} + {% endif %} + + steps: + {{ macros.github_checkout_arrow()|indent }} + + - uses: conda-incubator/setup-miniconda@v2 + - name: Install System Dependencies + run: | + choco install boost-msvc-14.1 + choco install wget + - name: Run verification + shell: cmd + run: | + cd arrow + dev/release/{{ script }} {{ release|default("1.0.0") }} {{ rc|default("0") }} |