diff options
Diffstat (limited to 'src/arrow/ci/docker')
42 files changed, 2644 insertions, 0 deletions
diff --git a/src/arrow/ci/docker/conda-cpp.dockerfile b/src/arrow/ci/docker/conda-cpp.dockerfile new file mode 100644 index 000000000..ff31930c0 --- /dev/null +++ b/src/arrow/ci/docker/conda-cpp.dockerfile @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch +FROM ${repo}:${arch}-conda + +# install the required conda packages into the test environment +COPY ci/conda_env_cpp.txt \ + ci/conda_env_gandiva.txt \ + /arrow/ci/ +RUN conda install \ + --file arrow/ci/conda_env_cpp.txt \ + --file arrow/ci/conda_env_gandiva.txt \ + compilers \ + doxygen \ + valgrind && \ + conda clean --all + +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DATASET=ON \ + ARROW_DEPENDENCY_SOURCE=CONDA \ + ARROW_FLIGHT=ON \ + ARROW_GANDIVA=ON \ + ARROW_HOME=$CONDA_PREFIX \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_S3=ON \ + ARROW_USE_CCACHE=ON \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + PARQUET_BUILD_EXAMPLES=ON \ + PARQUET_BUILD_EXECUTABLES=ON \ + PARQUET_HOME=$CONDA_PREFIX diff --git a/src/arrow/ci/docker/conda-integration.dockerfile b/src/arrow/ci/docker/conda-integration.dockerfile new file mode 100644 index 000000000..43d8d943b --- /dev/null +++ b/src/arrow/ci/docker/conda-integration.dockerfile @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +FROM ${repo}:${arch}-conda-cpp + +ARG arch=amd64 +ARG maven=3.5 +ARG node=14 +ARG jdk=8 +ARG go=1.15 + +# Install Archery and integration dependencies +COPY ci/conda_env_archery.txt /arrow/ci/ +RUN conda install -q \ + --file arrow/ci/conda_env_archery.txt \ + "python>=3.7" \ + numpy \ + compilers \ + maven=${maven} \ + nodejs=${node} \ + yarn \ + openjdk=${jdk} && \ + conda clean --all --force-pkgs-dirs + +# Install Rust with only the needed components +# (rustfmt is needed for tonic-build to compile the protobuf definitions) +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --profile=minimal -y && \ + $HOME/.cargo/bin/rustup component add rustfmt + +ENV GOROOT=/opt/go \ + GOBIN=/opt/go/bin \ + GOPATH=/go \ + PATH=/opt/go/bin:$PATH +RUN wget -nv -O - https://dl.google.com/go/go${go}.linux-${arch}.tar.gz | tar -xzf - -C /opt + +ENV DOTNET_ROOT=/opt/dotnet \ + PATH=/opt/dotnet:$PATH +RUN curl -sSL https://dot.net/v1/dotnet-install.sh | bash /dev/stdin -Channel 3.1 -InstallDir /opt/dotnet + +ENV ARROW_BUILD_INTEGRATION=ON \ + ARROW_BUILD_STATIC=OFF \ + ARROW_BUILD_TESTS=OFF \ + ARROW_COMPUTE=OFF \ + ARROW_CSV=OFF \ + ARROW_DATASET=OFF \ + ARROW_FILESYSTEM=OFF \ + ARROW_FLIGHT=ON \ + ARROW_GANDIVA=OFF \ + ARROW_HDFS=OFF \ + ARROW_JEMALLOC=OFF \ + ARROW_JSON=OFF \ + ARROW_ORC=OFF \ + ARROW_PARQUET=OFF \ + ARROW_PLASMA=OFF \ + ARROW_S3=OFF \ + ARROW_USE_GLOG=OFF \ + CMAKE_UNITY_BUILD=ON diff --git a/src/arrow/ci/docker/conda-python-dask.dockerfile b/src/arrow/ci/docker/conda-python-dask.dockerfile new file mode 100644 index 000000000..cd59a5538 --- /dev/null +++ b/src/arrow/ci/docker/conda-python-dask.dockerfile @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +ARG python=3.6 +FROM ${repo}:${arch}-conda-python-${python} + +ARG dask=latest +COPY ci/scripts/install_dask.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_dask.sh ${dask}
\ No newline at end of file diff --git a/src/arrow/ci/docker/conda-python-hdfs.dockerfile b/src/arrow/ci/docker/conda-python-hdfs.dockerfile new file mode 100644 index 000000000..f6ffc71ce --- /dev/null +++ b/src/arrow/ci/docker/conda-python-hdfs.dockerfile @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +ARG python=3.6 +FROM ${repo}:${arch}-conda-python-${python} + +ARG jdk=8 +ARG maven=3.5 +RUN conda install -q \ + maven=${maven} \ + openjdk=${jdk} \ + pandas && \ + conda clean --all + +# installing libhdfs (JNI) +ARG hdfs=3.2.1 +ENV HADOOP_HOME=/opt/hadoop-${hdfs} \ + HADOOP_OPTS=-Djava.library.path=/opt/hadoop-${hdfs}/lib/native \ + PATH=$PATH:/opt/hadoop-${hdfs}/bin:/opt/hadoop-${hdfs}/sbin +COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/util_download_apache.sh \ + "hadoop/common/hadoop-${hdfs}/hadoop-${hdfs}.tar.gz" /opt + +COPY ci/etc/hdfs-site.xml $HADOOP_HOME/etc/hadoop/ + +# build cpp with tests +ENV CC=gcc \ + CXX=g++ \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=OFF \ + ARROW_PLASMA=OFF \ + ARROW_PARQUET=ON \ + ARROW_ORC=OFF \ + ARROW_HDFS=ON \ + ARROW_PYTHON=ON \ + ARROW_BUILD_TESTS=ON diff --git a/src/arrow/ci/docker/conda-python-jpype.dockerfile b/src/arrow/ci/docker/conda-python-jpype.dockerfile new file mode 100644 index 000000000..f77ef9bf6 --- /dev/null +++ b/src/arrow/ci/docker/conda-python-jpype.dockerfile @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +ARG python=3.6 +FROM ${repo}:${arch}-conda-python-${python} + +ARG jdk=11 +ARG maven=3.6 +RUN conda install -q \ + maven=${maven} \ + openjdk=${jdk} \ + jpype1 && \ + conda clean --all diff --git a/src/arrow/ci/docker/conda-python-kartothek.dockerfile b/src/arrow/ci/docker/conda-python-kartothek.dockerfile new file mode 100644 index 000000000..d52316182 --- /dev/null +++ b/src/arrow/ci/docker/conda-python-kartothek.dockerfile @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +ARG python=3.6 +FROM ${repo}:${arch}-conda-python-${python} + +# install kartothek dependencies from conda-forge +RUN conda install -c conda-forge -q \ + attrs \ + click \ + cloudpickle \ + dask \ + decorator \ + freezegun \ + msgpack-python \ + prompt-toolkit \ + pytest-mock \ + pytest-xdist \ + pyyaml \ + simplejson \ + simplekv \ + storefact \ + toolz \ + urlquote \ + zstandard && \ + conda clean --all + +ARG kartothek=latest +COPY ci/scripts/install_kartothek.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_kartothek.sh ${kartothek} /kartothek diff --git a/src/arrow/ci/docker/conda-python-pandas.dockerfile b/src/arrow/ci/docker/conda-python-pandas.dockerfile new file mode 100644 index 000000000..303cc80e4 --- /dev/null +++ b/src/arrow/ci/docker/conda-python-pandas.dockerfile @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +ARG python=3.6 +FROM ${repo}:${arch}-conda-python-${python} + +ARG pandas=latest +ARG numpy=latest +COPY ci/scripts/install_pandas.sh /arrow/ci/scripts/ +RUN conda uninstall -q -y numpy && \ + /arrow/ci/scripts/install_pandas.sh ${pandas} ${numpy} diff --git a/src/arrow/ci/docker/conda-python-spark.dockerfile b/src/arrow/ci/docker/conda-python-spark.dockerfile new file mode 100644 index 000000000..a2af2ac13 --- /dev/null +++ b/src/arrow/ci/docker/conda-python-spark.dockerfile @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +ARG python=3.6 +FROM ${repo}:${arch}-conda-python-${python} + +ARG jdk=8 +ARG maven=3.5 + +RUN conda install -q \ + openjdk=${jdk} \ + maven=${maven} \ + pandas && \ + conda clean --all + +# installing specific version of spark +ARG spark=master +COPY ci/scripts/install_spark.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_spark.sh ${spark} /spark + +# build cpp with tests +ENV CC=gcc \ + CXX=g++ \ + ARROW_PYTHON=ON \ + ARROW_HDFS=ON \ + ARROW_BUILD_TESTS=OFF \ + SPARK_VERSION=${spark} diff --git a/src/arrow/ci/docker/conda-python-turbodbc.dockerfile b/src/arrow/ci/docker/conda-python-turbodbc.dockerfile new file mode 100644 index 000000000..e748604de --- /dev/null +++ b/src/arrow/ci/docker/conda-python-turbodbc.dockerfile @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch=amd64 +ARG python=3.6 +FROM ${repo}:${arch}-conda-python-${python} + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + odbc-postgresql \ + postgresql \ + sudo && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# install turbodbc dependencies from conda-forge +RUN conda install -c conda-forge -q \ + pybind11 \ + pytest-cov \ + mock \ + unixodbc && \ + conda clean --all + +RUN service postgresql start && \ + sudo -u postgres psql -U postgres -c \ + "CREATE DATABASE test_db;" && \ + sudo -u postgres psql -U postgres -c \ + "ALTER USER postgres WITH PASSWORD 'password';" + +ARG turbodbc=latest +COPY ci/scripts/install_turbodbc.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_turbodbc.sh ${turbodbc} /turbodbc + +ENV TURBODBC_TEST_CONFIGURATION_FILES "query_fixtures_postgresql.json" diff --git a/src/arrow/ci/docker/conda-python.dockerfile b/src/arrow/ci/docker/conda-python.dockerfile new file mode 100644 index 000000000..ab3f77be1 --- /dev/null +++ b/src/arrow/ci/docker/conda-python.dockerfile @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG repo +ARG arch +FROM ${repo}:${arch}-conda-cpp + +# install python specific packages +ARG python=3.6 +COPY ci/conda_env_python.txt /arrow/ci/ +RUN conda install -q \ + --file arrow/ci/conda_env_python.txt \ + $([ "$python" == "3.6" -o "$python" == "3.7" ] && echo "pickle5") \ + python=${python} \ + nomkl && \ + conda clean --all + +ENV ARROW_PYTHON=ON \ + ARROW_BUILD_STATIC=OFF \ + ARROW_BUILD_TESTS=OFF \ + ARROW_BUILD_UTILITIES=OFF \ + ARROW_TENSORFLOW=ON \ + ARROW_USE_GLOG=OFF diff --git a/src/arrow/ci/docker/conda.dockerfile b/src/arrow/ci/docker/conda.dockerfile new file mode 100644 index 000000000..adb64f9fa --- /dev/null +++ b/src/arrow/ci/docker/conda.dockerfile @@ -0,0 +1,55 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +FROM ${arch}/ubuntu:18.04 + +# arch is unset after the FROM statement, so need to define it again +ARG arch=amd64 +ARG prefix=/opt/conda + +# install build essentials +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update -y -q && \ + apt-get install -y -q wget tzdata libc6-dbg gdb \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH=${prefix}/bin:$PATH +# install conda and minio +COPY ci/scripts/install_conda.sh \ + ci/scripts/install_minio.sh \ + /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_conda.sh ${arch} linux latest ${prefix} +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest ${prefix} +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts +RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default + +# create a conda environment +ADD ci/conda_env_unix.txt /arrow/ci/ +RUN conda create -n arrow --file arrow/ci/conda_env_unix.txt git && \ + conda clean --all + +# activate the created environment by default +RUN echo "conda activate arrow" >> ~/.profile +ENV CONDA_PREFIX=${prefix}/envs/arrow + +# use login shell to activate arrow environment un the RUN commands +SHELL [ "/bin/bash", "-c", "-l" ] + +# use login shell when running the container +ENTRYPOINT [ "/bin/bash", "-c", "-l" ] diff --git a/src/arrow/ci/docker/debian-10-cpp.dockerfile b/src/arrow/ci/docker/debian-10-cpp.dockerfile new file mode 100644 index 000000000..16e867fc3 --- /dev/null +++ b/src/arrow/ci/docker/debian-10-cpp.dockerfile @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +FROM ${arch}/debian:10 +ARG arch + +ENV DEBIAN_FRONTEND noninteractive + +RUN \ + echo "deb http://deb.debian.org/debian buster-backports main" > \ + /etc/apt/sources.list.d/backports.list + +ARG llvm +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/buster/ llvm-toolchain-buster-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ccache \ + clang-${llvm} \ + cmake \ + g++ \ + gcc \ + gdb \ + git \ + libbenchmark-dev \ + libboost-all-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + libgmock-dev \ + libgoogle-glog-dev \ + liblz4-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + llvm-${llvm}-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + python3-pip \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default + +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DATASET=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_FLIGHT=ON \ + ARROW_GANDIVA=ON \ + ARROW_HOME=/usr/local \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_S3=ON \ + ARROW_USE_CCACHE=ON \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ + cares_SOURCE=BUNDLED \ + CC=gcc \ + CXX=g++ \ + gRPC_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + PATH=/usr/lib/ccache/:$PATH \ + Protobuf_SOURCE=BUNDLED \ + zstd_SOURCE=BUNDLED diff --git a/src/arrow/ci/docker/debian-10-go-cgo-python.dockerfile b/src/arrow/ci/docker/debian-10-go-cgo-python.dockerfile new file mode 100644 index 000000000..46455a42b --- /dev/null +++ b/src/arrow/ci/docker/debian-10-go-cgo-python.dockerfile @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +ENV DEBIAN_FRONTEND noninteractive + +# Install python3 and pip so we can install pyarrow to test the C data interface. +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + python3 \ + python3-pip && \ + apt-get clean + +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + +# Need a newer pip than Debian's to install manylinux201x wheels +RUN pip install -U pip + +RUN pip install pyarrow cffi --only-binary pyarrow diff --git a/src/arrow/ci/docker/debian-10-go.dockerfile b/src/arrow/ci/docker/debian-10-go.dockerfile new file mode 100644 index 000000000..3a24b8afe --- /dev/null +++ b/src/arrow/ci/docker/debian-10-go.dockerfile @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +ARG go=1.15 +FROM ${arch}/golang:${go}-buster + + +# TODO(kszucs): +# 1. add the files required to install the dependencies to .dockerignore +# 2. copy these files to their appropriate path +# 3. download and compile the dependencies diff --git a/src/arrow/ci/docker/debian-10-js.dockerfile b/src/arrow/ci/docker/debian-10-js.dockerfile new file mode 100644 index 000000000..5bb31f2e3 --- /dev/null +++ b/src/arrow/ci/docker/debian-10-js.dockerfile @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +ARG node=14 +FROM ${arch}/node:${node} + +ENV NODE_NO_WARNINGS=1 + +# TODO(kszucs): +# 1. add the files required to install the dependencies to .dockerignore +# 2. copy these files to their appropriate path +# 3. download and compile the dependencies diff --git a/src/arrow/ci/docker/debian-11-cpp.dockerfile b/src/arrow/ci/docker/debian-11-cpp.dockerfile new file mode 100644 index 000000000..659881b0c --- /dev/null +++ b/src/arrow/ci/docker/debian-11-cpp.dockerfile @@ -0,0 +1,100 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +FROM ${arch}/debian:11 +ARG arch + +ENV DEBIAN_FRONTEND noninteractive + +ARG llvm +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ccache \ + clang-${llvm} \ + cmake \ + g++ \ + gcc \ + gdb \ + git \ + libbenchmark-dev \ + libboost-all-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + libgmock-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + liblz4-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + llvm-${llvm}-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler-grpc \ + python3-pip \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default + +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DATASET=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_FLIGHT=ON \ + ARROW_GANDIVA=ON \ + ARROW_HOME=/usr/local \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_S3=ON \ + ARROW_USE_CCACHE=ON \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ + CC=gcc \ + CXX=g++ \ + ORC_SOURCE=BUNDLED \ + PATH=/usr/lib/ccache/:$PATH \ + Protobuf_SOURCE=BUNDLED diff --git a/src/arrow/ci/docker/debian-11-go-cgo-python.dockerfile b/src/arrow/ci/docker/debian-11-go-cgo-python.dockerfile new file mode 100644 index 000000000..46455a42b --- /dev/null +++ b/src/arrow/ci/docker/debian-11-go-cgo-python.dockerfile @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +ENV DEBIAN_FRONTEND noninteractive + +# Install python3 and pip so we can install pyarrow to test the C data interface. +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + python3 \ + python3-pip && \ + apt-get clean + +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + +# Need a newer pip than Debian's to install manylinux201x wheels +RUN pip install -U pip + +RUN pip install pyarrow cffi --only-binary pyarrow diff --git a/src/arrow/ci/docker/debian-11-go.dockerfile b/src/arrow/ci/docker/debian-11-go.dockerfile new file mode 100644 index 000000000..3a24b8afe --- /dev/null +++ b/src/arrow/ci/docker/debian-11-go.dockerfile @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +ARG go=1.15 +FROM ${arch}/golang:${go}-buster + + +# TODO(kszucs): +# 1. add the files required to install the dependencies to .dockerignore +# 2. copy these files to their appropriate path +# 3. download and compile the dependencies diff --git a/src/arrow/ci/docker/debian-11-js.dockerfile b/src/arrow/ci/docker/debian-11-js.dockerfile new file mode 100644 index 000000000..5bb31f2e3 --- /dev/null +++ b/src/arrow/ci/docker/debian-11-js.dockerfile @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +ARG node=14 +FROM ${arch}/node:${node} + +ENV NODE_NO_WARNINGS=1 + +# TODO(kszucs): +# 1. add the files required to install the dependencies to .dockerignore +# 2. copy these files to their appropriate path +# 3. download and compile the dependencies diff --git a/src/arrow/ci/docker/debian-9-java.dockerfile b/src/arrow/ci/docker/debian-9-java.dockerfile new file mode 100644 index 000000000..2cc36e3ea --- /dev/null +++ b/src/arrow/ci/docker/debian-9-java.dockerfile @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch=amd64 +ARG jdk=8 +ARG maven=3.5.4 +FROM ${arch}/maven:${maven}-jdk-${jdk} + +ENV ARROW_JAVA_SHADE_FLATBUFS=ON + +# TODO(kszucs): +# 1. add the files required to install the dependencies to .dockerignore +# 2. copy these files to their appropriate path +# 3. download and compile the dependencies diff --git a/src/arrow/ci/docker/debian-go-cgo.dockerfile b/src/arrow/ci/docker/debian-go-cgo.dockerfile new file mode 100644 index 000000000..a494d1e15 --- /dev/null +++ b/src/arrow/ci/docker/debian-go-cgo.dockerfile @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +ENV DEBIAN_FRONTEND noninteractive + +# install libarrow-dev to link against with CGO +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends ca-certificates lsb-release wget && \ + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \ + apt-get install -y -q --no-install-recommends ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + cmake \ + libarrow-dev && \ + apt-get clean diff --git a/src/arrow/ci/docker/fedora-33-cpp.dockerfile b/src/arrow/ci/docker/fedora-33-cpp.dockerfile new file mode 100644 index 000000000..61964a476 --- /dev/null +++ b/src/arrow/ci/docker/fedora-33-cpp.dockerfile @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch +FROM ${arch}/fedora:33 +ARG arch + +# install dependencies +RUN dnf update -y && \ + dnf install -y \ + autoconf \ + boost-devel \ + brotli-devel \ + bzip2-devel \ + c-ares-devel \ + ccache \ + clang-devel \ + cmake \ + curl-devel \ + flatbuffers-devel \ + gcc \ + gcc-c++ \ + gflags-devel \ + git \ + glog-devel \ + gmock-devel \ + google-benchmark-devel \ + grpc-devel \ + grpc-plugins \ + gtest-devel \ + java-latest-openjdk-devel \ + java-latest-openjdk-headless \ + libzstd-devel \ + llvm-devel \ + llvm-static \ + lz4-devel \ + make \ + ninja-build \ + openssl-devel \ + protobuf-devel \ + python \ + python-pip \ + rapidjson-devel \ + re2-devel \ + snappy-devel \ + thrift-devel \ + utf8proc-devel \ + wget \ + which \ + zlib-devel + +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default + +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_DATASET=ON \ + ARROW_FLIGHT=ON \ + ARROW_GANDIVA_JAVA=ON \ + ARROW_GANDIVA=ON \ + ARROW_HOME=/usr/local \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_S3=ON \ + ARROW_USE_CCACHE=ON \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ + CC=gcc \ + CXX=g++ \ + ORC_SOURCE=BUNDLED \ + PARQUET_BUILD_EXECUTABLES=ON \ + PARQUET_BUILD_EXAMPLES=ON \ + PATH=/usr/lib/ccache/:$PATH diff --git a/src/arrow/ci/docker/java-jni-manylinux-201x.dockerfile b/src/arrow/ci/docker/java-jni-manylinux-201x.dockerfile new file mode 100644 index 000000000..021dab686 --- /dev/null +++ b/src/arrow/ci/docker/java-jni-manylinux-201x.dockerfile @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +# Install the libaries required by the Gandiva to run +RUN vcpkg install --clean-after-build \ + llvm \ + boost-system \ + boost-date-time \ + boost-regex \ + boost-predef \ + boost-algorithm \ + boost-locale \ + boost-format \ + boost-variant \ + boost-multiprecision + +# Install Java +ARG java=1.8.0 +RUN yum install -y java-$java-openjdk-devel && yum clean all +ENV JAVA_HOME=/usr/lib/jvm/java-$java-openjdk/ diff --git a/src/arrow/ci/docker/linux-apt-c-glib.dockerfile b/src/arrow/ci/docker/linux-apt-c-glib.dockerfile new file mode 100644 index 000000000..12c6e23a0 --- /dev/null +++ b/src/arrow/ci/docker/linux-apt-c-glib.dockerfile @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +RUN apt-get update -y -q && \ + apt-get install -y -q \ + python3 \ + python3-pip \ + gtk-doc-tools \ + libgirepository1.0-dev \ + libglib2.0-doc \ + lsb-release \ + luarocks \ + pkg-config \ + ruby-dev && \ + if [ "$(lsb_release --codename --short)" = "xenial" ]; then \ + apt-get install -y -q --no-install-recommends -t xenial-backports \ + ninja-build; \ + fi && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN luarocks install lgi + +# pip on Ubuntu 20.04 may be buggy: +# +# Collecting meson +# Downloading meson-0.53.2.tar.gz (1.6 MB) +# Installing build dependencies: started +# Installing build dependencies: finished with status 'done' +# Getting requirements to build wheel: started +# Getting requirements to build wheel: finished with status 'error' +# ERROR: Command errored out with exit status 1: +# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay +# cwd: /tmp/pip-install-jn79a_kh/meson +# Complete output (1 lines): +# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py' +# ---------------------------------------- +# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output. +RUN (python3 -m pip install meson || \ + python3 -m pip install --no-use-pep517 meson) && \ + gem install --no-document bundler + +COPY c_glib/Gemfile /arrow/c_glib/ +RUN bundle install --gemfile /arrow/c_glib/Gemfile + +ENV ARROW_BUILD_TESTS=OFF \ + ARROW_BUILD_UTILITIES=OFF \ + ARROW_INSTALL_NAME_RPATH=OFF diff --git a/src/arrow/ci/docker/linux-apt-docs.dockerfile b/src/arrow/ci/docker/linux-apt-docs.dockerfile new file mode 100644 index 000000000..12c797f96 --- /dev/null +++ b/src/arrow/ci/docker/linux-apt-docs.dockerfile @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +ARG r=4.1 +ARG jdk=8 + +# See R install instructions at https://cloud.r-project.org/bin/linux/ubuntu/ +RUN apt-get update -y && \ + apt-get install -y \ + dirmngr \ + apt-transport-https \ + software-properties-common && \ + apt-key adv \ + --keyserver keyserver.ubuntu.com \ + --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran40/' && \ + apt-get install -y --no-install-recommends \ + autoconf-archive \ + automake \ + curl \ + doxygen \ + gobject-introspection \ + gtk-doc-tools \ + libcurl4-openssl-dev \ + libfontconfig1-dev \ + libfribidi-dev \ + libgirepository1.0-dev \ + libglib2.0-doc \ + libharfbuzz-dev \ + libtiff-dev \ + libtool \ + libxml2-dev \ + ninja-build \ + nvidia-cuda-toolkit \ + openjdk-${jdk}-jdk-headless \ + pandoc \ + r-recommended=${r}* \ + r-base=${r}* \ + rsync \ + ruby-dev \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV JAVA_HOME=/usr/lib/jvm/java-${jdk}-openjdk-amd64 + +ARG maven=3.5.4 +COPY ci/scripts/util_download_apache.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/util_download_apache.sh \ + "maven/maven-3/${maven}/binaries/apache-maven-${maven}-bin.tar.gz" /opt +ENV PATH=/opt/apache-maven-${maven}/bin:$PATH +RUN mvn -version + +ARG node=14 +RUN wget -q -O - https://deb.nodesource.com/setup_${node}.x | bash - && \ + apt-get install -y nodejs && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + npm install -g yarn + +# ARROW-13353: breathe >= 4.29.1 tries to parse template arguments, +# but Sphinx can't parse constructs like `typename...`. +RUN pip install \ + meson \ + breathe==4.29.0 \ + ipython \ + sphinx \ + pydata-sphinx-theme + +COPY c_glib/Gemfile /arrow/c_glib/ +RUN gem install --no-document bundler && \ + bundle install --gemfile /arrow/c_glib/Gemfile + +# Ensure parallel R package installation, set CRAN repo mirror, +# and use pre-built binaries where possible +COPY ci/etc/rprofile /arrow/ci/etc/ +RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site +# Also ensure parallel compilation of C/C++ code +RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Renviron.site + +COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ +COPY r/DESCRIPTION /arrow/r/ +RUN /arrow/ci/scripts/r_deps.sh /arrow && \ + R -e "install.packages('pkgdown')" + +ENV ARROW_FLIGHT=ON \ + ARROW_PYTHON=ON \ + ARROW_S3=ON \ + ARROW_BUILD_STATIC=OFF \ + ARROW_BUILD_TESTS=OFF \ + ARROW_BUILD_UTILITIES=OFF \ + ARROW_USE_GLOG=OFF \ + CMAKE_UNITY_BUILD=ON \ diff --git a/src/arrow/ci/docker/linux-apt-jni.dockerfile b/src/arrow/ci/docker/linux-apt-jni.dockerfile new file mode 100644 index 000000000..ddfa72e17 --- /dev/null +++ b/src/arrow/ci/docker/linux-apt-jni.dockerfile @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +# pipefail is enabled for proper error detection in the `wget | apt-key add` +# step +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ENV DEBIAN_FRONTEND noninteractive + +ARG llvm +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + lsb-release \ + software-properties-common \ + wget && \ + code_name=$(lsb_release --codename --short) && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + apt-add-repository -y \ + "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${llvm} main" && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + ca-certificates \ + ccache \ + clang-${llvm} \ + cmake \ + git \ + g++ \ + gcc \ + libboost-all-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libgtest-dev \ + liblz4-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + llvm-${llvm}-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + rapidjson-dev \ + tzdata \ + zlib1g-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ARG cmake=3.11.4 +RUN wget -nv -O - https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-x86_64.tar.gz | tar -xzf - -C /opt +ENV PATH=/opt/cmake-${cmake}-Linux-x86_64/bin:$PATH + +ENV ARROW_BUILD_TESTS=OFF \ + ARROW_DATASET=ON \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA_JAVA=ON \ + ARROW_GANDIVA=ON \ + ARROW_HOME=/usr/local \ + ARROW_JAVA_CDATA=ON \ + ARROW_JNI=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA_JAVA_CLIENT=ON \ + ARROW_PLASMA=ON \ + ARROW_USE_CCACHE=ON \ + CC=gcc \ + CXX=g++ \ + ORC_SOURCE=BUNDLED \ + PATH=/usr/lib/ccache/:$PATH \ + Protobuf_SOURCE=BUNDLED diff --git a/src/arrow/ci/docker/linux-apt-lint.dockerfile b/src/arrow/ci/docker/linux-apt-lint.dockerfile new file mode 100644 index 000000000..84de6b05f --- /dev/null +++ b/src/arrow/ci/docker/linux-apt-lint.dockerfile @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM hadolint/hadolint:v1.17.2 AS hadolint +FROM ${base} + +ARG clang_tools +RUN apt-get update && \ + apt-get install -y -q \ + clang-${clang_tools} \ + clang-format-${clang_tools} \ + clang-tidy-${clang_tools} \ + clang-tools-${clang_tools} \ + cmake \ + curl \ + libclang-${clang_tools}-dev \ + llvm-${clang_tools}-dev \ + openjdk-11-jdk-headless \ + python3 \ + python3-dev \ + python3-pip \ + ruby \ + apt-transport-https \ + software-properties-common \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ARG r=4.1 +RUN apt-key adv \ + --keyserver keyserver.ubuntu.com \ + --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + # NOTE: R 3.5 and 3.6 are available in the repos with -cran35 suffix + # for trusty, xenial, bionic, and eoan (as of May 2020) + # -cran40 has 4.0 versions for bionic and focal + # R 3.2, 3.3, 3.4 are available without the suffix but only for trusty and xenial + # TODO: make sure OS version and R version are valid together and conditionally set repo suffix + # This is a hack to turn 3.6 into 35, and 4.0/4.1 into 40: + add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran'$(echo "${r}" | tr -d . | tr 6 5 | tr 1 0)'/' && \ + apt-get install -y \ + r-base=${r}* \ + r-recommended=${r}* \ + libxml2-dev + +# Ensure parallel R package installation, set CRAN repo mirror, +# and use pre-built binaries where possible +COPY ci/etc/rprofile /arrow/ci/etc/ +RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site +# Also ensure parallel compilation of C/C++ code +RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Renviron.site + + +COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ +COPY r/DESCRIPTION /arrow/r/ +# We need to install Arrow's dependencies in order for lintr's namespace searching to work. +# This could be removed if lintr no longer loads the dependency namespaces (see issues/PRs below) +RUN /arrow/ci/scripts/r_deps.sh /arrow +# This fork has a number of changes that have PRs and Issues to resolve upstream: +# https://github.com/jimhester/lintr/pull/843 +# https://github.com/jimhester/lintr/pull/841 +# https://github.com/jimhester/lintr/pull/845 +# https://github.com/jimhester/lintr/issues/842 +# https://github.com/jimhester/lintr/issues/846 +RUN R -e "remotes::install_github('jonkeane/lintr@arrow-branch')" + +# Docker linter +COPY --from=hadolint /bin/hadolint /usr/bin/hadolint + +# IWYU +COPY ci/scripts/install_iwyu.sh /arrow/ci/scripts/ +RUN arrow/ci/scripts/install_iwyu.sh /tmp/iwyu /usr/local ${clang_tools} + +# Use python3 by default in scripts +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + +COPY dev/archery/setup.py /arrow/dev/archery/ +RUN pip install -e arrow/dev/archery[lint] + +ENV LC_ALL=C.UTF-8 \ + LANG=C.UTF-8 diff --git a/src/arrow/ci/docker/linux-apt-python-3.dockerfile b/src/arrow/ci/docker/linux-apt-python-3.dockerfile new file mode 100644 index 000000000..753ba0d3a --- /dev/null +++ b/src/arrow/ci/docker/linux-apt-python-3.dockerfile @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +RUN apt-get update -y -q && \ + apt-get install -y -q \ + python3 \ + python3-pip \ + python3-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + +RUN pip install -U pip setuptools + +COPY python/requirements-build.txt \ + python/requirements-test.txt \ + /arrow/python/ + +RUN pip install \ + -r arrow/python/requirements-build.txt \ + -r arrow/python/requirements-test.txt + +ENV ARROW_PYTHON=ON \ + ARROW_BUILD_STATIC=OFF \ + ARROW_BUILD_TESTS=OFF \ + ARROW_BUILD_UTILITIES=OFF \ + ARROW_USE_GLOG=OFF \ diff --git a/src/arrow/ci/docker/linux-apt-r.dockerfile b/src/arrow/ci/docker/linux-apt-r.dockerfile new file mode 100644 index 000000000..6d33d1800 --- /dev/null +++ b/src/arrow/ci/docker/linux-apt-r.dockerfile @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} +ARG arch + +ARG tz="UTC" +ENV TZ=${tz} + +# Build R +# [1] https://www.digitalocean.com/community/tutorials/how-to-install-r-on-ubuntu-18-04 +# [2] https://linuxize.com/post/how-to-install-r-on-ubuntu-18-04/#installing-r-packages-from-cran +ARG r=3.6 +RUN apt-get update -y && \ + apt-get install -y \ + dirmngr \ + apt-transport-https \ + software-properties-common && \ + apt-key adv \ + --keyserver keyserver.ubuntu.com \ + --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 && \ + # NOTE: R 3.5 and 3.6 are available in the repos with -cran35 suffix + # for trusty, xenial, bionic, and eoan (as of May 2020) + # -cran40 has 4.0 versions for bionic and focal + # R 3.2, 3.3, 3.4 are available without the suffix but only for trusty and xenial + # TODO: make sure OS version and R version are valid together and conditionally set repo suffix + # This is a hack to turn 3.6 into 35, and 4.0/4.1 into 40: + add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu '$(lsb_release -cs)'-cran'$(echo "${r}" | tr -d . | tr 6 5 | tr 1 0)'/' && \ + apt-get install -y \ + r-base=${r}* \ + r-recommended=${r}* \ + # system libs needed by core R packages + libxml2-dev \ + libgit2-dev \ + libssl-dev \ + # install clang to mirror what was done on Travis + clang \ + clang-format \ + clang-tidy \ + # R CMD CHECK --as-cran needs pdflatex to build the package manual + texlive-latex-base \ + # Need locales so we can set UTF-8 + locales \ + # Need Python to check py-to-r bridge + python3 \ + python3-pip \ + python3-dev && \ + locale-gen en_US.UTF-8 && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ARG gcc_version="" +RUN if [ "${gcc_version}" != "" ]; then \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \ + update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 && \ + update-alternatives --set cc /usr/bin/gcc && \ + update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 && \ + update-alternatives --set c++ /usr/bin/g++; \ + fi + +# Ensure parallel R package installation, set CRAN repo mirror, +# and use pre-built binaries where possible +COPY ci/etc/rprofile /arrow/ci/etc/ +RUN cat /arrow/ci/etc/rprofile >> $(R RHOME)/etc/Rprofile.site +# Also ensure parallel compilation of C/C++ code +RUN echo "MAKEFLAGS=-j$(R -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Renviron.site + +COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ +COPY r/DESCRIPTION /arrow/r/ +RUN /arrow/ci/scripts/r_deps.sh /arrow + +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default + +# Set up Python 3 and its dependencies +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + +COPY python/requirements-build.txt /arrow/python/ +RUN pip install -r arrow/python/requirements-build.txt + +ENV \ + ARROW_BUILD_STATIC=OFF \ + ARROW_BUILD_TESTS=OFF \ + ARROW_BUILD_UTILITIES=OFF \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_ORC=OFF \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=OFF \ + ARROW_PYTHON=ON \ + ARROW_S3=ON \ + ARROW_USE_CCACHE=ON \ + ARROW_USE_GLOG=OFF \ + LC_ALL=en_US.UTF-8 diff --git a/src/arrow/ci/docker/linux-apt-ruby.dockerfile b/src/arrow/ci/docker/linux-apt-ruby.dockerfile new file mode 100644 index 000000000..58fd65bd5 --- /dev/null +++ b/src/arrow/ci/docker/linux-apt-ruby.dockerfile @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# depends on a C GLib image +ARG base +FROM ${base} + +COPY ruby/ /arrow/ruby/ +RUN bundle install --gemfile /arrow/ruby/Gemfile +RUN \ + for package in /arrow/ruby/*; do \ + bundle install --gemfile ${package}/Gemfile; \ + done diff --git a/src/arrow/ci/docker/linux-dnf-python-3.dockerfile b/src/arrow/ci/docker/linux-dnf-python-3.dockerfile new file mode 100644 index 000000000..8c3c5c701 --- /dev/null +++ b/src/arrow/ci/docker/linux-dnf-python-3.dockerfile @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +RUN dnf install -y \ + python3 \ + python3-pip \ + python3-devel + +RUN ln -s /usr/bin/python3 /usr/local/bin/python && \ + ln -s /usr/bin/pip3 /usr/local/bin/pip + +COPY python/requirements-build.txt \ + python/requirements-test.txt \ + /arrow/python/ + +RUN pip install \ + -r arrow/python/requirements-build.txt \ + -r arrow/python/requirements-test.txt + +ENV ARROW_PYTHON=ON \ + ARROW_BUILD_STATIC=OFF \ + ARROW_BUILD_TESTS=OFF \ + ARROW_BUILD_UTILITIES=OFF \ + ARROW_USE_GLOG=OFF \ diff --git a/src/arrow/ci/docker/linux-r.dockerfile b/src/arrow/ci/docker/linux-r.dockerfile new file mode 100644 index 000000000..568b90c22 --- /dev/null +++ b/src/arrow/ci/docker/linux-r.dockerfile @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# General purpose Dockerfile to take a Docker image containing R +# and install Arrow R package dependencies + +ARG base +FROM ${base} + +ARG r_bin=R +ENV R_BIN=${r_bin} + +ARG r_dev=FALSE +ENV ARROW_R_DEV=${r_dev} + +ARG devtoolset_version=-1 +ENV DEVTOOLSET_VERSION=${devtoolset_version} + +ARG tz="UTC" +ENV TZ=${tz} + +# Make sure R is on the path for the R-hub devel versions (where RPREFIX is set in its dockerfile) +ENV PATH "${RPREFIX}/bin:${PATH}" + +# Patch up some of the docker images +COPY ci/scripts/r_docker_configure.sh /arrow/ci/scripts/ +COPY ci/etc/rprofile /arrow/ci/etc/ +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/r_docker_configure.sh + +COPY ci/scripts/r_deps.sh /arrow/ci/scripts/ +COPY r/DESCRIPTION /arrow/r/ +RUN /arrow/ci/scripts/r_deps.sh /arrow diff --git a/src/arrow/ci/docker/python-sdist.dockerfile b/src/arrow/ci/docker/python-sdist.dockerfile new file mode 100644 index 000000000..853b532ab --- /dev/null +++ b/src/arrow/ci/docker/python-sdist.dockerfile @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM amd64/ubuntu:20.04 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + git \ + python3-pip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +COPY python/requirements-build.txt \ + /arrow/python/requirements-build.txt +RUN pip3 install --requirement /arrow/python/requirements-build.txt + +ENV PYTHON=/usr/bin/python3 diff --git a/src/arrow/ci/docker/python-wheel-manylinux-201x.dockerfile b/src/arrow/ci/docker/python-wheel-manylinux-201x.dockerfile new file mode 100644 index 000000000..ae1b0a776 --- /dev/null +++ b/src/arrow/ci/docker/python-wheel-manylinux-201x.dockerfile @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base +FROM ${base} + +ARG arch_alias +ARG arch_short_alias + +RUN yum install -y git flex curl autoconf zip wget + +# Install CMake +ARG cmake=3.19.3 +RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-${arch_alias}.tar.gz -O - | \ + tar -xzf - --directory /usr/local --strip-components=1 + +# Install Ninja +ARG ninja=1.10.2 +RUN mkdir /tmp/ninja && \ + wget -q https://github.com/ninja-build/ninja/archive/v${ninja}.tar.gz -O - | \ + tar -xzf - --directory /tmp/ninja --strip-components=1 && \ + cd /tmp/ninja && \ + ./configure.py --bootstrap && \ + mv ninja /usr/local/bin && \ + rm -rf /tmp/ninja + +# Install ccache +ARG ccache=4.1 +RUN mkdir /tmp/ccache && \ + wget -q https://github.com/ccache/ccache/archive/v${ccache}.tar.gz -O - | \ + tar -xzf - --directory /tmp/ccache --strip-components=1 && \ + cd /tmp/ccache && \ + mkdir build && \ + cd build && \ + cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON .. && \ + ninja install && \ + rm -rf /tmp/ccache + +# Install vcpkg +ARG vcpkg +RUN git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \ + git -C /opt/vcpkg checkout ${vcpkg} && \ + /opt/vcpkg/bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics && \ + ln -s /opt/vcpkg/vcpkg /usr/bin/vcpkg + +# Patch ports files as needed +COPY ci/vcpkg/*.patch \ + ci/vcpkg/*linux*.cmake \ + arrow/ci/vcpkg/ +RUN cd /opt/vcpkg && git apply --ignore-whitespace /arrow/ci/vcpkg/ports.patch + +ARG build_type=release +ENV CMAKE_BUILD_TYPE=${build_type} \ + VCPKG_FORCE_SYSTEM_BINARIES=1 \ + VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \ + VCPKG_DEFAULT_TRIPLET=${arch_short_alias}-linux-static-${build_type} \ + VCPKG_FEATURE_FLAGS=-manifests + +# Need to install the boost-build prior installing the boost packages, otherwise +# vcpkg will raise an error. +# TODO(kszucs): factor out the package enumeration to a text file and reuse it +# from the windows image and potentially in a future macos wheel build +RUN vcpkg install --clean-after-build \ + boost-build:${arch_short_alias}-linux && \ + vcpkg install --clean-after-build \ + abseil \ + aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ + boost-filesystem \ + brotli \ + bzip2 \ + c-ares \ + curl \ + flatbuffers \ + gflags \ + glog \ + grpc \ + lz4 \ + openssl \ + orc \ + protobuf \ + rapidjson \ + re2 \ + snappy \ + thrift \ + utf8proc \ + zlib \ + zstd + +ARG python=3.6 +ENV PYTHON_VERSION=${python} +RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \ + echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh + +SHELL ["/bin/bash", "-i", "-c"] +ENTRYPOINT ["/bin/bash", "-i", "-c"] + +COPY python/requirements-wheel-build.txt /arrow/python/ +RUN pip install -r /arrow/python/requirements-wheel-build.txt diff --git a/src/arrow/ci/docker/python-wheel-manylinux-test.dockerfile b/src/arrow/ci/docker/python-wheel-manylinux-test.dockerfile new file mode 100644 index 000000000..55c27d1d7 --- /dev/null +++ b/src/arrow/ci/docker/python-wheel-manylinux-test.dockerfile @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG arch +ARG python +FROM ${arch}/python:${python} + +# RUN pip install --upgrade pip + +# pandas doesn't provide wheel for aarch64 yet, so cache the compiled +# test dependencies in a docker image +COPY python/requirements-wheel-test.txt /arrow/python/ +RUN pip install -r /arrow/python/requirements-wheel-test.txt diff --git a/src/arrow/ci/docker/python-wheel-windows-vs2017.dockerfile b/src/arrow/ci/docker/python-wheel-windows-vs2017.dockerfile new file mode 100644 index 000000000..9a2afb781 --- /dev/null +++ b/src/arrow/ci/docker/python-wheel-windows-vs2017.dockerfile @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# based on mcr.microsoft.com/windows/servercore:ltsc2019 +# contains choco and vs2017 preinstalled +FROM abrarov/msvc-2017:2.10.0 + +# Install CMake and Ninja +RUN choco install --no-progress -r -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' && \ + choco install --no-progress -r -y gzip wget ninja + +# Add unix tools to path +RUN setx path "%path%;C:\Program Files\Git\usr\bin" + +# Install vcpkg +# +# Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has +# started to ship precompiled binaries for the vcpkg-tool. +ARG vcpkg +RUN git clone https://github.com/Microsoft/vcpkg && \ + vcpkg\bootstrap-vcpkg.bat -disableMetrics && \ + setx PATH "%PATH%;C:\vcpkg" && \ + git -C vcpkg checkout %vcpkg% + +# Patch ports files as needed +COPY ci/vcpkg/*.patch \ + ci/vcpkg/*windows*.cmake \ + arrow/ci/vcpkg/ +RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch + +# Configure vcpkg and install dependencies +# NOTE: use windows batch environment notation for build arguments in RUN +# statements but bash notation in ENV statements +# VCPKG_FORCE_SYSTEM_BINARIES=1 spare around ~750MB of image size if the system +# cmake's and ninja's versions are recent enough +ARG build_type=release +ENV CMAKE_BUILD_TYPE=${build_type} \ + VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \ + VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \ + VCPKG_FEATURE_FLAGS=-manifests + +RUN vcpkg install --clean-after-build \ + abseil \ + aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \ + boost-filesystem \ + boost-multiprecision \ + boost-system \ + brotli \ + bzip2 \ + c-ares \ + curl \ + flatbuffers \ + gflags \ + glog \ + grpc \ + lz4 \ + openssl \ + orc \ + protobuf \ + rapidjson \ + re2 \ + snappy \ + thrift \ + utf8proc \ + zlib \ + zstd + +# Remove previous installations of python from the base image +RUN wmic product where "name like 'python%%'" call uninstall /nointeractive && \ + rm -rf Python* + +# Define the full version number otherwise choco falls back to patch number 0 (3.7 => 3.7.0) +ARG python=3.6 +RUN (if "%python%"=="3.6" setx PYTHON_VERSION 3.6.8) & \ + (if "%python%"=="3.7" setx PYTHON_VERSION 3.7.4) & \ + (if "%python%"=="3.8" setx PYTHON_VERSION 3.8.6) & \ + (if "%python%"=="3.9" setx PYTHON_VERSION 3.9.1) & \ + (if "%python%"=="3.10" setx PYTHON_VERSION 3.10.0) +RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% +RUN python -m pip install -U pip + +COPY python/requirements-wheel-build.txt arrow/python/ +RUN pip install -r arrow/python/requirements-wheel-build.txt + +# TODO(kszucs): set clcache as the compiler +ENV CLCACHE_DIR="C:\clcache" +RUN if "%python%" NEQ "3.10" pip install clcache + +# For debugging purposes +# RUN wget --no-check-certificate https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip +# RUN unzip Dependencies_x64_Release.zip -d Dependencies && setx path "%path%;C:\Depencencies" diff --git a/src/arrow/ci/docker/ubuntu-18.04-cpp.dockerfile b/src/arrow/ci/docker/ubuntu-18.04-cpp.dockerfile new file mode 100644 index 000000000..0c05ac4ee --- /dev/null +++ b/src/arrow/ci/docker/ubuntu-18.04-cpp.dockerfile @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base=amd64/ubuntu:18.04 +FROM ${base} + +# pipefail is enabled for proper error detection in the `wget | apt-key add` +# step +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive + +# Installs LLVM toolchain, for Gandiva and testing other compilers +# +# Note that this is installed before the base packages to improve iteration +# while debugging package list with docker build. +ARG clang_tools +ARG llvm +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -ge 10 ]; then \ + echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + clang-${clang_tools} \ + clang-${llvm} \ + clang-format-${clang_tools} \ + clang-tidy-${clang_tools} \ + llvm-${llvm}-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Installs C++ toolchain and dependencies +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ca-certificates \ + ccache \ + cmake \ + g++ \ + gcc \ + gdb \ + git \ + libbenchmark-dev \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + rapidjson-dev \ + tzdata && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Prioritize system packages and local installation +# The following dependencies will be downloaded due to missing/invalid packages +# provided by the distribution: +# - libc-ares-dev does not install CMake config files +# - flatbuffer is not packaged +# - libgtest-dev only provide sources +# - libprotobuf-dev only provide sources +# - thrift is too old +# - utf8proc is too old(v2.1.0) +# - s3 tests would require boost-asio that is included since Boost 1.66.0 +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DATASET=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=ON \ + ARROW_HDFS=ON \ + ARROW_HOME=/usr/local \ + ARROW_INSTALL_NAME_RPATH=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_USE_ASAN=OFF \ + ARROW_USE_CCACHE=ON \ + ARROW_USE_TSAN=OFF \ + ARROW_USE_UBSAN=OFF \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + PARQUET_BUILD_EXAMPLES=ON \ + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + Thrift_SOURCE=BUNDLED \ + utf8proc_SOURCE=BUNDLED \ + zstd_SOURCE=BUNDLED diff --git a/src/arrow/ci/docker/ubuntu-18.04-csharp.dockerfile b/src/arrow/ci/docker/ubuntu-18.04-csharp.dockerfile new file mode 100644 index 000000000..624ce259d --- /dev/null +++ b/src/arrow/ci/docker/ubuntu-18.04-csharp.dockerfile @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG platform=bionic +ARG dotnet=3.1 +FROM mcr.microsoft.com/dotnet/core/sdk:${dotnet}-${platform} + +RUN dotnet tool install --tool-path /usr/local/bin sourcelink diff --git a/src/arrow/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile b/src/arrow/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile new file mode 100644 index 000000000..d7076b45b --- /dev/null +++ b/src/arrow/ci/docker/ubuntu-20.04-cpp-minimal.dockerfile @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base=amd64/ubuntu:20.04 +FROM ${base} +ARG arch + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +RUN apt-get update -y -q && \ + apt-get install -y -q \ + build-essential \ + ccache \ + cmake \ + git \ + libssl-dev \ + libcurl4-openssl-dev \ + python3-pip \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default + +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DATASET=ON \ + ARROW_FLIGHT=ON \ + ARROW_GANDIVA=OFF \ + ARROW_GCS=ON \ + ARROW_HDFS=ON \ + ARROW_HOME=/usr/local \ + ARROW_INSTALL_NAME_RPATH=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_S3=ON \ + ARROW_USE_CCACHE=ON \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + CMAKE_GENERATOR="Unix Makefiles" \ + PARQUET_BUILD_EXAMPLES=ON \ + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + PYTHON=python3 diff --git a/src/arrow/ci/docker/ubuntu-20.04-cpp.dockerfile b/src/arrow/ci/docker/ubuntu-20.04-cpp.dockerfile new file mode 100644 index 000000000..de872da9a --- /dev/null +++ b/src/arrow/ci/docker/ubuntu-20.04-cpp.dockerfile @@ -0,0 +1,143 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base=amd64/ubuntu:20.04 +FROM ${base} +ARG arch + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +# Installs LLVM toolchain, for Gandiva and testing other compilers +# +# Note that this is installed before the base packages to improve iteration +# while debugging package list with docker build. +ARG clang_tools +ARG llvm +RUN if [ "${llvm}" -gt "10" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ + echo "deb https://apt.llvm.org/focal/ llvm-toolchain-focal-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi \ + fi && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + clang-${clang_tools} \ + clang-${llvm} \ + clang-format-${clang_tools} \ + clang-tidy-${clang_tools} \ + llvm-${llvm}-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Installs C++ toolchain and dependencies +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ca-certificates \ + ccache \ + cmake \ + g++ \ + gcc \ + gdb \ + git \ + libbenchmark-dev \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + python3-pip \ + rapidjson-dev \ + tzdata \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default + +# Prioritize system packages and local installation +# The following dependencies will be downloaded due to missing/invalid packages +# provided by the distribution: +# - libc-ares-dev does not install CMake config files +# - flatbuffer is not packaged +# - libgtest-dev only provide sources +# - libprotobuf-dev only provide sources +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_DATASET=ON \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=ON \ + ARROW_GCS=ON \ + ARROW_HDFS=ON \ + ARROW_HOME=/usr/local \ + ARROW_INSTALL_NAME_RPATH=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_S3=ON \ + ARROW_USE_ASAN=OFF \ + ARROW_USE_CCACHE=ON \ + ARROW_USE_UBSAN=OFF \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \ + AWSSDK_SOURCE=BUNDLED \ + google_cloud_cpp_storage_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + gRPC_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + PARQUET_BUILD_EXAMPLES=ON \ + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + Protobuf_SOURCE=BUNDLED \ + PYTHON=python3 diff --git a/src/arrow/ci/docker/ubuntu-20.10-cpp.dockerfile b/src/arrow/ci/docker/ubuntu-20.10-cpp.dockerfile new file mode 100644 index 000000000..59f5fa4c8 --- /dev/null +++ b/src/arrow/ci/docker/ubuntu-20.10-cpp.dockerfile @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base=amd64/ubuntu:20.10 +FROM ${base} +ARG arch + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +# Installs LLVM toolchain, for Gandiva and testing other compilers +# +# Note that this is installed before the base packages to improve iteration +# while debugging package list with docker build. +ARG clang_tools +ARG llvm +RUN if [ "${llvm}" -gt "10" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ + echo "deb https://apt.llvm.org/groovy/ llvm-toolchain-groovy-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi \ + fi && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + clang-${clang_tools} \ + clang-${llvm} \ + clang-format-${clang_tools} \ + clang-tidy-${clang_tools} \ + llvm-${llvm}-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Installs C++ toolchain and dependencies +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ca-certificates \ + ccache \ + cmake \ + g++ \ + gcc \ + gdb \ + git \ + libbenchmark-dev \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + protobuf-compiler-grpc \ + python3-pip \ + rapidjson-dev \ + tzdata \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default + +# Prioritize system packages and local installation +# The following dependencies will be downloaded due to missing/invalid packages +# provided by the distribution: +# - libc-ares-dev does not install CMake config files +# - flatbuffer is not packaged +# - libgtest-dev only provide sources +# - libprotobuf-dev only provide sources +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_DATASET=ON \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=ON \ + ARROW_HDFS=ON \ + ARROW_HOME=/usr/local \ + ARROW_INSTALL_NAME_RPATH=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_S3=ON \ + ARROW_USE_ASAN=OFF \ + ARROW_USE_CCACHE=ON \ + ARROW_USE_UBSAN=OFF \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + PARQUET_BUILD_EXAMPLES=ON \ + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + PYTHON=python3 diff --git a/src/arrow/ci/docker/ubuntu-21.04-cpp.dockerfile b/src/arrow/ci/docker/ubuntu-21.04-cpp.dockerfile new file mode 100644 index 000000000..9fc857c62 --- /dev/null +++ b/src/arrow/ci/docker/ubuntu-21.04-cpp.dockerfile @@ -0,0 +1,162 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +ARG base=amd64/ubuntu:20.04 +FROM ${base} +ARG arch + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +RUN echo "debconf debconf/frontend select Noninteractive" | \ + debconf-set-selections + +# Installs LLVM toolchain, for Gandiva and testing other compilers +# +# Note that this is installed before the base packages to improve iteration +# while debugging package list with docker build. +ARG clang_tools +ARG llvm +RUN if [ "${llvm}" -gt "10" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + apt-transport-https \ + ca-certificates \ + gnupg \ + wget && \ + wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + echo "deb https://apt.llvm.org/hirsute/ llvm-toolchain-hirsute-${llvm} main" > \ + /etc/apt/sources.list.d/llvm.list && \ + if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \ + echo "deb https://apt.llvm.org/hirsute/ llvm-toolchain-hirsute-${clang_tools} main" > \ + /etc/apt/sources.list.d/clang-tools.list; \ + fi \ + fi && \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + clang-${clang_tools} \ + clang-${llvm} \ + clang-format-${clang_tools} \ + clang-tidy-${clang_tools} \ + llvm-${llvm}-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +# Installs C++ toolchain and dependencies +RUN apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + autoconf \ + ca-certificates \ + ccache \ + cmake \ + gdb \ + git \ + libbenchmark-dev \ + libboost-filesystem-dev \ + libboost-system-dev \ + libbrotli-dev \ + libbz2-dev \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libgflags-dev \ + libgoogle-glog-dev \ + libgrpc++-dev \ + liblz4-dev \ + libprotobuf-dev \ + libprotoc-dev \ + libre2-dev \ + libsnappy-dev \ + libssl-dev \ + libthrift-dev \ + libutf8proc-dev \ + libzstd-dev \ + make \ + ninja-build \ + pkg-config \ + protobuf-compiler \ + protobuf-compiler-grpc \ + python3-pip \ + rapidjson-dev \ + tzdata \ + wget && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists* + +COPY ci/scripts/install_minio.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local +COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/ +RUN /arrow/ci/scripts/install_gcs_testbench.sh ${arch} default + +# Prioritize system packages and local installation +# The following dependencies will be downloaded due to missing/invalid packages +# provided by the distribution: +# - libc-ares-dev does not install CMake config files +# - flatbuffer is not packaged +# - libgtest-dev only provide sources +# - libprotobuf-dev only provide sources +ENV ARROW_BUILD_TESTS=ON \ + ARROW_DEPENDENCY_SOURCE=SYSTEM \ + ARROW_DATASET=ON \ + ARROW_FLIGHT=OFF \ + ARROW_GANDIVA=ON \ + ARROW_HDFS=ON \ + ARROW_HOME=/usr/local \ + ARROW_INSTALL_NAME_RPATH=OFF \ + ARROW_NO_DEPRECATED_API=ON \ + ARROW_ORC=ON \ + ARROW_PARQUET=ON \ + ARROW_PLASMA=ON \ + ARROW_S3=ON \ + ARROW_USE_ASAN=OFF \ + ARROW_USE_CCACHE=ON \ + ARROW_USE_UBSAN=OFF \ + ARROW_WITH_BROTLI=ON \ + ARROW_WITH_BZ2=ON \ + ARROW_WITH_LZ4=ON \ + ARROW_WITH_SNAPPY=ON \ + ARROW_WITH_ZLIB=ON \ + ARROW_WITH_ZSTD=ON \ + AWSSDK_SOURCE=BUNDLED \ + GTest_SOURCE=BUNDLED \ + ORC_SOURCE=BUNDLED \ + PARQUET_BUILD_EXAMPLES=ON \ + PARQUET_BUILD_EXECUTABLES=ON \ + PATH=/usr/lib/ccache/:$PATH \ + PYTHON=python3 + +ARG gcc_version="" +RUN if [ "${gcc_version}" = "" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + g++ \ + gcc; \ + else \ + if [ "${gcc_version}" -gt "10" ]; then \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends software-properties-common && \ + add-apt-repository ppa:ubuntu-toolchain-r/volatile; \ + fi; \ + apt-get update -y -q && \ + apt-get install -y -q --no-install-recommends \ + g++-${gcc_version} \ + gcc-${gcc_version} && \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \ + update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \ + update-alternatives --set cc /usr/bin/gcc && \ + update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \ + update-alternatives --set c++ /usr/bin/g++; \ + fi |