diff options
Diffstat (limited to '')
| -rw-r--r-- | src/arrow/dev/tasks/linux-packages/Rakefile | 249 |
1 files changed, 249 insertions, 0 deletions
diff --git a/src/arrow/dev/tasks/linux-packages/Rakefile b/src/arrow/dev/tasks/linux-packages/Rakefile new file mode 100644 index 000000000..58cd981fe --- /dev/null +++ b/src/arrow/dev/tasks/linux-packages/Rakefile @@ -0,0 +1,249 @@ +# -*- ruby -*- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "English" + +require_relative "../../release/binary-task" +require_relative "helper" + +packages = [ + "apache-arrow", + "apache-arrow-apt-source", + "apache-arrow-release", +] + + +namespace :apt do + desc "Build deb packages" + task :build do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "apt:build") + end + end + end +end + +namespace :yum do + desc "Build RPM packages" + task :build do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "yum:build") + end + end + end +end + +namespace :version do + desc "Update versions" + task :update do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "version:update") + end + end + end +end + +namespace :docker do + desc "Pull built images" + task :pull do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "docker:pull") + end + end + end + + desc "Push built images" + task :push do + packages.each do |package| + cd(package) do + ruby("-S", "rake", "docker:push") + end + end + end +end + + +class LocalBinaryTask < BinaryTask + include Helper::ApacheArrow + + def initialize(packages) + @packages = packages + super() + end + + def define + define_apt_test_task + define_yum_test_task + end + + private + def latest_commit_time(git_directory) + cd(git_directory) do + return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc + end + end + + def version + @version ||= detect_version(detect_release_time) + end + + def resolve_docker_image(target) + image = "" + target = target.gsub(/\Aamazon-linux/, "amazonlinux") + case target + when /-(?:arm64|aarch64)\z/ + target = $PREMATCH + image << "arm64v8/" + end + image << target.gsub(/-/, ":") + end + + def verify(target) + verify_command_line = [ + "docker", + "run", + "--rm", + "--log-driver", "none", + "--volume", "#{File.expand_path(arrow_source_dir)}:/arrow:delegated", + ] + if $stdin.tty? + verify_command_line << "--interactive" + verify_command_line << "--tty" + else + verify_command_line.concat(["--attach", "STDOUT"]) + verify_command_line.concat(["--attach", "STDERR"]) + end + verify_command_line << resolve_docker_image(target) + case target + when /\Adebian-/, /\Aubuntu-/ + verify_command_line << "/arrow/dev/release/verify-apt.sh" + else + verify_command_line << "/arrow/dev/release/verify-yum.sh" + end + verify_command_line << version + verify_command_line << "local" + sh(*verify_command_line) + end + + def apt_test_targets + targets = (ENV["APT_TARGETS"] || "").split(",") + targets = apt_test_targets_default if targets.empty? + targets + end + + def apt_test_targets_default + # Disable arm64 targets by default for now + # because they require some setups on host. + [ + "debian-buster", + # "debian-buster-arm64", + "debian-bullseye", + # "debian-bullseye-arm64", + "ubuntu-xenial", + # "ubuntu-xenial-arm64", + "ubuntu-bionic", + # "ubuntu-bionic-arm64", + "ubuntu-focal", + # "ubuntu-focal-arm64", + "ubuntu-hirsute", + # "ubuntu-hirsute-arm64", + "ubuntu-impish", + # "ubuntu-impish-arm64", + ] + end + + def define_apt_test_task + namespace :apt do + desc "Test deb packages" + task :test do + repositories_dir = "apt/repositories" + rm_rf(repositories_dir) + @packages.each do |package| + package_repositories = "#{package}/apt/repositories" + next unless File.exist?(package_repositories) + sh("rsync", "-a", "#{package_repositories}/", repositories_dir) + end + Dir.glob("#{repositories_dir}/ubuntu/pool/*") do |code_name_dir| + universe_dir = "#{code_name_dir}/universe" + next unless File.exist?(universe_dir) + mv(universe_dir, "#{code_name_dir}/main") + end + base_dir = "nonexistent" + merged_dir = "apt/merged" + apt_update(base_dir, repositories_dir, merged_dir) + Dir.glob("#{merged_dir}/*/dists/*") do |dists_code_name_dir| + prefix = dists_code_name_dir.split("/")[-3..-1].join("/") + mv(Dir.glob("#{dists_code_name_dir}/*Release*"), + "#{repositories_dir}/#{prefix}") + end + apt_test_targets.each do |target| + verify(target) + end + end + end + end + + def yum_test_targets + targets = (ENV["YUM_TARGETS"] || "").split(",") + targets = yum_test_targets_default if targets.empty? + targets + end + + def yum_test_targets_default + # Disable aarch64 targets by default for now + # because they require some setups on host. + [ + "almalinux-8", + # "almalinux-8-aarch64", + "amazon-linux-2", + # "amazon-linux-2-aarch64", + "centos-7", + "centos-8", + # "centos-8-aarch64", + ] + end + + def define_yum_test_task + namespace :yum do + desc "Test RPM packages" + task :test do + repositories_dir = "yum/repositories" + rm_rf(repositories_dir) + @packages.each do |package| + package_repositories = "#{package}/yum/repositories" + next unless File.exist?(package_repositories) + sh("rsync", "-a", "#{package_repositories}/", repositories_dir) + end + rpm_sign(repositories_dir) + base_dir = "nonexistent" + yum_update(base_dir, repositories_dir) + yum_test_targets.each do |target| + verify(target) + end + end + end + end +end + +local_binary_task = LocalBinaryTask.new(packages) +local_binary_task.define |