# -*- ruby -*- # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. require "English" require_relative "../../release/binary-task" require_relative "helper" packages = [ "apache-arrow", "apache-arrow-apt-source", "apache-arrow-release", ] namespace :apt do desc "Build deb packages" task :build do packages.each do |package| cd(package) do ruby("-S", "rake", "apt:build") end end end end namespace :yum do desc "Build RPM packages" task :build do packages.each do |package| cd(package) do ruby("-S", "rake", "yum:build") end end end end namespace :version do desc "Update versions" task :update do packages.each do |package| cd(package) do ruby("-S", "rake", "version:update") end end end end namespace :docker do desc "Pull built images" task :pull do packages.each do |package| cd(package) do ruby("-S", "rake", "docker:pull") end end end desc "Push built images" task :push do packages.each do |package| cd(package) do ruby("-S", "rake", "docker:push") end end end end class LocalBinaryTask < BinaryTask include Helper::ApacheArrow def initialize(packages) @packages = packages super() end def define define_apt_test_task define_yum_test_task end private def latest_commit_time(git_directory) cd(git_directory) do return Time.iso8601(`git log -n 1 --format=%aI`.chomp).utc end end def version @version ||= detect_version(detect_release_time) end def resolve_docker_image(target) image = "" target = target.gsub(/\Aamazon-linux/, "amazonlinux") case target when /-(?:arm64|aarch64)\z/ target = $PREMATCH image << "arm64v8/" end image << target.gsub(/-/, ":") end def verify(target) verify_command_line = [ "docker", "run", "--rm", "--log-driver", "none", "--volume", "#{File.expand_path(arrow_source_dir)}:/arrow:delegated", ] if $stdin.tty? verify_command_line << "--interactive" verify_command_line << "--tty" else verify_command_line.concat(["--attach", "STDOUT"]) verify_command_line.concat(["--attach", "STDERR"]) end verify_command_line << resolve_docker_image(target) case target when /\Adebian-/, /\Aubuntu-/ verify_command_line << "/arrow/dev/release/verify-apt.sh" else verify_command_line << "/arrow/dev/release/verify-yum.sh" end verify_command_line << version verify_command_line << "local" sh(*verify_command_line) end def apt_test_targets targets = (ENV["APT_TARGETS"] || "").split(",") targets = apt_test_targets_default if targets.empty? targets end def apt_test_targets_default # Disable arm64 targets by default for now # because they require some setups on host. [ "debian-buster", # "debian-buster-arm64", "debian-bullseye", # "debian-bullseye-arm64", "ubuntu-xenial", # "ubuntu-xenial-arm64", "ubuntu-bionic", # "ubuntu-bionic-arm64", "ubuntu-focal", # "ubuntu-focal-arm64", "ubuntu-hirsute", # "ubuntu-hirsute-arm64", "ubuntu-impish", # "ubuntu-impish-arm64", ] end def define_apt_test_task namespace :apt do desc "Test deb packages" task :test do repositories_dir = "apt/repositories" rm_rf(repositories_dir) @packages.each do |package| package_repositories = "#{package}/apt/repositories" next unless File.exist?(package_repositories) sh("rsync", "-a", "#{package_repositories}/", repositories_dir) end Dir.glob("#{repositories_dir}/ubuntu/pool/*") do |code_name_dir| universe_dir = "#{code_name_dir}/universe" next unless File.exist?(universe_dir) mv(universe_dir, "#{code_name_dir}/main") end base_dir = "nonexistent" merged_dir = "apt/merged" apt_update(base_dir, repositories_dir, merged_dir) Dir.glob("#{merged_dir}/*/dists/*") do |dists_code_name_dir| prefix = dists_code_name_dir.split("/")[-3..-1].join("/") mv(Dir.glob("#{dists_code_name_dir}/*Release*"), "#{repositories_dir}/#{prefix}") end apt_test_targets.each do |target| verify(target) end end end end def yum_test_targets targets = (ENV["YUM_TARGETS"] || "").split(",") targets = yum_test_targets_default if targets.empty? targets end def yum_test_targets_default # Disable aarch64 targets by default for now # because they require some setups on host. [ "almalinux-8", # "almalinux-8-aarch64", "amazon-linux-2", # "amazon-linux-2-aarch64", "centos-7", "centos-8", # "centos-8-aarch64", ] end def define_yum_test_task namespace :yum do desc "Test RPM packages" task :test do repositories_dir = "yum/repositories" rm_rf(repositories_dir) @packages.each do |package| package_repositories = "#{package}/yum/repositories" next unless File.exist?(package_repositories) sh("rsync", "-a", "#{package_repositories}/", repositories_dir) end rpm_sign(repositories_dir) base_dir = "nonexistent" yum_update(base_dir, repositories_dir) yum_test_targets.each do |target| verify(target) end end end end end local_binary_task = LocalBinaryTask.new(packages) local_binary_task.define