241 lines
8 KiB
Python
241 lines
8 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
"""
|
|
Support for running spidermonkey jobs via dedicated scripts
|
|
"""
|
|
|
|
|
|
import os
|
|
import re
|
|
|
|
import taskgraph
|
|
from taskgraph.util.schema import Schema
|
|
from taskgraph.util.taskcluster import get_root_url
|
|
from voluptuous import Any, Optional, Required
|
|
|
|
from gecko_taskgraph import GECKO
|
|
from gecko_taskgraph.transforms.job import run_job_using
|
|
from gecko_taskgraph.transforms.job.common import add_artifacts
|
|
from gecko_taskgraph.util.hash import hash_path
|
|
|
|
DSC_PACKAGE_RE = re.compile(".*(?=_)")
|
|
SOURCE_PACKAGE_RE = re.compile(r".*(?=[-_]\d)")
|
|
|
|
source_definition = {
|
|
Required("url"): str,
|
|
Required("sha256"): str,
|
|
}
|
|
|
|
common_schema = Schema(
|
|
{
|
|
# URL/SHA256 of a source file to build, which can either be a source
|
|
# control (.dsc), or a tarball.
|
|
Required(Any("dsc", "tarball")): source_definition,
|
|
# Package name. Normally derived from the source control or tarball file
|
|
# name. Use in case the name doesn't match DSC_PACKAGE_RE or
|
|
# SOURCE_PACKAGE_RE.
|
|
Optional("name"): str,
|
|
# Patch to apply to the extracted source.
|
|
Optional("patch"): str,
|
|
# Command to run before dpkg-buildpackage.
|
|
Optional("pre-build-command"): str,
|
|
# Architecture to build the package for.
|
|
Optional("arch"): str,
|
|
# List of package tasks to get build dependencies from.
|
|
Optional("packages"): [str],
|
|
# What resolver to use to install build dependencies. The default
|
|
# (apt-get) is good in most cases, but in subtle cases involving
|
|
# a *-backports archive, its solver might not be able to find a
|
|
# solution that satisfies the build dependencies.
|
|
Optional("resolver"): Any("apt-get", "aptitude"),
|
|
# Base work directory used to set up the task.
|
|
Required("workdir"): str,
|
|
}
|
|
)
|
|
|
|
debian_schema = common_schema.extend(
|
|
{
|
|
Required("using"): "debian-package",
|
|
# Debian distribution
|
|
Required("dist"): str,
|
|
}
|
|
)
|
|
|
|
ubuntu_schema = common_schema.extend(
|
|
{
|
|
Required("using"): "ubuntu-package",
|
|
# Ubuntu distribution
|
|
Required("dist"): str,
|
|
}
|
|
)
|
|
|
|
|
|
def common_package(config, job, taskdesc, distro, version):
|
|
run = job["run"]
|
|
|
|
name = taskdesc["label"].replace(f"{config.kind}-", "", 1)
|
|
|
|
arch = run.get("arch", "amd64")
|
|
|
|
worker = taskdesc["worker"]
|
|
worker.setdefault("artifacts", [])
|
|
|
|
image = "%s%d" % (distro, version)
|
|
if arch != "amd64":
|
|
image += "-" + arch
|
|
image += "-packages"
|
|
worker["docker-image"] = {"in-tree": image}
|
|
|
|
add_artifacts(config, job, taskdesc, path="/tmp/artifacts")
|
|
|
|
env = worker.setdefault("env", {})
|
|
env["DEBFULLNAME"] = "Mozilla build team"
|
|
env["DEBEMAIL"] = "dev-builds@lists.mozilla.org"
|
|
|
|
if "dsc" in run:
|
|
src = run["dsc"]
|
|
unpack = "dpkg-source -x {src_file} {package}"
|
|
package_re = DSC_PACKAGE_RE
|
|
elif "tarball" in run:
|
|
src = run["tarball"]
|
|
unpack = (
|
|
"mkdir {package} && "
|
|
"tar -C {package} -axf {src_file} --strip-components=1"
|
|
)
|
|
package_re = SOURCE_PACKAGE_RE
|
|
else:
|
|
raise RuntimeError("Unreachable")
|
|
src_url = src["url"]
|
|
src_file = os.path.basename(src_url)
|
|
src_sha256 = src["sha256"]
|
|
package = run.get("name")
|
|
if not package:
|
|
package = package_re.match(src_file).group(0)
|
|
unpack = unpack.format(src_file=src_file, package=package)
|
|
|
|
resolver = run.get("resolver", "apt-get")
|
|
if resolver == "apt-get":
|
|
resolver = "apt-get -yyq --no-install-recommends"
|
|
elif resolver == "aptitude":
|
|
resolver = (
|
|
"aptitude -y --without-recommends -o "
|
|
"Aptitude::ProblemResolver::Hints::KeepBuildDeps="
|
|
f'"reject {package}-build-deps :UNINST"'
|
|
)
|
|
else:
|
|
raise RuntimeError("Unreachable")
|
|
|
|
adjust = ""
|
|
if "patch" in run:
|
|
# We don't use robustcheckout or run-task to get a checkout. So for
|
|
# this one file we'd need from a checkout, download it.
|
|
env["PATCH_URL"] = config.params.file_url(
|
|
"build/debian-packages/{patch}".format(patch=run["patch"]),
|
|
)
|
|
adjust += "curl -sL $PATCH_URL | patch -p1 && "
|
|
if "pre-build-command" in run:
|
|
adjust += run["pre-build-command"] + " && "
|
|
if "tarball" in run:
|
|
adjust += "mv ../{src_file} ../{package}_{ver}.orig.tar.gz && ".format(
|
|
src_file=src_file,
|
|
package=package,
|
|
ver="$(dpkg-parsechangelog | awk '$1==\"Version:\"{print $2}' | cut -f 1 -d -)",
|
|
)
|
|
if "patch" not in run and "pre-build-command" not in run:
|
|
adjust += (
|
|
'debchange -l ".{prefix}moz" --distribution "{dist}"'
|
|
' "Mozilla backport for {dist}." < /dev/null && '
|
|
).format(
|
|
prefix=name.split("-", 1)[0],
|
|
dist=run["dist"],
|
|
)
|
|
|
|
worker["command"] = [
|
|
"sh",
|
|
"-x",
|
|
"-c",
|
|
# Add sources for packages coming from other package tasks.
|
|
"/usr/local/sbin/setup_packages.sh {root_url} $PACKAGES && "
|
|
"apt-get update && "
|
|
# Upgrade packages that might have new versions in package tasks.
|
|
"apt-get dist-upgrade && " "cd /tmp && "
|
|
# Get, validate and extract the package source.
|
|
"(dget -d -u {src_url} || exit 100) && "
|
|
'echo "{src_sha256} {src_file}" | sha256sum -c && '
|
|
"{unpack} && "
|
|
"cd {package} && "
|
|
# Optionally apply patch and/or pre-build command.
|
|
"{adjust}"
|
|
# Install the necessary build dependencies.
|
|
"(cd ..; mk-build-deps -i -r {package}/debian/control -t '{resolver}' || exit 100) && "
|
|
# Build the package
|
|
'DEB_BUILD_OPTIONS="parallel=$(nproc) nocheck" dpkg-buildpackage -sa && '
|
|
# Copy the artifacts
|
|
"mkdir -p {artifacts}/apt && "
|
|
"dcmd cp ../{package}_*.changes {artifacts}/apt/ && "
|
|
"cd {artifacts} && "
|
|
# Make the artifacts directory usable as an APT repository.
|
|
"apt-ftparchive sources apt | gzip -c9 > apt/Sources.gz && "
|
|
"apt-ftparchive packages apt | gzip -c9 > apt/Packages.gz".format(
|
|
root_url=get_root_url(False),
|
|
package=package,
|
|
src_url=src_url,
|
|
src_file=src_file,
|
|
src_sha256=src_sha256,
|
|
unpack=unpack,
|
|
adjust=adjust,
|
|
artifacts="/tmp/artifacts",
|
|
resolver=resolver,
|
|
),
|
|
]
|
|
|
|
if run.get("packages"):
|
|
env = worker.setdefault("env", {})
|
|
env["PACKAGES"] = {
|
|
"task-reference": " ".join(f"<{p}>" for p in run["packages"])
|
|
}
|
|
deps = taskdesc.setdefault("dependencies", {})
|
|
for p in run["packages"]:
|
|
deps[p] = f"packages-{p}"
|
|
|
|
# Use the command generated above as the base for the index hash.
|
|
# We rely on it not varying depending on the head_repository or head_rev.
|
|
digest_data = list(worker["command"])
|
|
if "patch" in run:
|
|
digest_data.append(
|
|
hash_path(os.path.join(GECKO, "build", "debian-packages", run["patch"]))
|
|
)
|
|
|
|
if not taskgraph.fast:
|
|
taskdesc["cache"] = {
|
|
"type": "packages.v1",
|
|
"name": name,
|
|
"digest-data": digest_data,
|
|
}
|
|
|
|
|
|
@run_job_using("docker-worker", "debian-package", schema=debian_schema)
|
|
def docker_worker_debian_package(config, job, taskdesc):
|
|
run = job["run"]
|
|
version = {
|
|
"wheezy": 7,
|
|
"jessie": 8,
|
|
"stretch": 9,
|
|
"buster": 10,
|
|
"bullseye": 11,
|
|
"bookworm": 12,
|
|
}[run["dist"]]
|
|
common_package(config, job, taskdesc, "debian", version)
|
|
|
|
|
|
@run_job_using("docker-worker", "ubuntu-package", schema=ubuntu_schema)
|
|
def docker_worker_ubuntu_package(config, job, taskdesc):
|
|
run = job["run"]
|
|
version = {
|
|
"bionic": 1804,
|
|
"focal": 2004,
|
|
"jammy": 2204,
|
|
"noble": 2404,
|
|
}[run["dist"]]
|
|
common_package(config, job, taskdesc, "ubuntu", version)
|