summaryrefslogtreecommitdiffstats
path: root/taskcluster/scripts/misc/repack_rust.py
diff options
context:
space:
mode:
Diffstat (limited to '')
-rwxr-xr-xtaskcluster/scripts/misc/repack_rust.py647
1 files changed, 647 insertions, 0 deletions
diff --git a/taskcluster/scripts/misc/repack_rust.py b/taskcluster/scripts/misc/repack_rust.py
new file mode 100755
index 0000000000..909ff379dc
--- /dev/null
+++ b/taskcluster/scripts/misc/repack_rust.py
@@ -0,0 +1,647 @@
+#!/usr/bin/env python3
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+This script downloads and repacks official rust language builds
+with the necessary tool and target support for the Firefox
+build environment.
+"""
+
+import argparse
+import errno
+import hashlib
+import os
+import shutil
+import subprocess
+import tarfile
+import textwrap
+from contextlib import contextmanager
+
+import requests
+import toml
+import zstandard
+
+
+def log(msg):
+ print("repack: %s" % msg, flush=True)
+
+
+def fetch_file(url):
+ """Download a file from the given url if it's not already present.
+
+ Returns the SHA-2 256-bit hash of the received file."""
+ filename = os.path.basename(url)
+ sha = hashlib.sha256()
+ size = 4096
+ if os.path.exists(filename):
+ with open(filename, "rb") as fd:
+ while True:
+ block = fd.read(size)
+ if not block:
+ return sha.hexdigest()
+ sha.update(block)
+ log("Could not calculate checksum!")
+ return None
+ r = requests.get(url, stream=True)
+ r.raise_for_status()
+ with open(filename, "wb") as fd:
+ for chunk in r.iter_content(size):
+ fd.write(chunk)
+ sha.update(chunk)
+ return sha.hexdigest()
+
+
+def check_call_with_input(cmd, input_data):
+ """Invoke a command, passing the input String over stdin.
+
+ This is like subprocess.check_call, but allows piping
+ input to interactive commands."""
+ p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
+ p.communicate(input_data)
+ if p.wait():
+ raise subprocess.CalledProcessError(p.returncode, cmd)
+
+
+def setup_gpg():
+ """Add the signing key to the current gpg config.
+
+ Import a hard-coded copy of the release signing public key
+ and mark it trusted in the gpg database so subsequent
+ signature checks can succeed or fail cleanly."""
+ keyid = "0x85AB96E6FA1BE5FE"
+ log("Importing signing key %s..." % keyid)
+ key = b"""
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQINBFJEwMkBEADlPACa2K7reD4x5zd8afKx75QYKmxqZwywRbgeICeD4bKiQoJZ
+dUjmn1LgrGaXuBMKXJQhyA34e/1YZel/8et+HPE5XpljBfNYXWbVocE1UMUTnFU9
+CKXa4AhJ33f7we2/QmNRMUifw5adPwGMg4D8cDKXk02NdnqQlmFByv0vSaArR5kn
+gZKnLY6o0zZ9Buyy761Im/ShXqv4ATUgYiFc48z33G4j+BDmn0ryGr1aFdP58tHp
+gjWtLZs0iWeFNRDYDje6ODyu/MjOyuAWb2pYDH47Xu7XedMZzenH2TLM9yt/hyOV
+xReDPhvoGkaO8xqHioJMoPQi1gBjuBeewmFyTSPS4deASukhCFOcTsw/enzJagiS
+ZAq6Imehduke+peAL1z4PuRmzDPO2LPhVS7CDXtuKAYqUV2YakTq8MZUempVhw5n
+LqVaJ5/XiyOcv405PnkT25eIVVVghxAgyz6bOU/UMjGQYlkUxI7YZ9tdreLlFyPR
+OUL30E8q/aCd4PGJV24yJ1uit+yS8xjyUiMKm4J7oMP2XdBN98TUfLGw7SKeAxyU
+92BHlxg7yyPfI4TglsCzoSgEIV6xoGOVRRCYlGzSjUfz0bCMCclhTQRBkegKcjB3
+sMTyG3SPZbjTlCqrFHy13e6hGl37Nhs8/MvXUysq2cluEISn5bivTKEeeQARAQAB
+tERSdXN0IExhbmd1YWdlIChUYWcgYW5kIFJlbGVhc2UgU2lnbmluZyBLZXkpIDxy
+dXN0LWtleUBydXN0LWxhbmcub3JnPokCOAQTAQIAIgUCUkTAyQIbAwYLCQgHAwIG
+FQgCCQoLBBYCAwECHgECF4AACgkQhauW5vob5f5fYQ//b1DWK1NSGx5nZ3zYZeHJ
+9mwGCftIaA2IRghAGrNf4Y8DaPqR+w1OdIegWn8kCoGfPfGAVW5XXJg+Oxk6QIaD
+2hJojBUrq1DALeCZVewzTVw6BN4DGuUexsc53a8DcY2Yk5WE3ll6UKq/YPiWiPNX
+9r8FE2MJwMABB6mWZLqJeg4RCrriBiCG26NZxGE7RTtPHyppoVxWKAFDiWyNdJ+3
+UnjldWrT9xFqjqfXWw9Bhz8/EoaGeSSbMIAQDkQQpp1SWpljpgqvctZlc5fHhsG6
+lmzW5RM4NG8OKvq3UrBihvgzwrIfoEDKpXbk3DXqaSs1o81NH5ftVWWbJp/ywM9Q
+uMC6n0YWiMZMQ1cFBy7tukpMkd+VPbPkiSwBhPkfZIzUAWd74nanN5SKBtcnymgJ
++OJcxfZLiUkXRj0aUT1GLA9/7wnikhJI+RvwRfHBgrssXBKNPOfXGWajtIAmZc2t
+kR1E8zjBVLId7r5M8g52HKk+J+y5fVgJY91nxG0zf782JjtYuz9+knQd55JLFJCO
+hhbv3uRvhvkqgauHagR5X9vCMtcvqDseK7LXrRaOdOUDrK/Zg/abi5d+NIyZfEt/
+ObFsv3idAIe/zpU6xa1nYNe3+Ixlb6mlZm3WCWGxWe+GvNW/kq36jZ/v/8pYMyVO
+p/kJqnf9y4dbufuYBg+RLqC5Ag0EUkTAyQEQANxy2tTSeRspfrpBk9+ju+KZ3zc4
+umaIsEa5DxJ2zIKHywVAR67Um0K1YRG07/F5+tD9TIRkdx2pcmpjmSQzqdk3zqa9
+2Zzeijjz2RNyBY8qYmyE08IncjTsFFB8OnvdXcsAgjCFmI1BKnePxrABL/2k8X18
+aysPb0beWqQVsi5FsSpAHu6k1kaLKc+130x6Hf/YJAjeo+S7HeU5NeOz3zD+h5bA
+Q25qMiVHX3FwH7rFKZtFFog9Ogjzi0TkDKKxoeFKyADfIdteJWFjOlCI9KoIhfXq
+Et9JMnxApGqsJElJtfQjIdhMN4Lnep2WkudHAfwJ/412fe7wiW0rcBMvr/BlBGRY
+vM4sTgN058EwIuY9Qmc8RK4gbBf6GsfGNJjWozJ5XmXElmkQCAvbQFoAfi5TGfVb
+77QQrhrQlSpfIYrvfpvjYoqj618SbU6uBhzh758gLllmMB8LOhxWtq9eyn1rMWyR
+KL1fEkfvvMc78zP+Px6yDMa6UIez8jZXQ87Zou9EriLbzF4QfIYAqR9LUSMnLk6K
+o61tSFmFEDobC3tc1jkSg4zZe/wxskn96KOlmnxgMGO0vJ7ASrynoxEnQE8k3WwA
++/YJDwboIR7zDwTy3Jw3mn1FgnH+c7Rb9h9geOzxKYINBFz5Hd0MKx7kZ1U6WobW
+KiYYxcCmoEeguSPHABEBAAGJAh8EGAECAAkFAlJEwMkCGwwACgkQhauW5vob5f7f
+FA//Ra+itJF4NsEyyhx4xYDOPq4uj0VWVjLdabDvFjQtbBLwIyh2bm8uO3AY4r/r
+rM5WWQ8oIXQ2vvXpAQO9g8iNlFez6OLzbfdSG80AG74pQqVVVyCQxD7FanB/KGge
+tAoOstFxaCAg4nxFlarMctFqOOXCFkylWl504JVIOvgbbbyj6I7qCUmbmqazBSMU
+K8c/Nz+FNu2Uf/lYWOeGogRSBgS0CVBcbmPUpnDHLxZWNXDWQOCxbhA1Uf58hcyu
+036kkiWHh2OGgJqlo2WIraPXx1cGw1Ey+U6exbtrZfE5kM9pZzRG7ZY83CXpYWMp
+kyVXNWmf9JcIWWBrXvJmMi0FDvtgg3Pt1tnoxqdilk6yhieFc8LqBn6CZgFUBk0t
+NSaWk3PsN0N6Ut8VXY6sai7MJ0Gih1gE1xadWj2zfZ9sLGyt2jZ6wK++U881YeXA
+ryaGKJ8sIs182hwQb4qN7eiUHzLtIh8oVBHo8Q4BJSat88E5/gOD6IQIpxc42iRL
+T+oNZw1hdwNyPOT1GMkkn86l3o7klwmQUWCPm6vl1aHp3omo+GHC63PpNFO5RncJ
+Ilo3aBKKmoE5lDSMGE8KFso5awTo9z9QnVPkRsk6qeBYit9xE3x3S+iwjcSg0nie
+aAkc0N00nc9V9jfPvt4z/5A5vjHh+NhFwH5h2vBJVPdsz6m5Ag0EVI9keAEQAL3R
+oVsHncJTmjHfBOV4JJsvCum4DuJDZ/rDdxauGcjMUWZaG338ZehnDqG1Yn/ys7zE
+aKYUmqyT+XP+M2IAQRTyxwlU1RsDlemQfWrESfZQCCmbnFScL0E7cBzy4xvtInQe
+UaFgJZ1BmxbzQrx+eBBdOTDv7RLnNVygRmMzmkDhxO1IGEu1+3ETIg/DxFE7VQY0
+It/Ywz+nHu1o4Hemc/GdKxu9hcYvcRVc/Xhueq/zcIM96l0m+CFbs0HMKCj8dgMe
+Ng6pbbDjNM+cV+5BgpRdIpE2l9W7ImpbLihqcZt47J6oWt/RDRVoKOzRxjhULVyV
+2VP9ESr48HnbvxcpvUAEDCQUhsGpur4EKHFJ9AmQ4zf91gWLrDc6QmlACn9o9ARU
+fOV5aFsZI9ni1MJEInJTP37stz/uDECRie4LTL4O6P4Dkto8ROM2wzZq5CiRNfnT
+PP7ARfxlCkpg+gpLYRlxGUvRn6EeYwDtiMQJUQPfpGHSvThUlgDEsDrpp4SQSmdA
+CB+rvaRqCawWKoXs0In/9wylGorRUupeqGC0I0/rh+f5mayFvORzwy/4KK4QIEV9
+aYTXTvSRl35MevfXU1Cumlaqle6SDkLr3ZnFQgJBqap0Y+Nmmz2HfO/pohsbtHPX
+92SN3dKqaoSBvzNGY5WT3CsqxDtik37kR3f9/DHpABEBAAGJBD4EGAECAAkFAlSP
+ZHgCGwICKQkQhauW5vob5f7BXSAEGQECAAYFAlSPZHgACgkQXLSpNHs7CdwemA/+
+KFoGuFqU0uKT9qblN4ugRyil5itmTRVffl4tm5OoWkW8uDnu7Ue3vzdzy+9NV8X2
+wRG835qjXijWP++AGuxgW6LB9nV5OWiKMCHOWnUjJQ6pNQMAgSN69QzkFXVF/q5f
+bkma9TgSbwjrVMyPzLSRwq7HsT3V02Qfr4cyq39QeILGy/NHW5z6LZnBy3BaVSd0
+lGjCEc3yfH5OaB79na4W86WCV5n4IT7cojFM+LdL6P46RgmEtWSG3/CDjnJl6BLR
+WqatRNBWLIMKMpn+YvOOL9TwuP1xbqWr1vZ66wksm53NIDcWhptpp0KEuzbU0/Dt
+OltBhcX8tOmO36LrSadX9rwckSETCVYklmpAHNxPml011YNDThtBidvsicw1vZwR
+HsXn+txlL6RAIRN+J/Rw3uOiJAqN9Qgedpx2q+E15t8MiTg/FXtB9SysnskFT/BH
+z0USNKJUY0btZBw3eXWzUnZf59D8VW1M/9JwznCHAx0c9wy/gRDiwt9w4RoXryJD
+VAwZg8rwByjldoiThUJhkCYvJ0R3xH3kPnPlGXDW49E9R8C2umRC3cYOL4U9dOQ1
+5hSlYydF5urFGCLIvodtE9q80uhpyt8L/5jj9tbwZWv6JLnfBquZSnCGqFZRfXlb
+Jphk9+CBQWwiZSRLZRzqQ4ffl4xyLuolx01PMaatkQbRaw/+JpgRNlurKQ0PsTrO
+8tztO/tpBBj/huc2DGkSwEWvkfWElS5RLDKdoMVs/j5CLYUJzZVikUJRm7m7b+OA
+P3W1nbDhuID+XV1CSBmGifQwpoPTys21stTIGLgznJrIfE5moFviOLqD/LrcYlsq
+CQg0yleu7SjOs//8dM3mC2FyLaE/dCZ8l2DCLhHw0+ynyRAvSK6aGCmZz6jMjmYF
+MXgiy7zESksMnVFMulIJJhR3eB0wx2GitibjY/ZhQ7tD3i0yy9ILR07dFz4pgkVM
+afxpVR7fmrMZ0t+yENd+9qzyAZs0ksxORoc2ze90SCx2jwEX/3K+m4I0hP2H/w5W
+gqdvuRLiqf+4BGW4zqWkLLlNIe/okt0r82SwHtDN0Ui1asmZTGj6sm8SXtwx+5cE
+38MttWqjDiibQOSthRVcETByRYM8KcjYSUCi4PoBc3NpDONkFbZm6XofR/f5mTcl
+2jDw6fIeVc4Hd1jBGajNzEqtneqqbdAkPQaLsuD2TMkQfTDJfE/IljwjrhDa9Mi+
+odtnMWq8vlwOZZ24/8/BNK5qXuCYL67O7AJB4ZQ6BT+g4z96iRLbupzu/XJyXkQF
+rOY/Ghegvn7fDrnt2KC9MpgeFBXzUp+k5rzUdF8jbCx5apVjA1sWXB9Kh3L+DUwF
+Mve696B5tlHyc1KxjHR6w9GRsh4=
+=5FXw
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+ check_call_with_input(["gpg", "--import"], key)
+ check_call_with_input(
+ ["gpg", "--command-fd", "0", "--edit-key", keyid], b"trust\n5\ny\n"
+ )
+
+
+def verify_sha(filename, sha):
+ """Verify that the checksum file matches the given sha digest."""
+ sha_filename = filename + ".sha256"
+ with open(sha_filename) as f:
+ # Older sha256 files would contain `sha filename`, but more recent
+ # ones only contain `sha`.
+ checksum = f.readline().split()[0]
+ if checksum != sha:
+ raise ValueError("Checksum mismatch in %s" % filename)
+ return True
+ log("No checksum file for %s!" % filename)
+ return False
+
+
+def fetch(url, validate=True):
+ """Download and verify a package url."""
+ base = os.path.basename(url)
+ log("Fetching %s..." % base)
+ if validate:
+ fetch_file(url + ".asc")
+ fetch_file(url + ".sha256")
+ sha = fetch_file(url)
+ if validate:
+ log("Verifying %s..." % base)
+ verify_sha(base, sha)
+ subprocess.check_call(
+ ["gpg", "--keyid-format", "0xlong", "--verify", base + ".asc", base]
+ )
+ return sha
+
+
+def install(filename, target):
+ """Run a package's installer script against the given target directory."""
+ log("Unpacking %s..." % filename)
+ subprocess.check_call(["tar", "xf", filename])
+ basename = filename.split(".tar")[0]
+ log("Installing %s..." % basename)
+ install_cmd = [os.path.join(basename, "install.sh")]
+ install_cmd += ["--prefix=" + os.path.abspath(target)]
+ install_cmd += ["--disable-ldconfig"]
+ subprocess.check_call(install_cmd)
+ log("Cleaning %s..." % basename)
+ shutil.rmtree(basename)
+
+
+def package(manifest, pkg, target):
+ """Pull out the package dict for a particular package and target
+ from the given manifest."""
+ version = manifest["pkg"][pkg]["version"]
+ if target in manifest["pkg"][pkg]["target"]:
+ info = manifest["pkg"][pkg]["target"][target]
+ else:
+ # rust-src is the same for all targets, and has a literal '*' in the
+ # section key/name instead of a target
+ info = manifest["pkg"][pkg]["target"]["*"]
+ if "xz_url" in info:
+ info["url"] = info.pop("xz_url")
+ info["hash"] = info.pop("xz_hash")
+ return (version, info)
+
+
+def fetch_package(manifest, pkg, host):
+ version, info = package(manifest, pkg, host)
+ if not info["available"]:
+ log("%s marked unavailable for %s" % (pkg, host))
+ raise KeyError
+
+ log("%s %s\n %s\n %s" % (pkg, version, info["url"], info["hash"]))
+ sha = fetch(info["url"], info["hash"] is not None)
+ if info["hash"] and sha != info["hash"]:
+ log(
+ "Checksum mismatch: package resource is different from manifest"
+ "\n %s" % sha
+ )
+ raise AssertionError
+ return info
+
+
+def fetch_std(manifest, targets):
+ stds = []
+ for target in targets:
+ stds.append(fetch_package(manifest, "rust-std", target))
+ # not available for i686
+ if target != "i686-unknown-linux-musl":
+ stds.append(fetch_package(manifest, "rust-analysis", target))
+ return stds
+
+
+def fetch_optional(manifest, pkg, host):
+ try:
+ return fetch_package(manifest, pkg, host)
+ except KeyError:
+ # The package is not available, oh well!
+ return None
+
+
+@contextmanager
+def chdir(path):
+ d = os.getcwd()
+ log('cd "%s"' % path)
+ os.chdir(path)
+ try:
+ yield
+ finally:
+ log('cd "%s"' % d)
+ os.chdir(d)
+
+
+def build_tar_package(name, base, directory):
+ name = os.path.realpath(name)
+ log("tarring {} from {}/{}".format(name, base, directory))
+ assert name.endswith(".tar.zst")
+
+ cctx = zstandard.ZstdCompressor()
+ with open(name, "wb") as f, cctx.stream_writer(f) as z:
+ with tarfile.open(mode="w|", fileobj=z) as tf:
+ with chdir(base):
+ tf.add(directory)
+
+
+def fetch_manifest(channel="stable", host=None, targets=()):
+ if channel.startswith("bors-"):
+ assert host
+ rev = channel[len("bors-") :]
+ base_url = "https://s3-us-west-1.amazonaws.com/rust-lang-ci2/rustc-builds"
+ manifest = {
+ "date": "some date",
+ "pkg": {},
+ }
+
+ def target(url):
+ return {
+ "url": url,
+ "hash": None,
+ "available": requests.head(url).status_code == 200,
+ }
+
+ for pkg in (
+ "cargo",
+ "rustc",
+ "rustfmt-preview",
+ "clippy-preview",
+ "rust-analyzer-preview",
+ ):
+ manifest["pkg"][pkg] = {
+ "version": "bors",
+ "target": {
+ host: target(
+ "{}/{}/{}-nightly-{}.tar.xz".format(base_url, rev, pkg, host)
+ ),
+ },
+ }
+ manifest["pkg"]["rust-src"] = {
+ "version": "bors",
+ "target": {
+ "*": target("{}/{}/rust-src-nightly.tar.xz".format(base_url, rev)),
+ },
+ }
+ for pkg in ("rust-std", "rust-analysis"):
+ manifest["pkg"][pkg] = {
+ "version": "bors",
+ "target": {
+ t: target(
+ "{}/{}/{}-nightly-{}.tar.xz".format(base_url, rev, pkg, t)
+ )
+ for t in sorted(set(targets) | set([host]))
+ },
+ }
+ return manifest
+ if "-" in channel:
+ channel, date = channel.split("-", 1)
+ prefix = "/" + date
+ else:
+ prefix = ""
+ url = "https://static.rust-lang.org/dist%s/channel-rust-%s.toml" % (prefix, channel)
+ req = requests.get(url)
+ req.raise_for_status()
+ manifest = toml.loads(req.text)
+ if manifest["manifest-version"] != "2":
+ raise NotImplementedError(
+ "Unrecognized manifest version %s." % manifest["manifest-version"]
+ )
+ return manifest
+
+
+def patch_src(patch, module):
+ log("Patching Rust src... {} with {}".format(module, patch))
+ patch = os.path.realpath(patch)
+ subprocess.check_call(["patch", "-d", module, "-p1", "-i", patch, "--fuzz=0", "-s"])
+
+
+def build_src(install_dir, host, targets, patches):
+ install_dir = os.path.abspath(install_dir)
+ fetches = os.environ["MOZ_FETCHES_DIR"]
+ rust_dir = os.path.join(fetches, "rust")
+ patch_dir = os.path.join(os.environ["GECKO_PATH"], "build", "build-rust")
+
+ # Clear and remake any previous install directory.
+ try:
+ shutil.rmtree(install_dir)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ os.makedirs(install_dir)
+
+ # Patch the src (see the --patch flag's description for details)
+ for p in patches:
+ module, colon, file = p.partition(":")
+ if not colon:
+ module, file = "", p
+ patch_file = os.path.join(patch_dir, file)
+ patch_module = os.path.join(rust_dir, module)
+ patch_src(patch_file, patch_module)
+
+ log("Building Rust...")
+
+ example_config = ""
+ for example_toml in ("config.example.toml", "config.toml.example"):
+ path = os.path.join(rust_dir, example_toml)
+ if os.path.exists(path):
+ with open(path) as file:
+ example_config = file.read()
+ break
+
+ if "ignore-git" in example_config:
+ omit_git_hash = "ignore-git"
+ else:
+ omit_git_hash = "omit-git-hash"
+
+ # Rust builds are configured primarily through a config.toml file.
+ #
+ # `sysconfdir` is overloaded to be relative instead of absolute.
+ # This is the default of `install.sh`, but for whatever reason
+ # `x.py install` has its own default of `/etc` which we don't want.
+ #
+ # `missing-tools` is set so `rustfmt` is allowed to fail. This means
+ # we can "succeed" at building Rust while failing to build, say, Cargo.
+ # Ideally the build system would have better granularity:
+ # https://github.com/rust-lang/rust/issues/79249
+ base_config = textwrap.dedent(
+ """
+ [build]
+ docs = false
+ sanitizers = true
+ extended = true
+ tools = ["analysis", "cargo", "rustfmt", "clippy", "src", "rust-analyzer"]
+
+ [rust]
+ {omit_git_hash} = false
+ use-lld = true
+
+ [install]
+ prefix = "{prefix}"
+ sysconfdir = "etc"
+
+ [dist]
+ missing-tools = true
+
+ [llvm]
+ download-ci-llvm = false
+ """.format(
+ prefix=install_dir,
+ omit_git_hash=omit_git_hash,
+ )
+ )
+
+ # Rust requires these to be specified per-target
+ target_config = textwrap.dedent(
+ """
+ [target.{target}]
+ cc = "clang"
+ cxx = "clang++"
+ linker = "clang"
+
+ """
+ )
+
+ final_config = base_config
+ for target in sorted(set(targets) | set([host])):
+ final_config = final_config + target_config.format(target=target)
+
+ with open(os.path.join(rust_dir, "config.toml"), "w") as file:
+ file.write(final_config)
+
+ # Setup the env so compilers and toolchains are visible
+ clang = os.path.join(fetches, "clang")
+ clang_bin = os.path.join(clang, "bin")
+ clang_lib = os.path.join(clang, "lib")
+
+ env = os.environ.copy()
+ env.update(
+ {
+ "PATH": os.pathsep.join((clang_bin, os.environ["PATH"])),
+ "LD_LIBRARY_PATH": clang_lib,
+ }
+ )
+
+ # x.py install does everything we need for us.
+ # If you're running into issues, consider using `-vv` to debug it.
+ command = ["python3", "x.py", "install", "-v", "--host", host]
+ for target in targets:
+ command.extend(["--target", target])
+
+ subprocess.check_call(command, stderr=subprocess.STDOUT, env=env, cwd=rust_dir)
+
+
+def repack(
+ host,
+ targets,
+ channel="stable",
+ cargo_channel=None,
+ patches=[],
+):
+ install_dir = "rustc"
+ if channel == "dev":
+ build_src(install_dir, host, targets, patches)
+ else:
+ if patches:
+ raise ValueError(
+ 'Patch specified, but channel "%s" is not "dev"!'
+ "\nPatches are only for building from source." % channel
+ )
+ log("Repacking rust for %s supporting %s..." % (host, targets))
+ manifest = fetch_manifest(channel, host, targets)
+ log("Using manifest for rust %s as of %s." % (channel, manifest["date"]))
+ if cargo_channel == channel:
+ cargo_manifest = manifest
+ else:
+ cargo_manifest = fetch_manifest(cargo_channel, host, targets)
+ log(
+ "Using manifest for cargo %s as of %s."
+ % (cargo_channel, cargo_manifest["date"])
+ )
+
+ log("Fetching packages...")
+ rustc = fetch_package(manifest, "rustc", host)
+ cargo = fetch_package(cargo_manifest, "cargo", host)
+ stds = fetch_std(manifest, targets)
+ rustsrc = fetch_package(manifest, "rust-src", host)
+ rustfmt = fetch_optional(manifest, "rustfmt-preview", host)
+ clippy = fetch_optional(manifest, "clippy-preview", host)
+ rust_analyzer = fetch_optional(manifest, "rust-analyzer-preview", host)
+
+ log("Installing packages...")
+
+ # Clear any previous install directory.
+ try:
+ shutil.rmtree(install_dir)
+ except OSError as e:
+ if e.errno != errno.ENOENT:
+ raise
+ install(os.path.basename(rustc["url"]), install_dir)
+ install(os.path.basename(cargo["url"]), install_dir)
+ install(os.path.basename(rustsrc["url"]), install_dir)
+ if rustfmt:
+ install(os.path.basename(rustfmt["url"]), install_dir)
+ if clippy:
+ install(os.path.basename(clippy["url"]), install_dir)
+ if rust_analyzer:
+ install(os.path.basename(rust_analyzer["url"]), install_dir)
+ for std in stds:
+ install(os.path.basename(std["url"]), install_dir)
+ pass
+
+ log("Creating archive...")
+ tar_file = install_dir + ".tar.zst"
+ build_tar_package(tar_file, ".", install_dir)
+ shutil.rmtree(install_dir)
+ log("%s is ready." % tar_file)
+
+ upload_dir = os.environ.get("UPLOAD_DIR")
+ if upload_dir:
+ # Create the upload directory if it doesn't exist.
+ try:
+ log("Creating upload directory in %s..." % os.path.abspath(upload_dir))
+ os.makedirs(upload_dir)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+ # Move the tarball to the output directory for upload.
+ log("Moving %s to the upload directory..." % tar_file)
+ shutil.move(tar_file, upload_dir)
+
+
+def expand_platform(name):
+ """Expand a shortcut name to a full Rust platform string."""
+ platforms = {
+ "android": "armv7-linux-androideabi",
+ "android_x86": "i686-linux-android",
+ "android_x86-64": "x86_64-linux-android",
+ "android_aarch64": "aarch64-linux-android",
+ "linux64": "x86_64-unknown-linux-gnu",
+ "linux32": "i686-unknown-linux-gnu",
+ "mac": "x86_64-apple-darwin",
+ "macos": "x86_64-apple-darwin",
+ "mac64": "x86_64-apple-darwin",
+ "mac32": "i686-apple-darwin",
+ "win64": "x86_64-pc-windows-msvc",
+ "win32": "i686-pc-windows-msvc",
+ "mingw32": "i686-pc-windows-gnu",
+ }
+ return platforms.get(name, name)
+
+
+def validate_channel(channel):
+ """Require a specific release version.
+
+ Packaging from meta-channels, like `stable`, `beta`, or `nightly`
+ doesn't give repeatable output. Reject such channels."""
+ channel_prefixes = ("stable", "beta", "nightly")
+ if any([channel.startswith(c) for c in channel_prefixes]):
+ if "-" not in channel:
+ raise ValueError(
+ 'Generic channel "%s" specified!'
+ "\nPlease give a specific release version"
+ ' like "1.24.0" or "beta-2018-02-20".' % channel
+ )
+
+
+def args():
+ """Read command line arguments and return options."""
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--channel",
+ help="Release channel to use:"
+ " 1.xx.y, beta-yyyy-mm-dd,"
+ " nightly-yyyy-mm-dd,"
+ " bors-$rev (grab a build from rust's CI),"
+ " or dev (build from source).",
+ required=True,
+ )
+ parser.add_argument(
+ "--patch",
+ dest="patches",
+ action="append",
+ default=[],
+ help="apply the given patch file to a dev build."
+ " Patch files should be placed in /build/build-rust."
+ " Patches can be prefixed with `module-path:` to specify they"
+ " apply to that git submodule in the Rust source."
+ " e.g. `src/llvm-project:mypatch.diff` patches rust's llvm."
+ " Can be given more than once.",
+ )
+ parser.add_argument(
+ "--cargo-channel",
+ help="Release channel version to use for cargo."
+ " Defaults to the same as --channel.",
+ )
+ parser.add_argument(
+ "--host",
+ help="Host platform for the toolchain executable:"
+ " e.g. linux64 or aarch64-linux-android."
+ " Defaults to linux64.",
+ )
+ parser.add_argument(
+ "--target",
+ dest="targets",
+ action="append",
+ default=[],
+ help="Additional target platform to support:"
+ " e.g. linux32 or i686-pc-windows-gnu."
+ " can be given more than once.",
+ )
+ args = parser.parse_args()
+ if not args.cargo_channel:
+ args.cargo_channel = args.channel
+ validate_channel(args.channel)
+ validate_channel(args.cargo_channel)
+ if not args.host:
+ args.host = "linux64"
+ args.host = expand_platform(args.host)
+ args.targets = [expand_platform(t) for t in args.targets]
+
+ return args
+
+
+if __name__ == "__main__":
+ args = vars(args())
+ setup_gpg()
+ repack(**args)