Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

6 files changed, 1130 insertions, 0 deletions

diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml
new file mode 100644
index 000000000..c022d3aa0
--- /dev/null
+++ b/src/tools/build-manifest/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+name = "build-manifest"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+toml = "0.5"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+anyhow = "1.0.32"
+flate2 = "1.0.16"
+tar = "0.4.29"
+sha2 = "0.10.1"
+rayon = "1.5.1"
+hex = "0.4.2"
+num_cpus = "1.13.0"
diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md
new file mode 100644
index 000000000..44c96f31d
--- /dev/null
+++ b/src/tools/build-manifest/README.md
@@ -0,0 +1,27 @@
+# build-manifest
+
+This tool generates the manifests uploaded to static.rust-lang.org and used by
+rustup. The tool is invoked by the bootstrap tool.
+
+## Testing changes locally
+
+In order to test the changes locally you need to have a valid dist directory
+available locally. If you don't want to build all the compiler, you can easily
+create one from the nightly artifacts with:
+
+```
+#!/bin/bash
+for cmpn in rust rustc rust-std rust-docs cargo; do
+    wget https://static.rust-lang.org/dist/${cmpn}-nightly-x86_64-unknown-linux-gnu.tar.gz
+done
+```
+
+Then, you can generate the manifest and all the packages from `path/to/dist` to
+`path/to/output` with:
+
+```
+$ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com CHANNEL
+```
+
+Remember to replace `CHANNEL` with the channel you produced dist artifacts of
+and `VERSION` with the current Rust version.
diff --git a/src/tools/build-manifest/src/checksum.rs b/src/tools/build-manifest/src/checksum.rs
new file mode 100644
index 000000000..c019c7a2f
--- /dev/null
+++ b/src/tools/build-manifest/src/checksum.rs
@@ -0,0 +1,97 @@
+use crate::manifest::{FileHash, Manifest};
+use rayon::prelude::*;
+use sha2::{Digest, Sha256};
+use std::collections::{HashMap, HashSet};
+use std::error::Error;
+use std::fs::File;
+use std::io::BufReader;
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+use std::time::Instant;
+
+pub(crate) struct Checksums {
+    cache_path: Option<PathBuf>,
+    collected: Mutex<HashMap<PathBuf, String>>,
+}
+
+impl Checksums {
+    pub(crate) fn new() -> Result<Self, Box<dyn Error>> {
+        let cache_path = std::env::var_os("BUILD_MANIFEST_CHECKSUM_CACHE").map(PathBuf::from);
+
+        let mut collected = HashMap::new();
+        if let Some(path) = &cache_path {
+            if path.is_file() {
+                collected = serde_json::from_slice(&std::fs::read(path)?)?;
+            }
+        }
+
+        Ok(Checksums { cache_path, collected: Mutex::new(collected) })
+    }
+
+    pub(crate) fn store_cache(&self) -> Result<(), Box<dyn Error>> {
+        if let Some(path) = &self.cache_path {
+            std::fs::write(path, &serde_json::to_vec(&self.collected)?)?;
+        }
+        Ok(())
+    }
+
+    pub(crate) fn fill_missing_checksums(&mut self, manifest: &mut Manifest) {
+        let need_checksums = self.find_missing_checksums(manifest);
+        if !need_checksums.is_empty() {
+            self.collect_checksums(&need_checksums);
+        }
+        self.replace_checksums(manifest);
+    }
+
+    fn find_missing_checksums(&mut self, manifest: &mut Manifest) -> HashSet<PathBuf> {
+        let collected = self.collected.lock().unwrap();
+        let mut need_checksums = HashSet::new();
+        crate::manifest::visit_file_hashes(manifest, |file_hash| {
+            if let FileHash::Missing(path) = file_hash {
+                let path = std::fs::canonicalize(path).unwrap();
+                if !collected.contains_key(&path) {
+                    need_checksums.insert(path);
+                }
+            }
+        });
+        need_checksums
+    }
+
+    fn replace_checksums(&mut self, manifest: &mut Manifest) {
+        let collected = self.collected.lock().unwrap();
+        crate::manifest::visit_file_hashes(manifest, |file_hash| {
+            if let FileHash::Missing(path) = file_hash {
+                let path = std::fs::canonicalize(path).unwrap();
+                match collected.get(&path) {
+                    Some(hash) => *file_hash = FileHash::Present(hash.clone()),
+                    None => panic!("missing hash for file {}", path.display()),
+                }
+            }
+        });
+    }
+
+    fn collect_checksums(&mut self, files: &HashSet<PathBuf>) {
+        let collection_start = Instant::now();
+        println!(
+            "collecting hashes for {} tarballs across {} threads",
+            files.len(),
+            rayon::current_num_threads().min(files.len()),
+        );
+
+        files.par_iter().for_each(|path| match hash(path) {
+            Ok(hash) => {
+                self.collected.lock().unwrap().insert(path.clone(), hash);
+            }
+            Err(err) => eprintln!("error while fetching the hash for {}: {}", path.display(), err),
+        });
+
+        println!("collected {} hashes in {:.2?}", files.len(), collection_start.elapsed());
+    }
+}
+
+fn hash(path: &Path) -> Result<String, Box<dyn Error>> {
+    let mut file = BufReader::new(File::open(path)?);
+    let mut sha256 = Sha256::default();
+    std::io::copy(&mut file, &mut sha256)?;
+    Ok(hex::encode(sha256.finalize()))
+}
diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs
new file mode 100644
index 000000000..efe3f2b61
--- /dev/null
+++ b/src/tools/build-manifest/src/main.rs
@@ -0,0 +1,608 @@
+//! Build a dist manifest, hash and sign everything.
+//! This gets called by `promote-release`
+//! (https://github.com/rust-lang/rust-central-station/tree/master/promote-release)
+//! via `x.py dist hash-and-sign`; the cmdline arguments are set up
+//! by rustbuild (in `src/bootstrap/dist.rs`).
+
+mod checksum;
+mod manifest;
+mod versions;
+
+use crate::checksum::Checksums;
+use crate::manifest::{Component, Manifest, Package, Rename, Target};
+use crate::versions::{PkgType, Versions};
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::env;
+use std::fs::{self, File};
+use std::path::{Path, PathBuf};
+
+static HOSTS: &[&str] = &[
+    "aarch64-apple-darwin",
+    "aarch64-pc-windows-msvc",
+    "aarch64-unknown-linux-gnu",
+    "aarch64-unknown-linux-musl",
+    "arm-unknown-linux-gnueabi",
+    "arm-unknown-linux-gnueabihf",
+    "armv7-unknown-linux-gnueabihf",
+    "i686-apple-darwin",
+    "i686-pc-windows-gnu",
+    "i686-pc-windows-msvc",
+    "i686-unknown-linux-gnu",
+    "mips-unknown-linux-gnu",
+    "mips64-unknown-linux-gnuabi64",
+    "mips64el-unknown-linux-gnuabi64",
+    "mipsel-unknown-linux-gnu",
+    "mipsisa32r6-unknown-linux-gnu",
+    "mipsisa32r6el-unknown-linux-gnu",
+    "mipsisa64r6-unknown-linux-gnuabi64",
+    "mipsisa64r6el-unknown-linux-gnuabi64",
+    "powerpc-unknown-linux-gnu",
+    "powerpc64-unknown-linux-gnu",
+    "powerpc64le-unknown-linux-gnu",
+    "riscv64gc-unknown-linux-gnu",
+    "s390x-unknown-linux-gnu",
+    "x86_64-apple-darwin",
+    "x86_64-pc-windows-gnu",
+    "x86_64-pc-windows-msvc",
+    "x86_64-unknown-freebsd",
+    "x86_64-unknown-illumos",
+    "x86_64-unknown-linux-gnu",
+    "x86_64-unknown-linux-musl",
+    "x86_64-unknown-netbsd",
+];
+
+static TARGETS: &[&str] = &[
+    "aarch64-apple-darwin",
+    "aarch64-apple-ios",
+    "aarch64-apple-ios-sim",
+    "aarch64-fuchsia",
+    "aarch64-linux-android",
+    "aarch64-pc-windows-msvc",
+    "aarch64-unknown-hermit",
+    "aarch64-unknown-linux-gnu",
+    "aarch64-unknown-linux-musl",
+    "aarch64-unknown-none",
+    "aarch64-unknown-none-softfloat",
+    "aarch64-unknown-redox",
+    "arm-linux-androideabi",
+    "arm-unknown-linux-gnueabi",
+    "arm-unknown-linux-gnueabihf",
+    "arm-unknown-linux-musleabi",
+    "arm-unknown-linux-musleabihf",
+    "armv5te-unknown-linux-gnueabi",
+    "armv5te-unknown-linux-musleabi",
+    "armv7-apple-ios",
+    "armv7-linux-androideabi",
+    "thumbv7neon-linux-androideabi",
+    "armv7-unknown-linux-gnueabi",
+    "armv7-unknown-linux-gnueabihf",
+    "armv7a-none-eabi",
+    "thumbv7neon-unknown-linux-gnueabihf",
+    "armv7-unknown-linux-musleabi",
+    "armv7-unknown-linux-musleabihf",
+    "armebv7r-none-eabi",
+    "armebv7r-none-eabihf",
+    "armv7r-none-eabi",
+    "armv7r-none-eabihf",
+    "armv7s-apple-ios",
+    "asmjs-unknown-emscripten",
+    "bpfeb-unknown-none",
+    "bpfel-unknown-none",
+    "i386-apple-ios",
+    "i586-pc-windows-msvc",
+    "i586-unknown-linux-gnu",
+    "i586-unknown-linux-musl",
+    "i686-apple-darwin",
+    "i686-linux-android",
+    "i686-pc-windows-gnu",
+    "i686-pc-windows-msvc",
+    "i686-unknown-freebsd",
+    "i686-unknown-linux-gnu",
+    "i686-unknown-linux-musl",
+    "m68k-unknown-linux-gnu",
+    "mips-unknown-linux-gnu",
+    "mips-unknown-linux-musl",
+    "mips64-unknown-linux-gnuabi64",
+    "mips64-unknown-linux-muslabi64",
+    "mips64el-unknown-linux-gnuabi64",
+    "mips64el-unknown-linux-muslabi64",
+    "mipsisa32r6-unknown-linux-gnu",
+    "mipsisa32r6el-unknown-linux-gnu",
+    "mipsisa64r6-unknown-linux-gnuabi64",
+    "mipsisa64r6el-unknown-linux-gnuabi64",
+    "mipsel-unknown-linux-gnu",
+    "mipsel-unknown-linux-musl",
+    "nvptx64-nvidia-cuda",
+    "powerpc-unknown-linux-gnu",
+    "powerpc64-unknown-linux-gnu",
+    "powerpc64le-unknown-linux-gnu",
+    "riscv32i-unknown-none-elf",
+    "riscv32im-unknown-none-elf",
+    "riscv32imc-unknown-none-elf",
+    "riscv32imac-unknown-none-elf",
+    "riscv32gc-unknown-linux-gnu",
+    "riscv64imac-unknown-none-elf",
+    "riscv64gc-unknown-none-elf",
+    "riscv64gc-unknown-linux-gnu",
+    "s390x-unknown-linux-gnu",
+    "sparc64-unknown-linux-gnu",
+    "sparcv9-sun-solaris",
+    "thumbv6m-none-eabi",
+    "thumbv7em-none-eabi",
+    "thumbv7em-none-eabihf",
+    "thumbv7m-none-eabi",
+    "thumbv8m.base-none-eabi",
+    "thumbv8m.main-none-eabi",
+    "thumbv8m.main-none-eabihf",
+    "wasm32-unknown-emscripten",
+    "wasm32-unknown-unknown",
+    "wasm32-wasi",
+    "x86_64-apple-darwin",
+    "x86_64-apple-ios",
+    "x86_64-fortanix-unknown-sgx",
+    "x86_64-fuchsia",
+    "x86_64-linux-android",
+    "x86_64-pc-windows-gnu",
+    "x86_64-pc-windows-msvc",
+    "x86_64-sun-solaris",
+    "x86_64-pc-solaris",
+    "x86_64-unknown-freebsd",
+    "x86_64-unknown-illumos",
+    "x86_64-unknown-linux-gnu",
+    "x86_64-unknown-linux-gnux32",
+    "x86_64-unknown-linux-musl",
+    "x86_64-unknown-netbsd",
+    "x86_64-unknown-none",
+    "x86_64-unknown-redox",
+    "x86_64-unknown-hermit",
+];
+
+/// This allows the manifest to contain rust-docs for hosts that don't build
+/// docs.
+///
+/// Tuples of `(host_partial, host_instead)`. If the host does not have the
+/// rust-docs component available, then if the host name contains
+/// `host_partial`, it will use the docs from `host_instead` instead.
+///
+/// The order here matters, more specific entries should be first.
+static DOCS_FALLBACK: &[(&str, &str)] = &[
+    ("-apple-", "x86_64-apple-darwin"),
+    ("aarch64", "aarch64-unknown-linux-gnu"),
+    ("arm-", "aarch64-unknown-linux-gnu"),
+    ("", "x86_64-unknown-linux-gnu"),
+];
+
+static MSI_INSTALLERS: &[&str] = &[
+    "aarch64-pc-windows-msvc",
+    "i686-pc-windows-gnu",
+    "i686-pc-windows-msvc",
+    "x86_64-pc-windows-gnu",
+    "x86_64-pc-windows-msvc",
+];
+
+static PKG_INSTALLERS: &[&str] = &["x86_64-apple-darwin", "aarch64-apple-darwin"];
+
+static MINGW: &[&str] = &["i686-pc-windows-gnu", "x86_64-pc-windows-gnu"];
+
+static NIGHTLY_ONLY_COMPONENTS: &[&str] = &["miri-preview"];
+
+macro_rules! t {
+    ($e:expr) => {
+        match $e {
+            Ok(e) => e,
+            Err(e) => panic!("{} failed with {}", stringify!($e), e),
+        }
+    };
+}
+
+struct Builder {
+    versions: Versions,
+    checksums: Checksums,
+    shipped_files: HashSet<String>,
+
+    input: PathBuf,
+    output: PathBuf,
+    s3_address: String,
+    date: String,
+}
+
+fn main() {
+    let num_threads = if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") {
+        num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS")
+    } else {
+        num_cpus::get()
+    };
+    rayon::ThreadPoolBuilder::new()
+        .num_threads(num_threads)
+        .build_global()
+        .expect("failed to initialize Rayon");
+
+    let mut args = env::args().skip(1);
+    let input = PathBuf::from(args.next().unwrap());
+    let output = PathBuf::from(args.next().unwrap());
+    let date = args.next().unwrap();
+    let s3_address = args.next().unwrap();
+    let channel = args.next().unwrap();
+
+    Builder {
+        versions: Versions::new(&channel, &input).unwrap(),
+        checksums: t!(Checksums::new()),
+        shipped_files: HashSet::new(),
+
+        input,
+        output,
+        s3_address,
+        date,
+    }
+    .build();
+}
+
+impl Builder {
+    fn build(&mut self) {
+        self.check_toolstate();
+        let manifest = self.build_manifest();
+
+        let channel = self.versions.channel().to_string();
+        self.write_channel_files(&channel, &manifest);
+        if channel == "stable" {
+            // channel-rust-1.XX.YY.toml
+            let rust_version = self.versions.rustc_version().to_string();
+            self.write_channel_files(&rust_version, &manifest);
+
+            // channel-rust-1.XX.toml
+            let major_minor = rust_version.split('.').take(2).collect::<Vec<_>>().join(".");
+            self.write_channel_files(&major_minor, &manifest);
+        }
+
+        if let Some(path) = std::env::var_os("BUILD_MANIFEST_SHIPPED_FILES_PATH") {
+            self.write_shipped_files(&Path::new(&path));
+        }
+
+        t!(self.checksums.store_cache());
+    }
+
+    /// If a tool does not pass its tests on *any* of Linux and Windows, don't ship
+    /// it on *all* targets, because tools like Miri can "cross-run" programs for
+    /// different targets, for example, run a program for `x86_64-pc-windows-msvc`
+    /// on `x86_64-unknown-linux-gnu`.
+    /// Right now, we do this only for Miri.
+    fn check_toolstate(&mut self) {
+        for file in &["toolstates-linux.json", "toolstates-windows.json"] {
+            let toolstates: Option<HashMap<String, String>> = File::open(self.input.join(file))
+                .ok()
+                .and_then(|f| serde_json::from_reader(&f).ok());
+            let toolstates = toolstates.unwrap_or_else(|| {
+                println!("WARNING: `{}` missing/malformed; assuming all tools failed", file);
+                HashMap::default() // Use empty map if anything went wrong.
+            });
+            // Mark some tools as missing based on toolstate.
+            if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") {
+                println!("Miri tests are not passing, removing component");
+                self.versions.disable_version(&PkgType::Miri);
+                break;
+            }
+        }
+    }
+
+    fn build_manifest(&mut self) -> Manifest {
+        let mut manifest = Manifest {
+            manifest_version: "2".to_string(),
+            date: self.date.to_string(),
+            pkg: BTreeMap::new(),
+            artifacts: BTreeMap::new(),
+            renames: BTreeMap::new(),
+            profiles: BTreeMap::new(),
+        };
+        self.add_packages_to(&mut manifest);
+        self.add_artifacts_to(&mut manifest);
+        self.add_profiles_to(&mut manifest);
+        self.add_renames_to(&mut manifest);
+        manifest.pkg.insert("rust".to_string(), self.rust_package(&manifest));
+
+        self.checksums.fill_missing_checksums(&mut manifest);
+
+        manifest
+    }
+
+    fn add_packages_to(&mut self, manifest: &mut Manifest) {
+        macro_rules! package {
+            ($name:expr, $targets:expr) => {
+                self.package($name, &mut manifest.pkg, $targets, &[])
+            };
+        }
+        package!("rustc", HOSTS);
+        package!("rustc-dev", HOSTS);
+        package!("reproducible-artifacts", HOSTS);
+        package!("rustc-docs", HOSTS);
+        package!("cargo", HOSTS);
+        package!("rust-mingw", MINGW);
+        package!("rust-std", TARGETS);
+        self.package("rust-docs", &mut manifest.pkg, HOSTS, DOCS_FALLBACK);
+        package!("rust-src", &["*"]);
+        package!("rls-preview", HOSTS);
+        package!("rust-analyzer-preview", HOSTS);
+        package!("clippy-preview", HOSTS);
+        package!("miri-preview", HOSTS);
+        package!("rustfmt-preview", HOSTS);
+        package!("rust-analysis", TARGETS);
+        package!("llvm-tools-preview", TARGETS);
+    }
+
+    fn add_artifacts_to(&mut self, manifest: &mut Manifest) {
+        manifest.add_artifact("source-code", |artifact| {
+            let tarball = self.versions.tarball_name(&PkgType::Rustc, "src").unwrap();
+            artifact.add_tarball(self, "*", &tarball);
+        });
+
+        manifest.add_artifact("installer-msi", |artifact| {
+            for target in MSI_INSTALLERS {
+                let msi = self.versions.archive_name(&PkgType::Rust, target, "msi").unwrap();
+                artifact.add_file(self, target, &msi);
+            }
+        });
+
+        manifest.add_artifact("installer-pkg", |artifact| {
+            for target in PKG_INSTALLERS {
+                let pkg = self.versions.archive_name(&PkgType::Rust, target, "pkg").unwrap();
+                artifact.add_file(self, target, &pkg);
+            }
+        });
+    }
+
+    fn add_profiles_to(&mut self, manifest: &mut Manifest) {
+        let mut profile = |name, pkgs| self.profile(name, &mut manifest.profiles, pkgs);
+        profile("minimal", &["rustc", "cargo", "rust-std", "rust-mingw"]);
+        profile(
+            "default",
+            &[
+                "rustc",
+                "cargo",
+                "rust-std",
+                "rust-mingw",
+                "rust-docs",
+                "rustfmt-preview",
+                "clippy-preview",
+            ],
+        );
+        profile(
+            "complete",
+            &[
+                "rustc",
+                "cargo",
+                "rust-std",
+                "rust-mingw",
+                "rust-docs",
+                "rustfmt-preview",
+                "clippy-preview",
+                "rls-preview",
+                "rust-analyzer-preview",
+                "rust-src",
+                "llvm-tools-preview",
+                "rust-analysis",
+                "miri-preview",
+            ],
+        );
+
+        // The compiler libraries are not stable for end users, and they're also huge, so we only
+        // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible
+        // for users to install the additional component manually, if needed.
+        if self.versions.channel() == "nightly" {
+            self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]);
+            // Do not include the rustc-docs component for now, as it causes
+            // conflicts with the rust-docs component when installed. See
+            // #75833.
+            // self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]);
+        }
+    }
+
+    fn add_renames_to(&self, manifest: &mut Manifest) {
+        let mut rename = |from: &str, to: &str| {
+            manifest.renames.insert(from.to_owned(), Rename { to: to.to_owned() })
+        };
+        rename("rls", "rls-preview");
+        rename("rustfmt", "rustfmt-preview");
+        rename("clippy", "clippy-preview");
+        rename("miri", "miri-preview");
+        rename("rust-analyzer", "rust-analyzer-preview");
+    }
+
+    fn rust_package(&mut self, manifest: &Manifest) -> Package {
+        let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball");
+        let mut pkg = Package {
+            version: version_info.version.expect("missing Rust version"),
+            git_commit_hash: version_info.git_commit,
+            target: BTreeMap::new(),
+        };
+        for host in HOSTS {
+            if let Some(target) = self.target_host_combination(host, &manifest) {
+                pkg.target.insert(host.to_string(), target);
+            } else {
+                pkg.target.insert(host.to_string(), Target::unavailable());
+                continue;
+            }
+        }
+        pkg
+    }
+
+    fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option<Target> {
+        let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap();
+
+        let mut target = Target::from_compressed_tar(self, &filename);
+        if !target.available {
+            return None;
+        }
+
+        let mut components = Vec::new();
+        let mut extensions = Vec::new();
+
+        let host_component = |pkg| Component::from_str(pkg, host);
+
+        // rustc/rust-std/cargo/docs are all required,
+        // and so is rust-mingw if it's available for the target.
+        components.extend(vec![
+            host_component("rustc"),
+            host_component("rust-std"),
+            host_component("cargo"),
+            host_component("rust-docs"),
+        ]);
+        if host.contains("pc-windows-gnu") {
+            components.push(host_component("rust-mingw"));
+        }
+
+        // Tools are always present in the manifest,
+        // but might be marked as unavailable if they weren't built.
+        extensions.extend(vec![
+            host_component("clippy-preview"),
+            host_component("miri-preview"),
+            host_component("rls-preview"),
+            host_component("rust-analyzer-preview"),
+            host_component("rustfmt-preview"),
+            host_component("llvm-tools-preview"),
+            host_component("rust-analysis"),
+        ]);
+
+        extensions.extend(
+            TARGETS
+                .iter()
+                .filter(|&&target| target != host)
+                .map(|target| Component::from_str("rust-std", target)),
+        );
+        extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-dev", target)));
+        extensions.extend(HOSTS.iter().map(|target| Component::from_str("rustc-docs", target)));
+        extensions.push(Component::from_str("rust-src", "*"));
+
+        // If the components/extensions don't actually exist for this
+        // particular host/target combination then nix it entirely from our
+        // lists.
+        let has_component = |c: &Component| {
+            if c.target == "*" {
+                return true;
+            }
+            let pkg = match manifest.pkg.get(&c.pkg) {
+                Some(p) => p,
+                None => return false,
+            };
+            pkg.target.get(&c.target).is_some()
+        };
+        extensions.retain(&has_component);
+        components.retain(&has_component);
+
+        target.components = Some(components);
+        target.extensions = Some(extensions);
+        Some(target)
+    }
+
+    fn profile(
+        &mut self,
+        profile_name: &str,
+        dst: &mut BTreeMap<String, Vec<String>>,
+        pkgs: &[&str],
+    ) {
+        dst.insert(profile_name.to_owned(), pkgs.iter().map(|s| (*s).to_owned()).collect());
+    }
+
+    fn extend_profile(
+        &mut self,
+        profile_name: &str,
+        dst: &mut BTreeMap<String, Vec<String>>,
+        pkgs: &[&str],
+    ) {
+        dst.get_mut(profile_name)
+            .expect("existing profile")
+            .extend(pkgs.iter().map(|s| (*s).to_owned()));
+    }
+
+    fn package(
+        &mut self,
+        pkgname: &str,
+        dst: &mut BTreeMap<String, Package>,
+        targets: &[&str],
+        fallback: &[(&str, &str)],
+    ) {
+        let version_info = self
+            .versions
+            .version(&PkgType::from_component(pkgname))
+            .expect("failed to load package version");
+        let mut is_present = version_info.present;
+
+        // Never ship nightly-only components for other trains.
+        if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) {
+            is_present = false; // Pretend the component is entirely missing.
+        }
+
+        macro_rules! tarball_name {
+            ($target_name:expr) => {
+                self.versions.tarball_name(&PkgType::from_component(pkgname), $target_name).unwrap()
+            };
+        }
+        let mut target_from_compressed_tar = |target_name| {
+            let target = Target::from_compressed_tar(self, &tarball_name!(target_name));
+            if target.available {
+                return target;
+            }
+            for (substr, fallback_target) in fallback {
+                if target_name.contains(substr) {
+                    let t = Target::from_compressed_tar(self, &tarball_name!(fallback_target));
+                    // Fallbacks must always be available.
+                    assert!(t.available);
+                    return t;
+                }
+            }
+            Target::unavailable()
+        };
+
+        let targets = targets
+            .iter()
+            .map(|name| {
+                let target = if is_present {
+                    target_from_compressed_tar(name)
+                } else {
+                    // If the component is not present for this build add it anyway but mark it as
+                    // unavailable -- this way rustup won't allow upgrades without --force
+                    Target::unavailable()
+                };
+                (name.to_string(), target)
+            })
+            .collect();
+
+        dst.insert(
+            pkgname.to_string(),
+            Package {
+                version: version_info.version.unwrap_or_default(),
+                git_commit_hash: version_info.git_commit,
+                target: targets,
+            },
+        );
+    }
+
+    fn url(&self, path: &Path) -> String {
+        let file_name = path.file_name().unwrap().to_str().unwrap();
+        format!("{}/{}/{}", self.s3_address, self.date, file_name)
+    }
+
+    fn write_channel_files(&mut self, channel_name: &str, manifest: &Manifest) {
+        self.write(&toml::to_string(&manifest).unwrap(), channel_name, ".toml");
+        self.write(&manifest.date, channel_name, "-date.txt");
+        self.write(
+            manifest.pkg["rust"].git_commit_hash.as_ref().unwrap(),
+            channel_name,
+            "-git-commit-hash.txt",
+        );
+    }
+
+    fn write(&mut self, contents: &str, channel_name: &str, suffix: &str) {
+        let name = format!("channel-rust-{}{}", channel_name, suffix);
+        self.shipped_files.insert(name.clone());
+
+        let dst = self.output.join(name);
+        t!(fs::write(&dst, contents));
+    }
+
+    fn write_shipped_files(&self, path: &Path) {
+        let mut files = self.shipped_files.iter().map(|s| s.as_str()).collect::<Vec<_>>();
+        files.sort();
+        let content = format!("{}\n", files.join("\n"));
+
+        t!(std::fs::write(path, content.as_bytes()));
+    }
+}
diff --git a/src/tools/build-manifest/src/manifest.rs b/src/tools/build-manifest/src/manifest.rs
new file mode 100644
index 000000000..547c270d8
--- /dev/null
+++ b/src/tools/build-manifest/src/manifest.rs
@@ -0,0 +1,182 @@
+use crate::Builder;
+use serde::{Serialize, Serializer};
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+#[derive(Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) struct Manifest {
+    pub(crate) manifest_version: String,
+    pub(crate) date: String,
+    pub(crate) pkg: BTreeMap<String, Package>,
+    pub(crate) artifacts: BTreeMap<String, Artifact>,
+    pub(crate) renames: BTreeMap<String, Rename>,
+    pub(crate) profiles: BTreeMap<String, Vec<String>>,
+}
+
+impl Manifest {
+    pub(crate) fn add_artifact(&mut self, name: &str, f: impl FnOnce(&mut Artifact)) {
+        let mut artifact = Artifact { target: BTreeMap::new() };
+        f(&mut artifact);
+        self.artifacts.insert(name.to_string(), artifact);
+    }
+}
+
+#[derive(Serialize)]
+pub(crate) struct Package {
+    pub(crate) version: String,
+    pub(crate) git_commit_hash: Option<String>,
+    pub(crate) target: BTreeMap<String, Target>,
+}
+
+#[derive(Serialize)]
+pub(crate) struct Rename {
+    pub(crate) to: String,
+}
+
+#[derive(Serialize)]
+pub(crate) struct Artifact {
+    pub(crate) target: BTreeMap<String, Vec<ArtifactFile>>,
+}
+
+impl Artifact {
+    pub(crate) fn add_file(&mut self, builder: &mut Builder, target: &str, path: &str) {
+        if let Some(path) = record_shipped_file(builder, builder.input.join(path)) {
+            self.target.entry(target.into()).or_insert_with(Vec::new).push(ArtifactFile {
+                url: builder.url(&path),
+                hash_sha256: FileHash::Missing(path),
+            });
+        }
+    }
+
+    pub(crate) fn add_tarball(&mut self, builder: &mut Builder, target: &str, base_path: &str) {
+        let files = self.target.entry(target.into()).or_insert_with(Vec::new);
+        let base_path = builder.input.join(base_path);
+        for compression in &["gz", "xz"] {
+            if let Some(tarball) = tarball_variant(builder, &base_path, compression) {
+                files.push(ArtifactFile {
+                    url: builder.url(&tarball),
+                    hash_sha256: FileHash::Missing(tarball),
+                });
+            }
+        }
+    }
+}
+
+#[derive(Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) struct ArtifactFile {
+    pub(crate) url: String,
+    pub(crate) hash_sha256: FileHash,
+}
+
+#[derive(Serialize, Default)]
+pub(crate) struct Target {
+    pub(crate) available: bool,
+    pub(crate) url: Option<String>,
+    pub(crate) hash: Option<FileHash>,
+    pub(crate) xz_url: Option<String>,
+    pub(crate) xz_hash: Option<FileHash>,
+    pub(crate) components: Option<Vec<Component>>,
+    pub(crate) extensions: Option<Vec<Component>>,
+}
+
+impl Target {
+    pub(crate) fn from_compressed_tar(builder: &mut Builder, base_path: &str) -> Self {
+        let base_path = builder.input.join(base_path);
+        let gz = tarball_variant(builder, &base_path, "gz");
+        let xz = tarball_variant(builder, &base_path, "xz");
+
+        if gz.is_none() {
+            return Self::unavailable();
+        }
+
+        Self {
+            available: true,
+            components: None,
+            extensions: None,
+            // .gz
+            url: gz.as_ref().map(|path| builder.url(path)),
+            hash: gz.map(FileHash::Missing),
+            // .xz
+            xz_url: xz.as_ref().map(|path| builder.url(path)),
+            xz_hash: xz.map(FileHash::Missing),
+        }
+    }
+
+    pub(crate) fn unavailable() -> Self {
+        Self::default()
+    }
+}
+
+#[derive(Serialize)]
+pub(crate) struct Component {
+    pub(crate) pkg: String,
+    pub(crate) target: String,
+}
+
+impl Component {
+    pub(crate) fn from_str(pkg: &str, target: &str) -> Self {
+        Self { pkg: pkg.to_string(), target: target.to_string() }
+    }
+}
+
+#[allow(unused)]
+pub(crate) enum FileHash {
+    Missing(PathBuf),
+    Present(String),
+}
+
+impl Serialize for FileHash {
+    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        match self {
+            FileHash::Missing(path) => Err(serde::ser::Error::custom(format!(
+                "can't serialize a missing hash for file {}",
+                path.display()
+            ))),
+            FileHash::Present(inner) => inner.serialize(serializer),
+        }
+    }
+}
+
+fn tarball_variant(builder: &mut Builder, base: &Path, ext: &str) -> Option<PathBuf> {
+    let mut path = base.to_path_buf();
+    path.set_extension(ext);
+    record_shipped_file(builder, path)
+}
+
+fn record_shipped_file(builder: &mut Builder, path: PathBuf) -> Option<PathBuf> {
+    if path.is_file() {
+        builder.shipped_files.insert(
+            path.file_name()
+                .expect("missing filename")
+                .to_str()
+                .expect("non-utf-8 filename")
+                .to_string(),
+        );
+        Some(path)
+    } else {
+        None
+    }
+}
+
+pub(crate) fn visit_file_hashes(manifest: &mut Manifest, mut f: impl FnMut(&mut FileHash)) {
+    for pkg in manifest.pkg.values_mut() {
+        for target in pkg.target.values_mut() {
+            if let Some(hash) = &mut target.hash {
+                f(hash);
+            }
+            if let Some(hash) = &mut target.xz_hash {
+                f(hash);
+            }
+        }
+    }
+
+    for artifact in manifest.artifacts.values_mut() {
+        for target in artifact.target.values_mut() {
+            for file in target {
+                f(&mut file.hash_sha256);
+            }
+        }
+    }
+}
diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs
new file mode 100644
index 000000000..95c2297de
--- /dev/null
+++ b/src/tools/build-manifest/src/versions.rs
@@ -0,0 +1,200 @@
+use anyhow::Error;
+use flate2::read::GzDecoder;
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::Read;
+use std::path::{Path, PathBuf};
+use tar::Archive;
+
+const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu";
+
+#[derive(Debug, Hash, Eq, PartialEq, Clone)]
+pub(crate) enum PkgType {
+    Rust,
+    RustSrc,
+    Rustc,
+    Cargo,
+    Rls,
+    RustAnalyzer,
+    Clippy,
+    Rustfmt,
+    LlvmTools,
+    Miri,
+    Other(String),
+}
+
+impl PkgType {
+    pub(crate) fn from_component(component: &str) -> Self {
+        match component {
+            "rust" => PkgType::Rust,
+            "rust-src" => PkgType::RustSrc,
+            "rustc" => PkgType::Rustc,
+            "cargo" => PkgType::Cargo,
+            "rls" | "rls-preview" => PkgType::Rls,
+            "rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer,
+            "clippy" | "clippy-preview" => PkgType::Clippy,
+            "rustfmt" | "rustfmt-preview" => PkgType::Rustfmt,
+            "llvm-tools" | "llvm-tools-preview" => PkgType::LlvmTools,
+            "miri" | "miri-preview" => PkgType::Miri,
+            other => PkgType::Other(other.into()),
+        }
+    }
+
+    /// First part of the tarball name.
+    fn tarball_component_name(&self) -> &str {
+        match self {
+            PkgType::Rust => "rust",
+            PkgType::RustSrc => "rust-src",
+            PkgType::Rustc => "rustc",
+            PkgType::Cargo => "cargo",
+            PkgType::Rls => "rls",
+            PkgType::RustAnalyzer => "rust-analyzer",
+            PkgType::Clippy => "clippy",
+            PkgType::Rustfmt => "rustfmt",
+            PkgType::LlvmTools => "llvm-tools",
+            PkgType::Miri => "miri",
+            PkgType::Other(component) => component,
+        }
+    }
+
+    /// Whether this package has the same version as Rust itself, or has its own `version` and
+    /// `git-commit-hash` files inside the tarball.
+    fn should_use_rust_version(&self) -> bool {
+        match self {
+            PkgType::Cargo => false,
+            PkgType::Rls => false,
+            PkgType::RustAnalyzer => false,
+            PkgType::Clippy => false,
+            PkgType::Rustfmt => false,
+            PkgType::LlvmTools => false,
+            PkgType::Miri => false,
+
+            PkgType::Rust => true,
+            PkgType::RustSrc => true,
+            PkgType::Rustc => true,
+            PkgType::Other(_) => true,
+        }
+    }
+
+    /// Whether this package is target-independent or not.
+    fn target_independent(&self) -> bool {
+        *self == PkgType::RustSrc
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+pub(crate) struct VersionInfo {
+    pub(crate) version: Option<String>,
+    pub(crate) git_commit: Option<String>,
+    pub(crate) present: bool,
+}
+
+pub(crate) struct Versions {
+    channel: String,
+    dist_path: PathBuf,
+    versions: HashMap<PkgType, VersionInfo>,
+}
+
+impl Versions {
+    pub(crate) fn new(channel: &str, dist_path: &Path) -> Result<Self, Error> {
+        Ok(Self { channel: channel.into(), dist_path: dist_path.into(), versions: HashMap::new() })
+    }
+
+    pub(crate) fn channel(&self) -> &str {
+        &self.channel
+    }
+
+    pub(crate) fn version(&mut self, mut package: &PkgType) -> Result<VersionInfo, Error> {
+        if package.should_use_rust_version() {
+            package = &PkgType::Rust;
+        }
+
+        match self.versions.get(package) {
+            Some(version) => Ok(version.clone()),
+            None => {
+                let version_info = self.load_version_from_tarball(package)?;
+                self.versions.insert(package.clone(), version_info.clone());
+                Ok(version_info)
+            }
+        }
+    }
+
+    fn load_version_from_tarball(&mut self, package: &PkgType) -> Result<VersionInfo, Error> {
+        let tarball_name = self.tarball_name(package, DEFAULT_TARGET)?;
+        let tarball = self.dist_path.join(tarball_name);
+
+        let file = match File::open(&tarball) {
+            Ok(file) => file,
+            Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
+                // Missing tarballs do not return an error, but return empty data.
+                return Ok(VersionInfo::default());
+            }
+            Err(err) => return Err(err.into()),
+        };
+        let mut tar = Archive::new(GzDecoder::new(file));
+
+        let mut version = None;
+        let mut git_commit = None;
+        for entry in tar.entries()? {
+            let mut entry = entry?;
+
+            let dest;
+            match entry.path()?.components().nth(1).and_then(|c| c.as_os_str().to_str()) {
+                Some("version") => dest = &mut version,
+                Some("git-commit-hash") => dest = &mut git_commit,
+                _ => continue,
+            }
+            let mut buf = String::new();
+            entry.read_to_string(&mut buf)?;
+            *dest = Some(buf);
+
+            // Short circuit to avoid reading the whole tar file if not necessary.
+            if version.is_some() && git_commit.is_some() {
+                break;
+            }
+        }
+
+        Ok(VersionInfo { version, git_commit, present: true })
+    }
+
+    pub(crate) fn disable_version(&mut self, package: &PkgType) {
+        match self.versions.get_mut(package) {
+            Some(version) => {
+                *version = VersionInfo::default();
+            }
+            None => {
+                self.versions.insert(package.clone(), VersionInfo::default());
+            }
+        }
+    }
+
+    pub(crate) fn archive_name(
+        &self,
+        package: &PkgType,
+        target: &str,
+        extension: &str,
+    ) -> Result<String, Error> {
+        let component_name = package.tarball_component_name();
+        let version = match self.channel.as_str() {
+            "stable" => self.rustc_version().into(),
+            "beta" => "beta".into(),
+            "nightly" => "nightly".into(),
+            _ => format!("{}-dev", self.rustc_version()),
+        };
+
+        if package.target_independent() {
+            Ok(format!("{}-{}.{}", component_name, version, extension))
+        } else {
+            Ok(format!("{}-{}-{}.{}", component_name, version, target, extension))
+        }
+    }
+
+    pub(crate) fn tarball_name(&self, package: &PkgType, target: &str) -> Result<String, Error> {
+        self.archive_name(package, target, "tar.gz")
+    }
+
+    pub(crate) fn rustc_version(&self) -> &str {
+        const RUSTC_VERSION: &str = include_str!("../../../version");
+        RUSTC_VERSION.trim()
+    }
+}